def test_create_np(cluster):
    test_id = "np1"
    org = create_org_object(api=cluster.api,
                            organization_name=test_id + ORG_NAME)
    org.obj["spec"]["resources"]["cpu"] = "10"
    org.obj["spec"]["resources"]["memory"] = "10G"
    org.create()
    time.sleep(TIMEOUT)

    tenant = create_tenant_object(api=cluster.api,
                                  organization_name=test_id + ORG_NAME,
                                  tenant_name=test_id + TENANT_NAME)
    tenant.obj["spec"]["resources"]["cpu"] = "5"
    tenant.obj["spec"]["resources"]["memory"] = "5G"
    tenant.create()
    time.sleep(TIMEOUT)

    space = create_space_object(api=cluster.api,
                                organization_name=test_id + ORG_NAME,
                                tenant_name=test_id + TENANT_NAME,
                                space_name=test_id + SPACE_NAME)
    space.obj["spec"]["resources"]["cpu"] = "1"
    space.obj["spec"]["resources"]["memory"] = "1G"
    space.create()
    time.sleep(TIMEOUT)

    namespace = Namespace.objects(cluster.api).get(
        name=space_namespacename_generator(organization_name=test_id +
                                           ORG_NAME,
                                           tenant_name=test_id + TENANT_NAME,
                                           space_name=test_id + SPACE_NAME))
    networkPolicies = NetworkPolicy.objects(cluster.api,
                                            namespace.name).filter()
    assert len(networkPolicies) == 1
def test_create_space_np(cluster):
    test_id = "np2"
    org = create_org_object(api=cluster.api,
                            organization_name=test_id + ORG_NAME)
    org.obj["spec"]["resources"]["cpu"] = "10"
    org.obj["spec"]["resources"]["memory"] = "10G"
    org.create()
    time.sleep(TIMEOUT)

    tenant = create_tenant_object(api=cluster.api,
                                  organization_name=test_id + ORG_NAME,
                                  tenant_name=test_id + TENANT_NAME)
    tenant.obj["spec"]["resources"]["cpu"] = "5"
    tenant.obj["spec"]["resources"]["memory"] = "5G"
    tenant.create()
    time.sleep(TIMEOUT)

    space = create_space_object(api=cluster.api,
                                organization_name=test_id + ORG_NAME,
                                tenant_name=test_id + TENANT_NAME,
                                space_name=test_id + SPACE_NAME)
    space.obj["spec"]["resources"]["cpu"] = "1"
    space.obj["spec"]["resources"]["memory"] = "1G"
    space.obj["spec"]["allowIncomingNetwork"] = {
        "organizations": [{
            "organization_name": "example"
        }],
        "tenants": [
            {
                "organization_name": "example",
                "tenant_name": "crm"
            },
            {
                "organization_name": "example",
                "tenant_name": "crm2"
            },
        ],
        "spaces": [{
            "organization_name": "example",
            "tenant_name": "crm",
            "space_name": "test"
        }]
    }

    space.create()
    time.sleep(TIMEOUT)

    namespace = Namespace.objects(cluster.api).get(
        name=space_namespacename_generator(organization_name=test_id +
                                           ORG_NAME,
                                           tenant_name=test_id + TENANT_NAME,
                                           space_name=test_id + SPACE_NAME))
    networkPolicies = list(
        NetworkPolicy.objects(cluster.api, namespace.name).iterator())
    assert len(networkPolicies) == 1

    print(networkPolicies[0])
    assert len(networkPolicies[0].ingress) == 5
Example #3
0
def clean_up(api, include_resources: frozenset, exclude_resources: frozenset,
             include_namespaces: frozenset, exclude_namespaces: frozenset,
             rules: list, delete_notification: int, dry_run: bool):

    counter = Counter()

    for namespace in Namespace.objects(api):
        if matches_resource_filter(namespace, include_resources,
                                   exclude_resources, include_namespaces,
                                   exclude_namespaces):
            counter.update(
                handle_resource_on_ttl(namespace, rules, delete_notification,
                                       dry_run))
            counter.update(
                handle_resource_on_expiry(namespace, rules,
                                          delete_notification, dry_run))
        else:
            logger.debug(f'Skipping {namespace.kind} {namespace}')

    already_seen = set()

    filtered_resources = []

    resource_types = get_namespaced_resource_types(api)
    for _type in resource_types:
        if _type.endpoint not in exclude_resources:
            try:
                for resource in _type.objects(api, namespace=pykube.all):
                    # objects might be available via multiple API versions (e.g. deployments appear as extensions/v1beta1 and apps/v1)
                    # => process them only once
                    object_id = (resource.kind, resource.namespace,
                                 resource.name)
                    if object_id in already_seen:
                        continue
                    already_seen.add(object_id)
                    if matches_resource_filter(resource, include_resources,
                                               exclude_resources,
                                               include_namespaces,
                                               exclude_namespaces):
                        filtered_resources.append(resource)
                    else:
                        logger.debug(
                            f'Skipping {resource.kind} {resource.namespace}/{resource.name}'
                        )
            except Exception as e:
                logger.error(f'Could not list {_type.kind} objects: {e}')

    for resource in filtered_resources:
        counter.update(
            handle_resource_on_ttl(resource, rules, delete_notification,
                                   dry_run))
        counter.update(
            handle_resource_on_expiry(resource, rules, delete_notification,
                                      dry_run))
    stats = ', '.join([f'{k}={v}' for k, v in counter.items()])
    logger.info(f'Clean up run completed: {stats}')
    return counter
Example #4
0
def test_create_org(cluster):
    test_id = "t1"
    org = create_org_object(api=cluster.api,
                            organization_name=test_id + ORG_NAME)
    org.create()
    time.sleep(TIMEOUT)

    namespace = Namespace.objects(cluster.api).get(
        name=organization_namespacename_generator(organization_name=test_id +
                                                  ORG_NAME))
    assert namespace.labels["k8spin.cloud/type"] == "organization"
    assert namespace.labels["k8spin.cloud/org"] == test_id + ORG_NAME
Example #5
0
def test_create_spaces(cluster):
    test_id = "t3"
    org = create_org_object(api=cluster.api,
                            organization_name=test_id + ORG_NAME)
    org.create()
    time.sleep(TIMEOUT)

    tenant = create_tenant_object(api=cluster.api,
                                  organization_name=test_id + ORG_NAME,
                                  tenant_name=test_id + TENANT_NAME)
    tenant.create()
    time.sleep(TIMEOUT)

    space = create_space_object(api=cluster.api,
                                organization_name=test_id + ORG_NAME,
                                tenant_name=test_id + TENANT_NAME,
                                space_name=test_id + SPACE_NAME)
    space.create()
    time.sleep(TIMEOUT)

    namespace_name = space_namespacename_generator(
        organization_name=test_id + ORG_NAME,
        tenant_name=test_id + TENANT_NAME,
        space_name=test_id + SPACE_NAME)
    namespace = Namespace.objects(cluster.api).get(name=namespace_name)

    assert namespace.labels["k8spin.cloud/type"] == "space"
    assert namespace.labels["k8spin.cloud/org"] == test_id + ORG_NAME
    assert namespace.labels["k8spin.cloud/tenant"] == test_id + TENANT_NAME
    assert namespace.labels["k8spin.cloud/space"] == test_id + SPACE_NAME
    assert namespace.labels["k8spin.cloud/name"] == namespace_name

    resourceQuotas = pykube.ResourceQuota.objects(
        cluster.api,
        namespace.name).filter(selector={"k8spin.cloud/type": "quotas"})
    assert len(resourceQuotas) == 1

    limitRanges = pykube.LimitRange.objects(
        cluster.api,
        namespace.name).filter(selector={"k8spin.cloud/type": "defaults"})
    assert len(limitRanges) == 1
Example #6
0
def query_cluster(cluster, executor, system_namespaces,
                  additional_cost_per_cluster, no_ingress_status, node_label):
    logger.info(f"Querying cluster {cluster.id} ({cluster.api_server_url})..")
    pods = {}
    nodes = {}
    namespaces = {}

    for namespace in Namespace.objects(cluster.client):
        email = namespace.annotations.get('email')
        namespaces[namespace.name] = {
            "status": namespace.obj['status']['phase'],
            "email": email,
        }

    cluster_capacity = collections.defaultdict(float)
    cluster_allocatable = collections.defaultdict(float)
    cluster_requests = collections.defaultdict(float)
    user_requests = collections.defaultdict(float)
    node_count = collections.defaultdict(int)
    cluster_cost = additional_cost_per_cluster

    for _node in Node.objects(cluster.client):
        node = _node.obj
        nodes[_node.name] = node
        node["capacity"] = {}
        node["allocatable"] = {}
        node["requests"] = new_resources()
        node["usage"] = new_resources()
        for k, v in node["status"].get("capacity", {}).items():
            parsed = parse_resource(v)
            node["capacity"][k] = parsed
            cluster_capacity[k] += parsed
        for k, v in node["status"].get("allocatable", {}).items():
            parsed = parse_resource(v)
            node["allocatable"][k] = parsed
            cluster_allocatable[k] += parsed
        role = _node.labels.get(NODE_LABEL_ROLE) or "worker"
        node_count[role] += 1
        region = _node.labels.get(NODE_LABEL_REGION, "unknown")
        instance_type = _node.labels.get(NODE_LABEL_INSTANCE_TYPE, "unknown")
        is_spot = _node.labels.get(NODE_LABEL_SPOT) == "true"
        node["spot"] = is_spot
        node["kubelet_version"] = (node["status"].get("nodeInfo", {}).get(
            "kubeletVersion", ""))
        node["role"] = role
        node["instance_type"] = instance_type
        node["cost"] = pricing.get_node_cost(region, instance_type, is_spot)
        cluster_cost += node["cost"]

    get_node_usage(cluster, nodes)

    cluster_usage = collections.defaultdict(float)
    for node in nodes.values():
        for k, v in node['usage'].items():
            cluster_usage[k] += v

    cost_per_cpu = cluster_cost / cluster_allocatable["cpu"]
    cost_per_memory = cluster_cost / cluster_allocatable["memory"]

    for pod in Pod.objects(cluster.client, namespace=pykube.all):
        if pod.obj["status"].get("phase") != "Running":
            # ignore unschedulable/completed pods
            continue
        application = get_application_from_labels(pod.labels)
        component = get_component_from_labels(pod.labels)
        requests = collections.defaultdict(float)
        ns = pod.namespace
        container_images = []
        for container in pod.obj["spec"]["containers"]:
            # note that the "image" field is optional according to Kubernetes docs
            image = container.get("image")
            if image:
                container_images.append(image)
            for k, v in container["resources"].get("requests", {}).items():
                pv = parse_resource(v)
                requests[k] += pv
                cluster_requests[k] += pv
                if ns not in system_namespaces:
                    user_requests[k] += pv
        if "nodeName" in pod.obj["spec"] and pod.obj["spec"][
                "nodeName"] in nodes:
            for k in ("cpu", "memory"):
                nodes[pod.obj["spec"]
                      ["nodeName"]]["requests"][k] += requests.get(k, 0)
        cost = max(requests["cpu"] * cost_per_cpu,
                   requests["memory"] * cost_per_memory)
        pods[(ns, pod.name)] = {
            "requests": requests,
            "application": application,
            "component": component,
            "container_images": container_images,
            "cost": cost,
            "usage": new_resources(),
        }

    hourly_cost = cluster_cost / HOURS_PER_MONTH

    cluster_summary = {
        "cluster":
        cluster,
        "nodes":
        nodes,
        "pods":
        pods,
        "namespaces":
        namespaces,
        "user_pods":
        len([p for ns, p in pods if ns not in system_namespaces]),
        "master_nodes":
        node_count["master"],
        "worker_nodes":
        node_count[node_label],
        "kubelet_versions":
        set([
            n["kubelet_version"] for n in nodes.values()
            if n["role"] == node_label
        ]),
        "worker_instance_types":
        set([
            n["instance_type"] for n in nodes.values()
            if n["role"] == node_label
        ]),
        "worker_instance_is_spot":
        any([n["spot"] for n in nodes.values() if n["role"] == node_label]),
        "capacity":
        cluster_capacity,
        "allocatable":
        cluster_allocatable,
        "requests":
        cluster_requests,
        "user_requests":
        user_requests,
        "usage":
        cluster_usage,
        "cost":
        cluster_cost,
        "cost_per_user_request_hour": {
            "cpu":
            0.5 * hourly_cost /
            max(user_requests["cpu"], MIN_CPU_USER_REQUESTS),
            "memory":
            0.5 * hourly_cost /
            max(user_requests["memory"] / ONE_GIBI, MIN_MEMORY_USER_REQUESTS),
        },
        "ingresses": [],
    }

    get_pod_usage(cluster, pods)

    cluster_slack_cost = 0
    for pod in pods.values():
        usage_cost = max(
            pod["usage"]["cpu"] * cost_per_cpu,
            pod["usage"]["memory"] * cost_per_memory,
        )
        pod["slack_cost"] = pod["cost"] - usage_cost
        cluster_slack_cost += pod["slack_cost"]

    cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost)

    with FuturesSession(max_workers=10, session=session) as futures_session:
        futures_by_host = {}  # hostname -> future
        futures = collections.defaultdict(list)  # future -> [ingress]

        for _ingress in Ingress.objects(cluster.client, namespace=pykube.all):
            application = get_application_from_labels(_ingress.labels)
            for rule in _ingress.obj["spec"].get("rules", []):
                host = rule.get('host', '')
                if not application:
                    # find the application by getting labels from pods
                    backend_application = find_backend_application(
                        cluster.client, _ingress, rule)
                else:
                    backend_application = None
                ingress = [
                    _ingress.namespace, _ingress.name, application
                    or backend_application, host, 0
                ]
                if host and not no_ingress_status:
                    try:
                        future = futures_by_host[host]
                    except KeyError:
                        future = futures_session.get(f"https://{host}/",
                                                     timeout=5)
                        futures_by_host[host] = future
                    futures[future].append(ingress)
                cluster_summary["ingresses"].append(ingress)

        if not no_ingress_status:
            logger.info(
                f'Waiting for ingress status for {cluster.id} ({cluster.api_server_url})..'
            )
            for future in concurrent.futures.as_completed(futures):
                ingresses = futures[future]
                try:
                    response = future.result()
                    status = response.status_code
                except:
                    status = 999
                for ingress in ingresses:
                    ingress[4] = status

    return cluster_summary
Example #7
0
def query_cluster(
    cluster,
    executor,
    system_namespaces,
    additional_cost_per_cluster,
    alpha_ema,
    prev_cluster_summaries,
    no_ingress_status,
    node_labels,
):
    logger.info(f"Querying cluster {cluster.id} ({cluster.api_server_url})..")
    pods = {}
    nodes = {}
    namespaces = {}

    for namespace in Namespace.objects(cluster.client):
        email = namespace.annotations.get("email")
        namespaces[namespace.name] = {
            "status": namespace.obj["status"]["phase"],
            "email": email,
        }

    cluster_capacity = collections.defaultdict(float)
    cluster_allocatable = collections.defaultdict(float)
    cluster_requests = collections.defaultdict(float)
    user_requests = collections.defaultdict(float)
    cluster_cost = additional_cost_per_cluster

    for _node in Node.objects(cluster.client):
        node = map_node(_node)
        nodes[_node.name] = node

        for k, v in node["capacity"].items():
            cluster_capacity[k] += v
        for k, v in node["allocatable"].items():
            cluster_allocatable[k] += v
        cluster_cost += node["cost"]

    metrics.get_node_usage(cluster, nodes,
                           prev_cluster_summaries.get("nodes", {}), alpha_ema)

    cluster_usage = collections.defaultdict(float)
    for node in nodes.values():
        for k, v in node["usage"].items():
            cluster_usage[k] += v

    try:
        vpas_by_namespace_label = get_vpas_by_match_labels(cluster.client)
    except Exception as e:
        logger.warning(f"Failed to query VPAs in cluster {cluster.id}: {e}")
        vpas_by_namespace_label = collections.defaultdict(list)

    cost_per_cpu = cluster_cost / cluster_allocatable["cpu"]
    cost_per_memory = cluster_cost / cluster_allocatable["memory"]

    for pod in Pod.objects(cluster.client, namespace=pykube.all):
        # ignore unschedulable/completed pods
        if not pod_active(pod):
            continue
        pod_ = map_pod(pod, cost_per_cpu, cost_per_memory)
        for k, v in pod_["requests"].items():
            cluster_requests[k] += v
            if pod.namespace not in system_namespaces:
                user_requests[k] += v
        node_name = pod.obj["spec"].get("nodeName")
        if node_name and node_name in nodes:
            for k in ("cpu", "memory"):
                nodes[node_name]["requests"][k] += pod_["requests"].get(k, 0)
        found_vpa = False
        for k, v in pod.labels.items():
            vpas = vpas_by_namespace_label[(pod.namespace, k, v)]
            for vpa in vpas:
                if vpa.matches_pod(pod):
                    recommendation = new_resources()
                    container_names = set()
                    for container in pod.obj["spec"]["containers"]:
                        container_names.add(container["name"])
                    for container in vpa.container_recommendations:
                        # VPA might contain recommendations for containers which are no longer there!
                        if container["containerName"] in container_names:
                            for k in ("cpu", "memory"):
                                recommendation[k] += parse_resource(
                                    container["target"][k])
                    pod_["recommendation"] = recommendation
                    found_vpa = True
                    break
            if found_vpa:
                break
        pods[(pod.namespace, pod.name)] = pod_

    hourly_cost = cluster_cost / HOURS_PER_MONTH

    cluster_summary = {
        "cluster":
        cluster,
        "nodes":
        nodes,
        "pods":
        pods,
        "namespaces":
        namespaces,
        "user_pods":
        len([p for ns, p in pods if ns not in system_namespaces]),
        "master_nodes":
        len([n for n in nodes.values() if n["role"] == "master"]),
        "worker_nodes":
        len([n for n in nodes.values() if n["role"] in node_labels]),
        "kubelet_versions":
        set([
            n["kubelet_version"] for n in nodes.values()
            if n["role"] in node_labels
        ]),
        "worker_instance_types":
        set([
            n["instance_type"] for n in nodes.values()
            if n["role"] in node_labels
        ]),
        "worker_instance_is_spot":
        any([n["spot"] for n in nodes.values() if n["role"] in node_labels]),
        "capacity":
        cluster_capacity,
        "allocatable":
        cluster_allocatable,
        "requests":
        cluster_requests,
        "user_requests":
        user_requests,
        "usage":
        cluster_usage,
        "cost":
        cluster_cost,
        "cost_per_user_request_hour": {
            "cpu":
            0.5 * hourly_cost /
            max(user_requests["cpu"], MIN_CPU_USER_REQUESTS),
            "memory":
            0.5 * hourly_cost /
            max(user_requests["memory"] / ONE_GIBI, MIN_MEMORY_USER_REQUESTS),
        },
        "ingresses": [],
    }

    metrics.get_pod_usage(cluster, pods,
                          prev_cluster_summaries.get("pods", {}), alpha_ema)

    cluster_slack_cost = 0
    for pod in pods.values():
        usage_cost = max(
            pod["usage"]["cpu"] * cost_per_cpu,
            pod["usage"]["memory"] * cost_per_memory,
        )
        pod["slack_cost"] = pod["cost"] - usage_cost
        cluster_slack_cost += pod["slack_cost"]

    cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost)

    with FuturesSession(max_workers=10, session=session) as futures_session:
        futures_by_host = {}  # hostname -> future
        futures = collections.defaultdict(list)  # future -> [ingress]

        for _ingress in Ingress.objects(cluster.client, namespace=pykube.all):
            application = get_application_from_labels(_ingress.labels)
            for rule in _ingress.obj["spec"].get("rules", []):
                host = rule.get("host", "")
                if not application:
                    # find the application by getting labels from pods
                    backend_application = find_backend_application(
                        cluster.client, _ingress, rule)
                else:
                    backend_application = None
                ingress = [
                    _ingress.namespace,
                    _ingress.name,
                    application or backend_application,
                    host,
                    0,
                ]
                if host and not no_ingress_status:
                    try:
                        future = futures_by_host[host]
                    except KeyError:
                        future = futures_session.get(f"https://{host}/",
                                                     timeout=5)
                        futures_by_host[host] = future
                    futures[future].append(ingress)
                cluster_summary["ingresses"].append(ingress)

        if not no_ingress_status:
            logger.info(
                f"Waiting for ingress status for {cluster.id} ({cluster.api_server_url}).."
            )
            for future in concurrent.futures.as_completed(futures):
                ingresses = futures[future]
                try:
                    response = future.result()
                    status = response.status_code
                except Exception:
                    status = 999
                for ingress in ingresses:
                    ingress[4] = status

    return cluster_summary
Example #8
0
def clean_up(
    api,
    include_resources: frozenset,
    exclude_resources: frozenset,
    include_namespaces: frozenset,
    exclude_namespaces: frozenset,
    rules: list,
    delete_notification: int,
    deployment_time_annotation: Optional[str] = None,
    resource_context_hook: Optional[Callable[[APIObject, dict], Dict[str, Any]]] = None,
    dry_run: bool = False,
):

    counter: Counter = Counter()
    cache: Dict[str, Any] = {}

    for namespace in Namespace.objects(api):
        if matches_resource_filter(
            namespace,
            include_resources,
            exclude_resources,
            include_namespaces,
            exclude_namespaces,
        ):
            counter.update(
                handle_resource_on_ttl(
                    namespace,
                    rules,
                    delete_notification,
                    deployment_time_annotation,
                    resource_context_hook,
                    cache,
                    dry_run,
                )
            )
            counter.update(
                handle_resource_on_expiry(
                    namespace, rules, delete_notification, dry_run
                )
            )
        else:
            logger.debug(f"Skipping {namespace.kind} {namespace}")

    already_seen: set = set()

    filtered_resources = []

    resource_types = get_namespaced_resource_types(api)
    for _type in resource_types:
        if _type.endpoint not in exclude_resources:
            try:
                for resource in _type.objects(api, namespace=pykube.all):
                    # objects might be available via multiple API versions (e.g. deployments appear as extensions/v1beta1 and apps/v1)
                    # => process them only once
                    object_id = (resource.kind, resource.namespace, resource.name)
                    if object_id in already_seen:
                        continue
                    already_seen.add(object_id)
                    if matches_resource_filter(
                        resource,
                        include_resources,
                        exclude_resources,
                        include_namespaces,
                        exclude_namespaces,
                    ):
                        filtered_resources.append(resource)
                    else:
                        logger.debug(
                            f"Skipping {resource.kind} {resource.namespace}/{resource.name}"
                        )
            except Exception as e:
                logger.error(f"Could not list {_type.kind} objects: {e}")

    for resource in filtered_resources:
        counter.update(
            handle_resource_on_ttl(
                resource,
                rules,
                delete_notification,
                deployment_time_annotation,
                resource_context_hook,
                cache,
                dry_run,
            )
        )
        counter.update(
            handle_resource_on_expiry(resource, rules, delete_notification, dry_run)
        )
    stats = ", ".join([f"{k}={v}" for k, v in counter.items()])
    logger.info(f"Clean up run completed: {stats}")
    return counter
Example #9
0
def autoscale_resources(
    api,
    kind,
    namespace: str,
    exclude_namespaces: FrozenSet[str],
    exclude_names: FrozenSet[str],
    upscale_period: str,
    downscale_period: str,
    default_uptime: str,
    default_downtime: str,
    forced_uptime: bool,
    dry_run: bool,
    now: datetime.datetime,
    grace_period: int,
    downtime_replicas: int,
    deployment_time_annotation: Optional[str] = None,
):
    resources_by_namespace = collections.defaultdict(list)
    for resource in kind.objects(api, namespace=(namespace or pykube.all)):
        if resource.name in exclude_names:
            logger.debug(
                f"{resource.kind} {resource.namespace}/{resource.name} was excluded (name matches exclusion list)"
            )
            continue
        resources_by_namespace[resource.namespace].append(resource)

    for current_namespace, resources in sorted(resources_by_namespace.items()):

        if current_namespace in exclude_namespaces:
            logger.debug(
                f"Namespace {current_namespace} was excluded (exclusion list matches)"
            )
            continue

        logger.debug(
            f"Processing {len(resources)} {kind.endpoint} in namespace {current_namespace}.."
        )

        # Override defaults with (optional) annotations from Namespace
        namespace_obj = Namespace.objects(api).get_by_name(current_namespace)

        excluded = ignore_resource(namespace_obj, now)

        default_uptime_for_namespace = namespace_obj.annotations.get(
            UPTIME_ANNOTATION, default_uptime)
        default_downtime_for_namespace = namespace_obj.annotations.get(
            DOWNTIME_ANNOTATION, default_downtime)
        default_downtime_replicas_for_namespace = get_annotation_value_as_int(
            namespace_obj, DOWNTIME_REPLICAS_ANNOTATION)
        if default_downtime_replicas_for_namespace is None:
            default_downtime_replicas_for_namespace = downtime_replicas

        upscale_period_for_namespace = namespace_obj.annotations.get(
            UPSCALE_PERIOD_ANNOTATION, upscale_period)
        downscale_period_for_namespace = namespace_obj.annotations.get(
            DOWNSCALE_PERIOD_ANNOTATION, downscale_period)
        forced_uptime_for_namespace = namespace_obj.annotations.get(
            FORCE_UPTIME_ANNOTATION, forced_uptime)
        for resource in resources:
            autoscale_resource(
                resource,
                upscale_period_for_namespace,
                downscale_period_for_namespace,
                default_uptime_for_namespace,
                default_downtime_for_namespace,
                forced_uptime_for_namespace,
                dry_run,
                now,
                grace_period,
                default_downtime_replicas_for_namespace,
                namespace_excluded=excluded,
                deployment_time_annotation=deployment_time_annotation,
            )
Example #10
0
def autoscale_resources(
    api,
    kind,
    namespace: str,
    exclude_namespaces: FrozenSet[Pattern],
    exclude_names: FrozenSet[str],
    upscale_period: str,
    downscale_period: str,
    default_uptime: str,
    default_downtime: str,
    forced_uptime: bool,
    dry_run: bool,
    now: datetime.datetime,
    grace_period: int,
    downtime_replicas: int,
    deployment_time_annotation: Optional[str] = None,
    enable_events: bool = False,
    upscale_step_size: int = 0,
):
    resources_by_namespace = collections.defaultdict(list)
    for resource in kind.objects(api, namespace=(namespace or pykube.all)):
        if resource.name in exclude_names:
            logger.debug(
                f"{resource.kind} {resource.namespace}/{resource.name} was excluded (name matches exclusion list)"
            )
            continue
        resources_by_namespace[resource.namespace].append(resource)

    for current_namespace, resources in sorted(resources_by_namespace.items()):

        if any([
                pattern.fullmatch(current_namespace)
                for pattern in exclude_namespaces
        ]):
            logger.debug(
                f"Namespace {current_namespace} was excluded (exclusion list regex matches)"
            )
            continue

        logger.debug(
            f"Processing {len(resources)} {kind.endpoint} in namespace {current_namespace}.."
        )

        # Override defaults with (optional) annotations from Namespace
        namespace_obj = Namespace.objects(api).get_by_name(current_namespace)

        excluded = ignore_resource(namespace_obj, now)

        default_uptime_for_namespace = namespace_obj.annotations.get(
            UPTIME_ANNOTATION, default_uptime)
        default_downtime_for_namespace = namespace_obj.annotations.get(
            DOWNTIME_ANNOTATION, default_downtime)
        default_downtime_replicas_for_namespace = get_annotation_value_as_int(
            namespace_obj, DOWNTIME_REPLICAS_ANNOTATION)
        if default_downtime_replicas_for_namespace is None:
            default_downtime_replicas_for_namespace = downtime_replicas

        upscale_period_for_namespace = namespace_obj.annotations.get(
            UPSCALE_PERIOD_ANNOTATION, upscale_period)
        downscale_period_for_namespace = namespace_obj.annotations.get(
            DOWNSCALE_PERIOD_ANNOTATION, downscale_period)
        forced_uptime_value_for_namespace = str(
            namespace_obj.annotations.get(FORCE_UPTIME_ANNOTATION,
                                          forced_uptime))
        if forced_uptime_value_for_namespace.lower() == "true":
            forced_uptime_for_namespace = True
        elif forced_uptime_value_for_namespace.lower() == "false":
            forced_uptime_for_namespace = False
        elif forced_uptime_value_for_namespace:
            forced_uptime_for_namespace = matches_time_spec(
                now, forced_uptime_value_for_namespace)
        else:
            forced_uptime_for_namespace = False

        upscale_step_size_for_namespace = get_annotation_value_as_int(
            namespace_obj, UPSTCALE_STEP_SIZE_ANNOTATION)
        if upscale_step_size_for_namespace is None:
            upscale_step_size_for_namespace = upscale_step_size

        for resource in resources:
            autoscale_resource(
                resource,
                upscale_period_for_namespace,
                downscale_period_for_namespace,
                default_uptime_for_namespace,
                default_downtime_for_namespace,
                forced_uptime_for_namespace,
                dry_run,
                now,
                grace_period,
                default_downtime_replicas_for_namespace,
                namespace_excluded=excluded,
                deployment_time_annotation=deployment_time_annotation,
                enable_events=enable_events,
                upscale_step_size=upscale_step_size_for_namespace,
            )