def test_create_np(cluster): test_id = "np1" org = create_org_object(api=cluster.api, organization_name=test_id + ORG_NAME) org.obj["spec"]["resources"]["cpu"] = "10" org.obj["spec"]["resources"]["memory"] = "10G" org.create() time.sleep(TIMEOUT) tenant = create_tenant_object(api=cluster.api, organization_name=test_id + ORG_NAME, tenant_name=test_id + TENANT_NAME) tenant.obj["spec"]["resources"]["cpu"] = "5" tenant.obj["spec"]["resources"]["memory"] = "5G" tenant.create() time.sleep(TIMEOUT) space = create_space_object(api=cluster.api, organization_name=test_id + ORG_NAME, tenant_name=test_id + TENANT_NAME, space_name=test_id + SPACE_NAME) space.obj["spec"]["resources"]["cpu"] = "1" space.obj["spec"]["resources"]["memory"] = "1G" space.create() time.sleep(TIMEOUT) namespace = Namespace.objects(cluster.api).get( name=space_namespacename_generator(organization_name=test_id + ORG_NAME, tenant_name=test_id + TENANT_NAME, space_name=test_id + SPACE_NAME)) networkPolicies = NetworkPolicy.objects(cluster.api, namespace.name).filter() assert len(networkPolicies) == 1
def test_create_space_np(cluster): test_id = "np2" org = create_org_object(api=cluster.api, organization_name=test_id + ORG_NAME) org.obj["spec"]["resources"]["cpu"] = "10" org.obj["spec"]["resources"]["memory"] = "10G" org.create() time.sleep(TIMEOUT) tenant = create_tenant_object(api=cluster.api, organization_name=test_id + ORG_NAME, tenant_name=test_id + TENANT_NAME) tenant.obj["spec"]["resources"]["cpu"] = "5" tenant.obj["spec"]["resources"]["memory"] = "5G" tenant.create() time.sleep(TIMEOUT) space = create_space_object(api=cluster.api, organization_name=test_id + ORG_NAME, tenant_name=test_id + TENANT_NAME, space_name=test_id + SPACE_NAME) space.obj["spec"]["resources"]["cpu"] = "1" space.obj["spec"]["resources"]["memory"] = "1G" space.obj["spec"]["allowIncomingNetwork"] = { "organizations": [{ "organization_name": "example" }], "tenants": [ { "organization_name": "example", "tenant_name": "crm" }, { "organization_name": "example", "tenant_name": "crm2" }, ], "spaces": [{ "organization_name": "example", "tenant_name": "crm", "space_name": "test" }] } space.create() time.sleep(TIMEOUT) namespace = Namespace.objects(cluster.api).get( name=space_namespacename_generator(organization_name=test_id + ORG_NAME, tenant_name=test_id + TENANT_NAME, space_name=test_id + SPACE_NAME)) networkPolicies = list( NetworkPolicy.objects(cluster.api, namespace.name).iterator()) assert len(networkPolicies) == 1 print(networkPolicies[0]) assert len(networkPolicies[0].ingress) == 5
def clean_up(api, include_resources: frozenset, exclude_resources: frozenset, include_namespaces: frozenset, exclude_namespaces: frozenset, rules: list, delete_notification: int, dry_run: bool): counter = Counter() for namespace in Namespace.objects(api): if matches_resource_filter(namespace, include_resources, exclude_resources, include_namespaces, exclude_namespaces): counter.update( handle_resource_on_ttl(namespace, rules, delete_notification, dry_run)) counter.update( handle_resource_on_expiry(namespace, rules, delete_notification, dry_run)) else: logger.debug(f'Skipping {namespace.kind} {namespace}') already_seen = set() filtered_resources = [] resource_types = get_namespaced_resource_types(api) for _type in resource_types: if _type.endpoint not in exclude_resources: try: for resource in _type.objects(api, namespace=pykube.all): # objects might be available via multiple API versions (e.g. deployments appear as extensions/v1beta1 and apps/v1) # => process them only once object_id = (resource.kind, resource.namespace, resource.name) if object_id in already_seen: continue already_seen.add(object_id) if matches_resource_filter(resource, include_resources, exclude_resources, include_namespaces, exclude_namespaces): filtered_resources.append(resource) else: logger.debug( f'Skipping {resource.kind} {resource.namespace}/{resource.name}' ) except Exception as e: logger.error(f'Could not list {_type.kind} objects: {e}') for resource in filtered_resources: counter.update( handle_resource_on_ttl(resource, rules, delete_notification, dry_run)) counter.update( handle_resource_on_expiry(resource, rules, delete_notification, dry_run)) stats = ', '.join([f'{k}={v}' for k, v in counter.items()]) logger.info(f'Clean up run completed: {stats}') return counter
def test_create_org(cluster): test_id = "t1" org = create_org_object(api=cluster.api, organization_name=test_id + ORG_NAME) org.create() time.sleep(TIMEOUT) namespace = Namespace.objects(cluster.api).get( name=organization_namespacename_generator(organization_name=test_id + ORG_NAME)) assert namespace.labels["k8spin.cloud/type"] == "organization" assert namespace.labels["k8spin.cloud/org"] == test_id + ORG_NAME
def test_create_spaces(cluster): test_id = "t3" org = create_org_object(api=cluster.api, organization_name=test_id + ORG_NAME) org.create() time.sleep(TIMEOUT) tenant = create_tenant_object(api=cluster.api, organization_name=test_id + ORG_NAME, tenant_name=test_id + TENANT_NAME) tenant.create() time.sleep(TIMEOUT) space = create_space_object(api=cluster.api, organization_name=test_id + ORG_NAME, tenant_name=test_id + TENANT_NAME, space_name=test_id + SPACE_NAME) space.create() time.sleep(TIMEOUT) namespace_name = space_namespacename_generator( organization_name=test_id + ORG_NAME, tenant_name=test_id + TENANT_NAME, space_name=test_id + SPACE_NAME) namespace = Namespace.objects(cluster.api).get(name=namespace_name) assert namespace.labels["k8spin.cloud/type"] == "space" assert namespace.labels["k8spin.cloud/org"] == test_id + ORG_NAME assert namespace.labels["k8spin.cloud/tenant"] == test_id + TENANT_NAME assert namespace.labels["k8spin.cloud/space"] == test_id + SPACE_NAME assert namespace.labels["k8spin.cloud/name"] == namespace_name resourceQuotas = pykube.ResourceQuota.objects( cluster.api, namespace.name).filter(selector={"k8spin.cloud/type": "quotas"}) assert len(resourceQuotas) == 1 limitRanges = pykube.LimitRange.objects( cluster.api, namespace.name).filter(selector={"k8spin.cloud/type": "defaults"}) assert len(limitRanges) == 1
def query_cluster(cluster, executor, system_namespaces, additional_cost_per_cluster, no_ingress_status, node_label): logger.info(f"Querying cluster {cluster.id} ({cluster.api_server_url})..") pods = {} nodes = {} namespaces = {} for namespace in Namespace.objects(cluster.client): email = namespace.annotations.get('email') namespaces[namespace.name] = { "status": namespace.obj['status']['phase'], "email": email, } cluster_capacity = collections.defaultdict(float) cluster_allocatable = collections.defaultdict(float) cluster_requests = collections.defaultdict(float) user_requests = collections.defaultdict(float) node_count = collections.defaultdict(int) cluster_cost = additional_cost_per_cluster for _node in Node.objects(cluster.client): node = _node.obj nodes[_node.name] = node node["capacity"] = {} node["allocatable"] = {} node["requests"] = new_resources() node["usage"] = new_resources() for k, v in node["status"].get("capacity", {}).items(): parsed = parse_resource(v) node["capacity"][k] = parsed cluster_capacity[k] += parsed for k, v in node["status"].get("allocatable", {}).items(): parsed = parse_resource(v) node["allocatable"][k] = parsed cluster_allocatable[k] += parsed role = _node.labels.get(NODE_LABEL_ROLE) or "worker" node_count[role] += 1 region = _node.labels.get(NODE_LABEL_REGION, "unknown") instance_type = _node.labels.get(NODE_LABEL_INSTANCE_TYPE, "unknown") is_spot = _node.labels.get(NODE_LABEL_SPOT) == "true" node["spot"] = is_spot node["kubelet_version"] = (node["status"].get("nodeInfo", {}).get( "kubeletVersion", "")) node["role"] = role node["instance_type"] = instance_type node["cost"] = pricing.get_node_cost(region, instance_type, is_spot) cluster_cost += node["cost"] get_node_usage(cluster, nodes) cluster_usage = collections.defaultdict(float) for node in nodes.values(): for k, v in node['usage'].items(): cluster_usage[k] += v cost_per_cpu = cluster_cost / cluster_allocatable["cpu"] cost_per_memory = cluster_cost / cluster_allocatable["memory"] for pod in Pod.objects(cluster.client, namespace=pykube.all): if pod.obj["status"].get("phase") != "Running": # ignore unschedulable/completed pods continue application = get_application_from_labels(pod.labels) component = get_component_from_labels(pod.labels) requests = collections.defaultdict(float) ns = pod.namespace container_images = [] for container in pod.obj["spec"]["containers"]: # note that the "image" field is optional according to Kubernetes docs image = container.get("image") if image: container_images.append(image) for k, v in container["resources"].get("requests", {}).items(): pv = parse_resource(v) requests[k] += pv cluster_requests[k] += pv if ns not in system_namespaces: user_requests[k] += pv if "nodeName" in pod.obj["spec"] and pod.obj["spec"][ "nodeName"] in nodes: for k in ("cpu", "memory"): nodes[pod.obj["spec"] ["nodeName"]]["requests"][k] += requests.get(k, 0) cost = max(requests["cpu"] * cost_per_cpu, requests["memory"] * cost_per_memory) pods[(ns, pod.name)] = { "requests": requests, "application": application, "component": component, "container_images": container_images, "cost": cost, "usage": new_resources(), } hourly_cost = cluster_cost / HOURS_PER_MONTH cluster_summary = { "cluster": cluster, "nodes": nodes, "pods": pods, "namespaces": namespaces, "user_pods": len([p for ns, p in pods if ns not in system_namespaces]), "master_nodes": node_count["master"], "worker_nodes": node_count[node_label], "kubelet_versions": set([ n["kubelet_version"] for n in nodes.values() if n["role"] == node_label ]), "worker_instance_types": set([ n["instance_type"] for n in nodes.values() if n["role"] == node_label ]), "worker_instance_is_spot": any([n["spot"] for n in nodes.values() if n["role"] == node_label]), "capacity": cluster_capacity, "allocatable": cluster_allocatable, "requests": cluster_requests, "user_requests": user_requests, "usage": cluster_usage, "cost": cluster_cost, "cost_per_user_request_hour": { "cpu": 0.5 * hourly_cost / max(user_requests["cpu"], MIN_CPU_USER_REQUESTS), "memory": 0.5 * hourly_cost / max(user_requests["memory"] / ONE_GIBI, MIN_MEMORY_USER_REQUESTS), }, "ingresses": [], } get_pod_usage(cluster, pods) cluster_slack_cost = 0 for pod in pods.values(): usage_cost = max( pod["usage"]["cpu"] * cost_per_cpu, pod["usage"]["memory"] * cost_per_memory, ) pod["slack_cost"] = pod["cost"] - usage_cost cluster_slack_cost += pod["slack_cost"] cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost) with FuturesSession(max_workers=10, session=session) as futures_session: futures_by_host = {} # hostname -> future futures = collections.defaultdict(list) # future -> [ingress] for _ingress in Ingress.objects(cluster.client, namespace=pykube.all): application = get_application_from_labels(_ingress.labels) for rule in _ingress.obj["spec"].get("rules", []): host = rule.get('host', '') if not application: # find the application by getting labels from pods backend_application = find_backend_application( cluster.client, _ingress, rule) else: backend_application = None ingress = [ _ingress.namespace, _ingress.name, application or backend_application, host, 0 ] if host and not no_ingress_status: try: future = futures_by_host[host] except KeyError: future = futures_session.get(f"https://{host}/", timeout=5) futures_by_host[host] = future futures[future].append(ingress) cluster_summary["ingresses"].append(ingress) if not no_ingress_status: logger.info( f'Waiting for ingress status for {cluster.id} ({cluster.api_server_url})..' ) for future in concurrent.futures.as_completed(futures): ingresses = futures[future] try: response = future.result() status = response.status_code except: status = 999 for ingress in ingresses: ingress[4] = status return cluster_summary
def query_cluster( cluster, executor, system_namespaces, additional_cost_per_cluster, alpha_ema, prev_cluster_summaries, no_ingress_status, node_labels, ): logger.info(f"Querying cluster {cluster.id} ({cluster.api_server_url})..") pods = {} nodes = {} namespaces = {} for namespace in Namespace.objects(cluster.client): email = namespace.annotations.get("email") namespaces[namespace.name] = { "status": namespace.obj["status"]["phase"], "email": email, } cluster_capacity = collections.defaultdict(float) cluster_allocatable = collections.defaultdict(float) cluster_requests = collections.defaultdict(float) user_requests = collections.defaultdict(float) cluster_cost = additional_cost_per_cluster for _node in Node.objects(cluster.client): node = map_node(_node) nodes[_node.name] = node for k, v in node["capacity"].items(): cluster_capacity[k] += v for k, v in node["allocatable"].items(): cluster_allocatable[k] += v cluster_cost += node["cost"] metrics.get_node_usage(cluster, nodes, prev_cluster_summaries.get("nodes", {}), alpha_ema) cluster_usage = collections.defaultdict(float) for node in nodes.values(): for k, v in node["usage"].items(): cluster_usage[k] += v try: vpas_by_namespace_label = get_vpas_by_match_labels(cluster.client) except Exception as e: logger.warning(f"Failed to query VPAs in cluster {cluster.id}: {e}") vpas_by_namespace_label = collections.defaultdict(list) cost_per_cpu = cluster_cost / cluster_allocatable["cpu"] cost_per_memory = cluster_cost / cluster_allocatable["memory"] for pod in Pod.objects(cluster.client, namespace=pykube.all): # ignore unschedulable/completed pods if not pod_active(pod): continue pod_ = map_pod(pod, cost_per_cpu, cost_per_memory) for k, v in pod_["requests"].items(): cluster_requests[k] += v if pod.namespace not in system_namespaces: user_requests[k] += v node_name = pod.obj["spec"].get("nodeName") if node_name and node_name in nodes: for k in ("cpu", "memory"): nodes[node_name]["requests"][k] += pod_["requests"].get(k, 0) found_vpa = False for k, v in pod.labels.items(): vpas = vpas_by_namespace_label[(pod.namespace, k, v)] for vpa in vpas: if vpa.matches_pod(pod): recommendation = new_resources() container_names = set() for container in pod.obj["spec"]["containers"]: container_names.add(container["name"]) for container in vpa.container_recommendations: # VPA might contain recommendations for containers which are no longer there! if container["containerName"] in container_names: for k in ("cpu", "memory"): recommendation[k] += parse_resource( container["target"][k]) pod_["recommendation"] = recommendation found_vpa = True break if found_vpa: break pods[(pod.namespace, pod.name)] = pod_ hourly_cost = cluster_cost / HOURS_PER_MONTH cluster_summary = { "cluster": cluster, "nodes": nodes, "pods": pods, "namespaces": namespaces, "user_pods": len([p for ns, p in pods if ns not in system_namespaces]), "master_nodes": len([n for n in nodes.values() if n["role"] == "master"]), "worker_nodes": len([n for n in nodes.values() if n["role"] in node_labels]), "kubelet_versions": set([ n["kubelet_version"] for n in nodes.values() if n["role"] in node_labels ]), "worker_instance_types": set([ n["instance_type"] for n in nodes.values() if n["role"] in node_labels ]), "worker_instance_is_spot": any([n["spot"] for n in nodes.values() if n["role"] in node_labels]), "capacity": cluster_capacity, "allocatable": cluster_allocatable, "requests": cluster_requests, "user_requests": user_requests, "usage": cluster_usage, "cost": cluster_cost, "cost_per_user_request_hour": { "cpu": 0.5 * hourly_cost / max(user_requests["cpu"], MIN_CPU_USER_REQUESTS), "memory": 0.5 * hourly_cost / max(user_requests["memory"] / ONE_GIBI, MIN_MEMORY_USER_REQUESTS), }, "ingresses": [], } metrics.get_pod_usage(cluster, pods, prev_cluster_summaries.get("pods", {}), alpha_ema) cluster_slack_cost = 0 for pod in pods.values(): usage_cost = max( pod["usage"]["cpu"] * cost_per_cpu, pod["usage"]["memory"] * cost_per_memory, ) pod["slack_cost"] = pod["cost"] - usage_cost cluster_slack_cost += pod["slack_cost"] cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost) with FuturesSession(max_workers=10, session=session) as futures_session: futures_by_host = {} # hostname -> future futures = collections.defaultdict(list) # future -> [ingress] for _ingress in Ingress.objects(cluster.client, namespace=pykube.all): application = get_application_from_labels(_ingress.labels) for rule in _ingress.obj["spec"].get("rules", []): host = rule.get("host", "") if not application: # find the application by getting labels from pods backend_application = find_backend_application( cluster.client, _ingress, rule) else: backend_application = None ingress = [ _ingress.namespace, _ingress.name, application or backend_application, host, 0, ] if host and not no_ingress_status: try: future = futures_by_host[host] except KeyError: future = futures_session.get(f"https://{host}/", timeout=5) futures_by_host[host] = future futures[future].append(ingress) cluster_summary["ingresses"].append(ingress) if not no_ingress_status: logger.info( f"Waiting for ingress status for {cluster.id} ({cluster.api_server_url}).." ) for future in concurrent.futures.as_completed(futures): ingresses = futures[future] try: response = future.result() status = response.status_code except Exception: status = 999 for ingress in ingresses: ingress[4] = status return cluster_summary
def clean_up( api, include_resources: frozenset, exclude_resources: frozenset, include_namespaces: frozenset, exclude_namespaces: frozenset, rules: list, delete_notification: int, deployment_time_annotation: Optional[str] = None, resource_context_hook: Optional[Callable[[APIObject, dict], Dict[str, Any]]] = None, dry_run: bool = False, ): counter: Counter = Counter() cache: Dict[str, Any] = {} for namespace in Namespace.objects(api): if matches_resource_filter( namespace, include_resources, exclude_resources, include_namespaces, exclude_namespaces, ): counter.update( handle_resource_on_ttl( namespace, rules, delete_notification, deployment_time_annotation, resource_context_hook, cache, dry_run, ) ) counter.update( handle_resource_on_expiry( namespace, rules, delete_notification, dry_run ) ) else: logger.debug(f"Skipping {namespace.kind} {namespace}") already_seen: set = set() filtered_resources = [] resource_types = get_namespaced_resource_types(api) for _type in resource_types: if _type.endpoint not in exclude_resources: try: for resource in _type.objects(api, namespace=pykube.all): # objects might be available via multiple API versions (e.g. deployments appear as extensions/v1beta1 and apps/v1) # => process them only once object_id = (resource.kind, resource.namespace, resource.name) if object_id in already_seen: continue already_seen.add(object_id) if matches_resource_filter( resource, include_resources, exclude_resources, include_namespaces, exclude_namespaces, ): filtered_resources.append(resource) else: logger.debug( f"Skipping {resource.kind} {resource.namespace}/{resource.name}" ) except Exception as e: logger.error(f"Could not list {_type.kind} objects: {e}") for resource in filtered_resources: counter.update( handle_resource_on_ttl( resource, rules, delete_notification, deployment_time_annotation, resource_context_hook, cache, dry_run, ) ) counter.update( handle_resource_on_expiry(resource, rules, delete_notification, dry_run) ) stats = ", ".join([f"{k}={v}" for k, v in counter.items()]) logger.info(f"Clean up run completed: {stats}") return counter
def autoscale_resources( api, kind, namespace: str, exclude_namespaces: FrozenSet[str], exclude_names: FrozenSet[str], upscale_period: str, downscale_period: str, default_uptime: str, default_downtime: str, forced_uptime: bool, dry_run: bool, now: datetime.datetime, grace_period: int, downtime_replicas: int, deployment_time_annotation: Optional[str] = None, ): resources_by_namespace = collections.defaultdict(list) for resource in kind.objects(api, namespace=(namespace or pykube.all)): if resource.name in exclude_names: logger.debug( f"{resource.kind} {resource.namespace}/{resource.name} was excluded (name matches exclusion list)" ) continue resources_by_namespace[resource.namespace].append(resource) for current_namespace, resources in sorted(resources_by_namespace.items()): if current_namespace in exclude_namespaces: logger.debug( f"Namespace {current_namespace} was excluded (exclusion list matches)" ) continue logger.debug( f"Processing {len(resources)} {kind.endpoint} in namespace {current_namespace}.." ) # Override defaults with (optional) annotations from Namespace namespace_obj = Namespace.objects(api).get_by_name(current_namespace) excluded = ignore_resource(namespace_obj, now) default_uptime_for_namespace = namespace_obj.annotations.get( UPTIME_ANNOTATION, default_uptime) default_downtime_for_namespace = namespace_obj.annotations.get( DOWNTIME_ANNOTATION, default_downtime) default_downtime_replicas_for_namespace = get_annotation_value_as_int( namespace_obj, DOWNTIME_REPLICAS_ANNOTATION) if default_downtime_replicas_for_namespace is None: default_downtime_replicas_for_namespace = downtime_replicas upscale_period_for_namespace = namespace_obj.annotations.get( UPSCALE_PERIOD_ANNOTATION, upscale_period) downscale_period_for_namespace = namespace_obj.annotations.get( DOWNSCALE_PERIOD_ANNOTATION, downscale_period) forced_uptime_for_namespace = namespace_obj.annotations.get( FORCE_UPTIME_ANNOTATION, forced_uptime) for resource in resources: autoscale_resource( resource, upscale_period_for_namespace, downscale_period_for_namespace, default_uptime_for_namespace, default_downtime_for_namespace, forced_uptime_for_namespace, dry_run, now, grace_period, default_downtime_replicas_for_namespace, namespace_excluded=excluded, deployment_time_annotation=deployment_time_annotation, )
def autoscale_resources( api, kind, namespace: str, exclude_namespaces: FrozenSet[Pattern], exclude_names: FrozenSet[str], upscale_period: str, downscale_period: str, default_uptime: str, default_downtime: str, forced_uptime: bool, dry_run: bool, now: datetime.datetime, grace_period: int, downtime_replicas: int, deployment_time_annotation: Optional[str] = None, enable_events: bool = False, upscale_step_size: int = 0, ): resources_by_namespace = collections.defaultdict(list) for resource in kind.objects(api, namespace=(namespace or pykube.all)): if resource.name in exclude_names: logger.debug( f"{resource.kind} {resource.namespace}/{resource.name} was excluded (name matches exclusion list)" ) continue resources_by_namespace[resource.namespace].append(resource) for current_namespace, resources in sorted(resources_by_namespace.items()): if any([ pattern.fullmatch(current_namespace) for pattern in exclude_namespaces ]): logger.debug( f"Namespace {current_namespace} was excluded (exclusion list regex matches)" ) continue logger.debug( f"Processing {len(resources)} {kind.endpoint} in namespace {current_namespace}.." ) # Override defaults with (optional) annotations from Namespace namespace_obj = Namespace.objects(api).get_by_name(current_namespace) excluded = ignore_resource(namespace_obj, now) default_uptime_for_namespace = namespace_obj.annotations.get( UPTIME_ANNOTATION, default_uptime) default_downtime_for_namespace = namespace_obj.annotations.get( DOWNTIME_ANNOTATION, default_downtime) default_downtime_replicas_for_namespace = get_annotation_value_as_int( namespace_obj, DOWNTIME_REPLICAS_ANNOTATION) if default_downtime_replicas_for_namespace is None: default_downtime_replicas_for_namespace = downtime_replicas upscale_period_for_namespace = namespace_obj.annotations.get( UPSCALE_PERIOD_ANNOTATION, upscale_period) downscale_period_for_namespace = namespace_obj.annotations.get( DOWNSCALE_PERIOD_ANNOTATION, downscale_period) forced_uptime_value_for_namespace = str( namespace_obj.annotations.get(FORCE_UPTIME_ANNOTATION, forced_uptime)) if forced_uptime_value_for_namespace.lower() == "true": forced_uptime_for_namespace = True elif forced_uptime_value_for_namespace.lower() == "false": forced_uptime_for_namespace = False elif forced_uptime_value_for_namespace: forced_uptime_for_namespace = matches_time_spec( now, forced_uptime_value_for_namespace) else: forced_uptime_for_namespace = False upscale_step_size_for_namespace = get_annotation_value_as_int( namespace_obj, UPSTCALE_STEP_SIZE_ANNOTATION) if upscale_step_size_for_namespace is None: upscale_step_size_for_namespace = upscale_step_size for resource in resources: autoscale_resource( resource, upscale_period_for_namespace, downscale_period_for_namespace, default_uptime_for_namespace, default_downtime_for_namespace, forced_uptime_for_namespace, dry_run, now, grace_period, default_downtime_replicas_for_namespace, namespace_excluded=excluded, deployment_time_annotation=deployment_time_annotation, enable_events=enable_events, upscale_step_size=upscale_step_size_for_namespace, )