def autoscale_resource(resource: pykube.objects.NamespacedAPIObject, default_uptime: str, default_downtime: str, forced_uptime: bool, dry_run: bool, now: datetime.datetime, grace_period: int): try: # any value different from "false" will ignore the resource (to be on the safe side) exclude = resource.annotations.get(EXCLUDE_ANNOTATION, 'false').lower() != 'false' if exclude: logger.debug('%s %s/%s was excluded', resource.kind, resource.namespace, resource.name) else: replicas = resource.get_replicas() if forced_uptime: uptime = "forced" downtime = "ignored" is_uptime = True else: uptime = resource.annotations.get('downscaler/uptime', default_uptime) downtime = resource.annotations.get('downscaler/downtime', default_downtime) is_uptime = helper.matches_time_spec(now, uptime) and not helper.matches_time_spec(now, downtime) original_replicas = resource.annotations.get(ORIGINAL_REPLICAS_ANNOTATION) logger.debug('%s %s/%s has %s replicas (original: %s, uptime: %s)', resource.kind, resource.namespace, resource.name, replicas, original_replicas, uptime) update_needed = False if is_uptime and replicas == 0 and original_replicas and int(original_replicas) > 0: logger.info('Scaling up %s %s/%s from %s to %s replicas (uptime: %s, downtime: %s)', resource.kind, resource.namespace, resource.name, replicas, original_replicas, uptime, downtime) resource.set_replicas(int(original_replicas)) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = None update_needed = True elif not is_uptime and replicas > 0: if within_grace_period(resource, grace_period): logger.info('%s %s/%s within grace period (%ds), not scaling down (yet)', resource.kind, resource.namespace, resource.name, grace_period) else: target_replicas = 0 logger.info('Scaling down %s %s/%s from %s to %s replicas (uptime: %s, downtime: %s)', resource.kind, resource.namespace, resource.name, replicas, target_replicas, uptime, downtime) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = str(replicas) resource.set_replicas(target_replicas) update_needed = True if update_needed: if dry_run: logger.info('**DRY-RUN**: would update %s %s/%s', resource.kind, resource.namespace, resource.name) else: resource.update() except Exception as e: logger.exception('Failed to process %s %s/%s : %s', resource.kind, resource.namespace, resource.name, str(e))
def autoscale_resource(resource: pykube.objects.NamespacedAPIObject, upscale_period: str, downscale_period: str, default_uptime: str, default_downtime: str, forced_uptime: bool, dry_run: bool, now: datetime.datetime, grace_period: int, downtime_replicas: int, namespace_excluded=False): try: # any value different from "false" will ignore the resource (to be on the safe side) exclude = namespace_excluded or (resource.annotations.get( EXCLUDE_ANNOTATION, 'false').lower() != 'false') original_replicas = resource.annotations.get( ORIGINAL_REPLICAS_ANNOTATION) downtime_replicas = resource.annotations.get( DOWNTIME_REPLICAS_ANNOTATION, downtime_replicas) if exclude and not original_replicas: logger.debug('%s %s/%s was excluded', resource.kind, resource.namespace, resource.name) else: replicas = resource.replicas ignore = False if forced_uptime or (exclude and original_replicas): uptime = "forced" downtime = "ignored" is_uptime = True elif upscale_period != 'never' or downscale_period != 'never': uptime = resource.annotations.get(UPSCALE_PERIOD_ANNOTATION, upscale_period) downtime = resource.annotations.get( DOWNSCALE_PERIOD_ANNOTATION, downscale_period) if helper.matches_time_spec( now, uptime) and helper.matches_time_spec( now, downtime): logger.debug( 'Upscale and downscale periods overlap, do nothing') ignore = True elif helper.matches_time_spec(now, uptime): is_uptime = True elif helper.matches_time_spec(now, downtime): is_uptime = False else: ignore = True else: uptime = resource.annotations.get(UPTIME_ANNOTATION, default_uptime) downtime = resource.annotations.get(DOWNTIME_ANNOTATION, default_downtime) is_uptime = helper.matches_time_spec( now, uptime) and not helper.matches_time_spec(now, downtime) logger.debug('%s %s/%s has %s replicas (original: %s, uptime: %s)', resource.kind, resource.namespace, resource.name, replicas, original_replicas, uptime) update_needed = False if not ignore and is_uptime and replicas == downtime_replicas and original_replicas and int( original_replicas) > 0: logger.info( 'Scaling up %s %s/%s from %s to %s replicas (uptime: %s, downtime: %s)', resource.kind, resource.namespace, resource.name, replicas, original_replicas, uptime, downtime) resource.replicas = int(original_replicas) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = None update_needed = True elif not ignore and not is_uptime and replicas > 0: target_replicas = int( resource.annotations.get(DOWNTIME_REPLICAS_ANNOTATION, downtime_replicas)) if within_grace_period(resource, grace_period, now): logger.info( '%s %s/%s within grace period (%ds), not scaling down (yet)', resource.kind, resource.namespace, resource.name, grace_period) else: logger.info( 'Scaling down %s %s/%s from %s to %s replicas (uptime: %s, downtime: %s)', resource.kind, resource.namespace, resource.name, replicas, target_replicas, uptime, downtime) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = str( replicas) resource.replicas = target_replicas update_needed = True if update_needed: if dry_run: logger.info('**DRY-RUN**: would update %s %s/%s', resource.kind, resource.namespace, resource.name) else: resource.update() except Exception as e: logger.exception('Failed to process %s %s/%s : %s', resource.kind, resource.namespace, resource.name, str(e))
def autoscale_resource( resource: pykube.objects.NamespacedAPIObject, upscale_period: str, downscale_period: str, default_uptime: str, default_downtime: str, forced_uptime: bool, dry_run: bool, now: datetime.datetime, grace_period: int = 0, downtime_replicas: int = 0, namespace_excluded=False, deployment_time_annotation: Optional[str] = None, ): try: exclude = namespace_excluded or ignore_resource(resource, now) original_replicas = resource.annotations.get( ORIGINAL_REPLICAS_ANNOTATION) downtime_replicas = int( resource.annotations.get(DOWNTIME_REPLICAS_ANNOTATION, downtime_replicas)) if exclude and not original_replicas: logger.debug( "%s %s/%s was excluded", resource.kind, resource.namespace, resource.name, ) else: ignore = False is_uptime = True upscale_period = resource.annotations.get( UPSCALE_PERIOD_ANNOTATION, upscale_period) downscale_period = resource.annotations.get( DOWNSCALE_PERIOD_ANNOTATION, downscale_period) if forced_uptime or (exclude and original_replicas): uptime = "forced" downtime = "ignored" is_uptime = True elif upscale_period != "never" or downscale_period != "never": uptime = upscale_period downtime = downscale_period if helper.matches_time_spec( now, uptime) and helper.matches_time_spec( now, downtime): logger.debug( "Upscale and downscale periods overlap, do nothing") ignore = True elif helper.matches_time_spec(now, uptime): is_uptime = True elif helper.matches_time_spec(now, downtime): is_uptime = False else: ignore = True logger.debug( "Periods checked: upscale=%s, downscale=%s, ignore=%s, is_uptime=%s", upscale_period, downscale_period, ignore, is_uptime, ) else: uptime = resource.annotations.get(UPTIME_ANNOTATION, default_uptime) downtime = resource.annotations.get(DOWNTIME_ANNOTATION, default_downtime) is_uptime = helper.matches_time_spec( now, uptime) and not helper.matches_time_spec(now, downtime) if resource.kind == "CronJob": suspended = resource.obj["spec"]["suspend"] replicas = 0 if suspended else 1 logger.debug( "%s %s/%s is %s (original: %s, uptime: %s)", resource.kind, resource.namespace, resource.name, "suspended" if suspended else "not suspended", "suspended" if original_replicas == 0 else "not suspended", uptime, ) else: replicas = resource.replicas logger.debug( "%s %s/%s has %s replicas (original: %s, uptime: %s)", resource.kind, resource.namespace, resource.name, replicas, original_replicas, uptime, ) update_needed = False if (not ignore and is_uptime and replicas == downtime_replicas and original_replicas and int(original_replicas) > 0): if resource.kind == "CronJob": resource.obj["spec"]["suspend"] = False resource.obj["spec"]["startingDeadlineSeconds"] = 0 logger.info( "Unsuspending %s %s/%s (uptime: %s, downtime: %s)", resource.kind, resource.namespace, resource.name, uptime, downtime, ) else: resource.replicas = int(original_replicas) logger.info( "Scaling up %s %s/%s from %s to %s replicas (uptime: %s, downtime: %s)", resource.kind, resource.namespace, resource.name, replicas, original_replicas, uptime, downtime, ) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = None update_needed = True elif (not ignore and not is_uptime and replicas > 0 and replicas > int(downtime_replicas)): target_replicas = int( resource.annotations.get(DOWNTIME_REPLICAS_ANNOTATION, downtime_replicas)) if within_grace_period(resource, grace_period, now, deployment_time_annotation): logger.info( "%s %s/%s within grace period (%ds), not scaling down (yet)", resource.kind, resource.namespace, resource.name, grace_period, ) else: if resource.kind == "CronJob": resource.obj["spec"]["suspend"] = True logger.info( "Suspending %s %s/%s (uptime: %s, downtime: %s)", resource.kind, resource.namespace, resource.name, uptime, downtime, ) else: resource.replicas = target_replicas logger.info( "Scaling down %s %s/%s from %s to %s replicas (uptime: %s, downtime: %s)", resource.kind, resource.namespace, resource.name, replicas, target_replicas, uptime, downtime, ) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = str( replicas) update_needed = True if update_needed: if dry_run: logger.info( "**DRY-RUN**: would update %s %s/%s", resource.kind, resource.namespace, resource.name, ) else: resource.update() except Exception as e: logger.exception( "Failed to process %s %s/%s : %s", resource.kind, resource.namespace, resource.name, str(e), )