def scale_up( resource: NamespacedAPIObject, replicas: int, original_replicas: int, original_max_replicas: int, uptime, downtime, ): if resource.kind == "CronJob": resource.obj["spec"]["suspend"] = False logger.info( f"Unsuspending {resource.kind} {resource.namespace}/{resource.name} (uptime: {uptime}, downtime: {downtime})" ) elif resource.kind == "HorizontalPodAutoscaler": resource.obj["spec"]["minReplicas"] = original_replicas logger.info( f"Scaling up {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {original_replicas} minReplicas (uptime: {uptime}, downtime: {downtime})" ) else: resource.obj["spec"]["template"]["metadata"]["annotations"]["hpa.autoscaling.banzaicloud.io/minReplicas"] = str(original_replicas) resource.obj["spec"]["template"]["metadata"]["annotations"]["hpa.autoscaling.banzaicloud.io/maxReplicas"] = str(original_max_replicas) logger.info( f"Scaling up {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {original_replicas} replicas (uptime: {uptime}, downtime: {downtime})" ) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = None resource.annotations[ORIGINAL_MAX_REPLICAS_ANNOTATION] = None
def scale_down( resource: NamespacedAPIObject, replicas: int, target_replicas: int, uptime, downtime, dry_run: bool, enable_events: bool, ): event_message = "Scaling down replicas" if resource.kind == "CronJob": resource.obj["spec"]["suspend"] = True logger.info( f"Suspending {resource.kind} {resource.namespace}/{resource.name} (uptime: {uptime}, downtime: {downtime})" ) event_message = "Suspending CronJob" elif resource.kind == "HorizontalPodAutoscaler": resource.obj["spec"]["minReplicas"] = target_replicas logger.info( f"Scaling down {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {target_replicas} minReplicas (uptime: {uptime}, downtime: {downtime})" ) else: resource.replicas = target_replicas logger.info( f"Scaling down {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {target_replicas} replicas (uptime: {uptime}, downtime: {downtime})" ) if enable_events: helper.add_event( resource, event_message, "ScaleDown", "Normal", dry_run, ) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = str(replicas)
def scale_up( resource: NamespacedAPIObject, replicas: int, original_replicas: int, uptime, downtime, dry_run: bool, enable_events: bool, upscale_step_size: int, ): # Increase replicas by upscale_step_size, but not greater than original_replicas. if upscale_step_size > 0: new_replicas = replicas + upscale_step_size if new_replicas > original_replicas: new_replicas = original_replicas else: new_replicas = original_replicas event_message = "Scaling up replicas" if resource.kind == "CronJob": resource.obj["spec"]["suspend"] = False logger.info( f"Unsuspending {resource.kind} {resource.namespace}/{resource.name} (uptime: {uptime}, downtime: {downtime})" ) event_message = "Unsuspending CronJob" elif resource.kind == "HorizontalPodAutoscaler": resource.obj["spec"]["minReplicas"] = new_replicas logger.info( f"Scaling up {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {new_replicas} minReplicas (uptime: {uptime}, downtime: {downtime})" ) else: resource.replicas = new_replicas logger.info( f"Scaling up {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {new_replicas} replicas (uptime: {uptime}, downtime: {downtime})" ) if enable_events: helper.add_event( resource, event_message, "ScaleUp", "Normal", dry_run, ) if new_replicas == original_replicas: # If scaling up is done. resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = None
def scale_down(resource: NamespacedAPIObject, replicas: int, target_replicas: int, uptime, downtime): if resource.kind == "CronJob": resource.obj["spec"]["suspend"] = True logger.info( f"Suspending {resource.kind} {resource.namespace}/{resource.name} (uptime: {uptime}, downtime: {downtime})" ) elif resource.kind == "HorizontalPodAutoscaler": resource.obj["spec"]["minReplicas"] = target_replicas logger.info( f"Scaling down {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {target_replicas} minReplicas (uptime: {uptime}, downtime: {downtime})" ) else: resource.replicas = target_replicas logger.info( f"Scaling down {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {target_replicas} replicas (uptime: {uptime}, downtime: {downtime})" ) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = str(replicas)
def scale_down( resource: NamespacedAPIObject, replicas: int, target_replicas: int, uptime, downtime ): resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = str(resource.obj["spec"]["template"]["metadata"]["annotations"]["hpa.autoscaling.banzaicloud.io/minReplicas"]) resource.annotations[ORIGINAL_MAX_REPLICAS_ANNOTATION] = str(resource.obj["spec"]["template"]["metadata"]["annotations"]["hpa.autoscaling.banzaicloud.io/maxReplicas"]) if resource.kind == "CronJob": resource.obj["spec"]["suspend"] = True logger.info( f"Suspending {resource.kind} {resource.namespace}/{resource.name} (uptime: {uptime}, downtime: {downtime})" ) elif resource.kind == "HorizontalPodAutoscaler": resource.obj["spec"]["minReplicas"] = target_replicas logger.info( f"Scaling down {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {target_replicas} minReplicas (uptime: {uptime}, downtime: {downtime})" ) else: resource.obj["spec"]["template"]["metadata"]["annotations"]["hpa.autoscaling.banzaicloud.io/minReplicas"] = str(target_replicas) resource.obj["spec"]["template"]["metadata"]["annotations"]["hpa.autoscaling.banzaicloud.io/maxReplicas"] = str(target_replicas) logger.info( f"Scaling down {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {target_replicas} replicas (uptime: {uptime}, downtime: {downtime})" )
def scale_up( resource: NamespacedAPIObject, replicas: int, original_replicas: int, uptime, downtime, ): if resource.kind == "CronJob": resource.obj["spec"]["suspend"] = False logger.info( f"Unsuspending {resource.kind} {resource.namespace}/{resource.name} (uptime: {uptime}, downtime: {downtime})" ) elif resource.kind == "HorizontalPodAutoscaler": resource.obj["spec"]["minReplicas"] = original_replicas logger.info( f"Scaling up {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {original_replicas} minReplicas (uptime: {uptime}, downtime: {downtime})" ) else: resource.replicas = original_replicas logger.info( f"Scaling up {resource.kind} {resource.namespace}/{resource.name} from {replicas} to {original_replicas} replicas (uptime: {uptime}, downtime: {downtime})" ) resource.annotations[ORIGINAL_REPLICAS_ANNOTATION] = None
def autoscale_resource( resource: NamespacedAPIObject, upscale_period: str, downscale_period: str, default_uptime: str, default_downtime: str, forced_uptime: bool, dry_run: bool, now: datetime.datetime, grace_period: int = 0, downtime_replicas: int = 0, namespace_excluded=False, deployment_time_annotation: Optional[str] = None, ): try: exclude = namespace_excluded or ignore_resource(resource, now) original_replicas = get_annotation_value_as_int( resource, ORIGINAL_REPLICAS_ANNOTATION) downtime_replicas_from_annotation = get_annotation_value_as_int( resource, DOWNTIME_REPLICAS_ANNOTATION) if downtime_replicas_from_annotation is not None: downtime_replicas = downtime_replicas_from_annotation if exclude and not original_replicas: logger.debug( f"{resource.kind} {resource.namespace}/{resource.name} was excluded" ) else: ignore = False is_uptime = True upscale_period = resource.annotations.get( UPSCALE_PERIOD_ANNOTATION, upscale_period) downscale_period = resource.annotations.get( DOWNSCALE_PERIOD_ANNOTATION, downscale_period) if forced_uptime or (exclude and original_replicas): uptime = "forced" downtime = "ignored" is_uptime = True elif upscale_period != "never" or downscale_period != "never": uptime = upscale_period downtime = downscale_period if matches_time_spec(now, uptime) and matches_time_spec( now, downtime): logger.debug( "Upscale and downscale periods overlap, do nothing") ignore = True elif matches_time_spec(now, uptime): is_uptime = True elif matches_time_spec(now, downtime): is_uptime = False else: ignore = True logger.debug( f"Periods checked: upscale={upscale_period}, downscale={downscale_period}, ignore={ignore}, is_uptime={is_uptime}" ) else: uptime = resource.annotations.get(UPTIME_ANNOTATION, default_uptime) downtime = resource.annotations.get(DOWNTIME_ANNOTATION, default_downtime) is_uptime = matches_time_spec( now, uptime) and not matches_time_spec(now, downtime) replicas = get_replicas(resource, original_replicas, uptime) update_needed = False if (not ignore and is_uptime and replicas == downtime_replicas and original_replicas and original_replicas > 0): scale_up(resource, replicas, original_replicas, uptime, downtime) update_needed = True elif (not ignore and not is_uptime and replicas > 0 and replicas > downtime_replicas): if within_grace_period(resource, grace_period, now, deployment_time_annotation): logger.info( f"{resource.kind} {resource.namespace}/{resource.name} within grace period ({grace_period}s), not scaling down (yet)" ) else: scale_down(resource, replicas, downtime_replicas, uptime, downtime) update_needed = True if update_needed: if dry_run: logger.info( f"**DRY-RUN**: would update {resource.kind} {resource.namespace}/{resource.name}" ) else: resource.update() except Exception as e: logger.exception( f"Failed to process {resource.kind} {resource.namespace}/{resource.name}: {e}" )