def run(self, state_table, scheduler_config, logger, lambda_account=None, context=None): """ Runs the scheduler for a service :param state_table: name of the instance state table :param scheduler_config: scheduler configuration data :param logger: logger to log output of scheduling process :param lambda_account: lambda account :param context: Lambda context :return: """ self._lambda_account = lambda_account self._configuration = scheduler_config self._logger = logger self._context = context self._logger.debug_enabled = self._configuration.trace # stored instance desired states self._instance_states = InstanceStates(state_table, self._service.service_name, self._logger, self._context) # time to use for metrics self._schedule_metrics = SchedulerMetrics(datetime.utcnow(), self._context) # response to caller, contains list off all processed accounts with started and stopped instances response = {} for account in self._accounts: response[account.name] = self._process_account(account) if allow_send_metrics(): self._send_usage_metrics() return response
def handle_metrics(result): self._logger.info( "Sending metrics data is {}", "enabled" if allow_send_metrics() else "disabled") if allow_send_metrics(): try: result_data = result if isinstance( result, dict) else json.loads(result) if actions.METRICS_DATA in result_data: send_metrics_data( metrics=result_data[actions.METRICS_DATA], logger=self._logger) except Exception as ex: self._logger.warning( "Error processing or sending metrics data ({})", str(ex))
def _delete_request(self): """ Handles delete request from cloudformation custom resource :return: """ try: self.delete_templates() self.delete_external_task_config_stacks() if allow_send_metrics(): self._send_delete_metrics() return True except Exception as ex: self.response["Reason"] = str(ex) return False
def _create_request(self): """ Handles create request from cloudformation custom resource :return: """ try: self._setup() self.physical_resource_id = self.__class__.__name__.lower() if allow_send_metrics(): self._send_create_metrics() return True except Exception as ex: self.response["Reason"] = str(ex) return False
def _delete_request(self): if allow_send_metrics(): self._send_delete_metrics() return True
def _create_request(self): if allow_send_metrics(): self._send_create_metrics() self._create_sample_schemas() return self._update_settings( ) and self.set_lambda_logs_retention_period()
def _process_account(self, account): # processes instances for a service in an account started_instances = {} stopped_instances = {} resized_instances = {} self._logger.info(INF_PROCESSING_ACCOUNT, self._service.service_name.upper(), account.name, " using role " + account.role if account.role else "", ", ".join(self._configuration.regions)) # gets the desired state and type, uses caching for each schedule def get_desired_state_and_type(schedule, inst): # test if the instance has a maintenance window in which it must be running if instance.maintenance_window is not None and schedule.use_maintenance_window is True: self._logger.info(INF_MAINTENANCE_WINDOW) # get the desired start for the maintenance window at current UTC time inst_state, inst_type, running_period = instance.maintenance_window.get_desired_state( inst, logger=self._logger, dt=datetime.utcnow().replace(tzinfo=pytz.timezone("UTC"))) # if we're in the maintenance window return running state if inst_state == InstanceSchedule.STATE_RUNNING: return inst_state, inst_type # based on the schedule get the desired state and instance type for this instance inst_state, inst_type, _ = instance_schedule.get_desired_state(inst, logger=self._logger) return inst_state, inst_type for region in self._regions: state_loaded = False instances = [] self._scheduler_start_list = [] self._scheduler_stop_list = [] self._schedule_resize_list = [] for instance in self._scheduled_instances_in_region(account, region): # delay loading instance state until first instance is returned if not state_loaded: self._instance_states.load(account.name, region) state_loaded = True instances.append(instance) # handle terminated instances if instance.is_terminated: self._logger.debug(DEBUG_SKIPPING_TERMINATED_INSTANCE, instance.instance_str, region, instance.account) self._instance_states.delete_instance_state(instance.id) continue # get the schedule for this instance instance_schedule = self._configuration.get_schedule(instance.schedule_name) if not instance_schedule: self._logger.warning(WARN_SKIPPING_UNKNOWN_SCHEDULE, instance.instance_str, region, instance.account, instance.schedule_name) continue self._logger.debug(DEBUG_INSTANCE_HEADER, instance.instance_str) self._logger.debug(DEBUG_CURRENT_INSTANCE_STATE, instance.current_state, instance.instancetype, instance_schedule.name) # based on the schedule get the desired state and instance type for this instance desired_state, desired_type = get_desired_state_and_type(instance_schedule, instance) # get the previous desired instance state last_desired_state = self._instance_states.get_instance_state(instance.id) self._logger.debug(DEBUG_CURRENT_AND_DESIRED_STATE, instance_schedule.name, desired_state, last_desired_state, instance.current_state, INF_DESIRED_TYPE.format(desired_type) if desired_type else "") # last desired state None means this is the first time the instance is seen by the scheduler if last_desired_state is InstanceSchedule.STATE_UNKNOWN: # new instances that are running are optionally not stopped to allow them to finish possible initialization if instance.is_running and desired_state == InstanceSchedule.STATE_STOPPED: if not instance_schedule.stop_new_instances: self._logger.debug(DEBUG_NEW_INSTANCE, instance.instance_str) continue self._process_new_desired_state(account, region, instance, desired_state, desired_type, last_desired_state, instance_schedule.retain_running) else: self._process_new_desired_state(account, region, instance, desired_state, desired_type, last_desired_state, instance_schedule.retain_running) # existing instance # if enforced check the actual state with the desired state enforcing the schedule state elif instance_schedule.enforced: if (instance.is_running and desired_state == InstanceSchedule.STATE_STOPPED) or ( not instance.is_running and desired_state == InstanceSchedule.STATE_RUNNING): self._logger.debug(DEBUG_ENFORCED_STATE, instance.instance_str, InstanceSchedule.STATE_RUNNING if instance.is_running else InstanceSchedule.STATE_STOPPED, desired_state) self._process_new_desired_state(account, region, instance, desired_state, desired_type, last_desired_state, instance_schedule.retain_running) # if not enforced then compare the schedule state with the actual state so state of manually started/stopped # instance it will honor that state elif last_desired_state != desired_state: self._process_new_desired_state(account, region, instance, desired_state, desired_type, last_desired_state, instance_schedule.retain_running) self._schedule_metrics.add_schedule_metrics(self._service.service_name, instance_schedule, instance) # process lists of instances that must be started or stopped self._start_and_stop_instances(account, region=region) # cleanup desired instance states and save self._instance_states.cleanup([i.id for i in instances]) self._instance_states.save() # build output structure, hold started, stopped and resized instances per region if len(self._scheduler_start_list) > 0: started_instances[region] = [{i.id: {"schedule": i.schedule_name}} for i in self._scheduler_start_list] if len(self._scheduler_stop_list): stopped_instances[region] = [{i.id: {"schedule": i.schedule_name}} for i in self._scheduler_stop_list] if len(self._schedule_resize_list) > 0: resized_instances[region] = [{i[0].id: {"schedule": i[0].schedule_name, "old": i[0].instancetype, "new": i[1]}} for i in self._schedule_resize_list] if allow_send_metrics(): self._collect_usage_metrics() # put cloudwatch metrics if self._configuration.use_metrics: self._schedule_metrics.put_schedule_metrics() # output data result = {"started": started_instances, "stopped": stopped_instances} if self._service.allow_resize: result["resized"] = resized_instances return result