def run(self, state_table, scheduler_config, logger, lambda_account=None, context=None):
"""
Runs the scheduler for a service
:param state_table: name of the instance state table
:param scheduler_config: scheduler configuration data
:param logger: logger to log output of scheduling process
:param lambda_account: lambda account
:param context: Lambda context
:return:
"""
self._lambda_account = lambda_account
self._configuration = scheduler_config
self._logger = logger
self._context = context
self._logger.debug_enabled = self._configuration.trace
# stored instance desired states
self._instance_states = InstanceStates(state_table, self._service.service_name, self._logger, self._context)
# time to use for metrics
self._schedule_metrics = SchedulerMetrics(datetime.utcnow(), self._context)
# response to caller, contains list off all processed accounts with started and stopped instances
response = {}
for account in self._accounts:
response[account.name] = self._process_account(account)
if allow_send_metrics():
self._send_usage_metrics()
return response
def handle_metrics(result):
self._logger.info(
"Sending metrics data is {}",
"enabled" if allow_send_metrics() else "disabled")
if allow_send_metrics():
try:
result_data = result if isinstance(
result, dict) else json.loads(result)
if actions.METRICS_DATA in result_data:
send_metrics_data(
metrics=result_data[actions.METRICS_DATA],
logger=self._logger)
except Exception as ex:
self._logger.warning(
"Error processing or sending metrics data ({})",
str(ex))
def _delete_request(self):
"""
Handles delete request from cloudformation custom resource
:return:
"""
try:
self.delete_templates()
self.delete_external_task_config_stacks()
if allow_send_metrics():
self._send_delete_metrics()
return True
except Exception as ex:
self.response["Reason"] = str(ex)
return False
def _create_request(self):
"""
Handles create request from cloudformation custom resource
:return:
"""
try:
self._setup()
self.physical_resource_id = self.__class__.__name__.lower()
if allow_send_metrics():
self._send_create_metrics()
return True
except Exception as ex:
self.response["Reason"] = str(ex)
return False
def _delete_request(self):
if allow_send_metrics():
self._send_delete_metrics()
return True
def _create_request(self):
if allow_send_metrics():
self._send_create_metrics()
self._create_sample_schemas()
return self._update_settings(
) and self.set_lambda_logs_retention_period()
def _process_account(self, account):
# processes instances for a service in an account
started_instances = {}
stopped_instances = {}
resized_instances = {}
self._logger.info(INF_PROCESSING_ACCOUNT,
self._service.service_name.upper(), account.name, " using role " + account.role if account.role else "",
", ".join(self._configuration.regions))
# gets the desired state and type, uses caching for each schedule
def get_desired_state_and_type(schedule, inst):
# test if the instance has a maintenance window in which it must be running
if instance.maintenance_window is not None and schedule.use_maintenance_window is True:
self._logger.info(INF_MAINTENANCE_WINDOW)
# get the desired start for the maintenance window at current UTC time
inst_state, inst_type, running_period = instance.maintenance_window.get_desired_state(
inst, logger=self._logger, dt=datetime.utcnow().replace(tzinfo=pytz.timezone("UTC")))
# if we're in the maintenance window return running state
if inst_state == InstanceSchedule.STATE_RUNNING:
return inst_state, inst_type
# based on the schedule get the desired state and instance type for this instance
inst_state, inst_type, _ = instance_schedule.get_desired_state(inst, logger=self._logger)
return inst_state, inst_type
for region in self._regions:
state_loaded = False
instances = []
self._scheduler_start_list = []
self._scheduler_stop_list = []
self._schedule_resize_list = []
for instance in self._scheduled_instances_in_region(account, region):
# delay loading instance state until first instance is returned
if not state_loaded:
self._instance_states.load(account.name, region)
state_loaded = True
instances.append(instance)
# handle terminated instances
if instance.is_terminated:
self._logger.debug(DEBUG_SKIPPING_TERMINATED_INSTANCE, instance.instance_str, region, instance.account)
self._instance_states.delete_instance_state(instance.id)
continue
# get the schedule for this instance
instance_schedule = self._configuration.get_schedule(instance.schedule_name)
if not instance_schedule:
self._logger.warning(WARN_SKIPPING_UNKNOWN_SCHEDULE, instance.instance_str, region, instance.account,
instance.schedule_name)
continue
self._logger.debug(DEBUG_INSTANCE_HEADER, instance.instance_str)
self._logger.debug(DEBUG_CURRENT_INSTANCE_STATE, instance.current_state, instance.instancetype,
instance_schedule.name)
# based on the schedule get the desired state and instance type for this instance
desired_state, desired_type = get_desired_state_and_type(instance_schedule, instance)
# get the previous desired instance state
last_desired_state = self._instance_states.get_instance_state(instance.id)
self._logger.debug(DEBUG_CURRENT_AND_DESIRED_STATE, instance_schedule.name, desired_state, last_desired_state,
instance.current_state, INF_DESIRED_TYPE.format(desired_type) if desired_type else "")
# last desired state None means this is the first time the instance is seen by the scheduler
if last_desired_state is InstanceSchedule.STATE_UNKNOWN:
# new instances that are running are optionally not stopped to allow them to finish possible initialization
if instance.is_running and desired_state == InstanceSchedule.STATE_STOPPED:
if not instance_schedule.stop_new_instances:
self._logger.debug(DEBUG_NEW_INSTANCE, instance.instance_str)
continue
self._process_new_desired_state(account, region, instance, desired_state, desired_type, last_desired_state,
instance_schedule.retain_running)
else:
self._process_new_desired_state(account, region, instance, desired_state, desired_type, last_desired_state,
instance_schedule.retain_running)
# existing instance
# if enforced check the actual state with the desired state enforcing the schedule state
elif instance_schedule.enforced:
if (instance.is_running and desired_state == InstanceSchedule.STATE_STOPPED) or (
not instance.is_running and desired_state == InstanceSchedule.STATE_RUNNING):
self._logger.debug(DEBUG_ENFORCED_STATE, instance.instance_str,
InstanceSchedule.STATE_RUNNING
if instance.is_running
else InstanceSchedule.STATE_STOPPED,
desired_state)
self._process_new_desired_state(account, region, instance, desired_state, desired_type, last_desired_state,
instance_schedule.retain_running)
# if not enforced then compare the schedule state with the actual state so state of manually started/stopped
# instance it will honor that state
elif last_desired_state != desired_state:
self._process_new_desired_state(account, region, instance, desired_state, desired_type, last_desired_state,
instance_schedule.retain_running)
self._schedule_metrics.add_schedule_metrics(self._service.service_name, instance_schedule, instance)
# process lists of instances that must be started or stopped
self._start_and_stop_instances(account, region=region)
# cleanup desired instance states and save
self._instance_states.cleanup([i.id for i in instances])
self._instance_states.save()
# build output structure, hold started, stopped and resized instances per region
if len(self._scheduler_start_list) > 0:
started_instances[region] = [{i.id: {"schedule": i.schedule_name}} for i in self._scheduler_start_list]
if len(self._scheduler_stop_list):
stopped_instances[region] = [{i.id: {"schedule": i.schedule_name}} for i in self._scheduler_stop_list]
if len(self._schedule_resize_list) > 0:
resized_instances[region] = [{i[0].id: {"schedule": i[0].schedule_name, "old": i[0].instancetype, "new": i[1]}} for
i in self._schedule_resize_list]
if allow_send_metrics():
self._collect_usage_metrics()
# put cloudwatch metrics
if self._configuration.use_metrics:
self._schedule_metrics.put_schedule_metrics()
# output data
result = {"started": started_instances, "stopped": stopped_instances}
if self._service.allow_resize:
result["resized"] = resized_instances
return result
