def _set_scheduler_cloudwatch_rule_expression(expression, context=None):
"""
Sets a new expression for a CloudWatch rule
:param expression: new cloudwatch expression in the syntax cron(x x x x x x). Note that this format has an additional year
field that is not used by the scheduler.
:param context: Lambda execution context
:return:
"""
events_client = _get_cloudwatch_events_client(context)
event_rule = _get_cloudwatch_rule(SCHEDULE_RULE, events_client)
try:
# get the con expression from the expression
cron_str = " ".join(
expression[expression.index("(") +
1:expression.index(")")].split(" ")[0:5])
cron = CronExpression(cron_str)
next_execution_time = cron.first_within_next(
start_dt=datetime.utcnow(), timespan=timedelta(hours=24))
if next_execution_time is not None:
description = DESC_EXPRESSION_SET.format(
expression, next_execution_time.isoformat())
else:
description = DESC_NO_EXECUTIONS_FOR_EXPR
except ValueError:
description = ""
if event_rule["ScheduleExpression"] != expression:
args = {
"Name": event_rule["Name"],
"ScheduleExpression": expression,
"Description": description
}
events_client.put_rule_with_retries(**args)
def _set_scheduler_cloudwatch_rule_expression(expression,
context=None,
task=None,
logger=None):
"""
Sets a new expression for a CloudWatch rule
:param expression: new cloudwatch expression in the syntax cron(x x x x x x). Note that this format has an additional year
field that is not used by the scheduler.
:param context: Lambda execution context
:return:
"""
events_client = _get_cloudwatch_events_client(context)
event_rule = _get_cloudwatch_rule(os.getenv(ENV_OPS_AUTOMATOR_RULE),
events_client)
try:
# get the con expression from the expression
cron_str = " ".join(
expression[expression.index("(") +
1:expression.index(")")].split(" ")[0:5])
if logger is not None:
logger.info(INF_NEW_CRON, expression)
cron = CronExpression(cron_str)
next_execution_time = cron.first_within_next(
start_dt=datetime.utcnow(), timespan=timedelta(hours=24))
if next_execution_time is not None:
description = DESC_EXPRESSION_SET.format(expression)
if task is not None:
description += " for task {} scheduled at {}".format(
task.get(TASK_NAME, ""),
CronExpression(task[TASK_INTERVAL]).first_within_next(
start_dt=datetime.utcnow(),
timespan=timedelta(hours=24)).isoformat())
else:
description = DESC_NO_EXECUTIONS_FOR_EXPR
except ValueError:
description = ""
if event_rule["ScheduleExpression"] != expression or event_rule.get(
"Description", "") != description:
args = {
"Name": event_rule["Name"],
"ScheduleExpression": expression,
"Description": description
}
events_client.put_rule_with_retries(**args)
def _get_next_task_execution(name,
context=None,
logger=None,
days=None,
hours=None,
minutes=None,
include_disabled=False):
def get_period():
period = 0
if days is not None:
period += days * 60 * 24
if hours is not None:
period += 60 * hours
if minutes is not None:
period += minutes
return timedelta(minutes=period if period != 0 else 24 * 60)
result = {"Name": name}
task = _get_task(name, context=context, logger=logger)
enabled = task[configuration.CONFIG_ENABLED]
result["Enabled"] = enabled
if not enabled and include_disabled:
return safe_json(result)
task_interval = task.get(configuration.CONFIG_INTERVAL, None)
result["Interval"] = task_interval
if task_interval is None:
return safe_json(result)
task_cron_expression = CronExpression(expression=task_interval)
task_timezone = task.get(configuration.CONFIG_TIMEZONE, "UTC")
result["Timezone"] = task_timezone
now = datetime.now(tz=pytz.timezone(task_timezone))
next_execution = task_cron_expression.first_within_next(get_period(), now)
if next_execution is None:
return safe_json(result)
result["NextExecution"] = next_execution
return result
def handle_scheduler_tasks(self, task_config):
started_tasks = {}
start = datetime.now()
last_run_dt = self._get_last_run()
self._logger.info(INFO_LAST_SAVED, last_run_dt.isoformat())
if self.configuration_update:
self._logger.info(INFO_CONFIG_RUN, self.updated_task)
current_dt = self._set_last_run()
already_ran_this_minute = last_run_dt == current_dt
if already_ran_this_minute and not (self.configuration_update
or self.execute_task_request):
self._logger.info(INFO_TASK_SCHEDULER_ALREADY_RAN)
else:
self._logger.info(INFO_CURRENT_SCHEDULING_DT, current_dt)
task = None
enabled_tasks = 0
next_executed_task = None
utc = pytz.timezone("UTC")
tasks = [
t for t in task_config.get_tasks()
if t.get(handlers.TASK_INTERVAL) is not None
and t.get(handlers.TASK_ENABLED, True)
]
try:
for task in tasks:
enabled_tasks += 1
self._logger.debug_enabled = task[handlers.TASK_DEBUG]
task_name = task[handlers.TASK_NAME]
# timezone for specific task
task_timezone = pytz.timezone(task[handlers.TASK_TIMEZONE])
# create cron expression to test if task needs te be executed
task_cron_expression = CronExpression(
expression=task[handlers.TASK_INTERVAL])
localized_last_run = last_run_dt.astimezone(task_timezone)
localized_current_dt = current_dt.astimezone(task_timezone)
next_execution = task_cron_expression.first_within_next(
timedelta(hours=24), localized_current_dt)
next_execution_utc = next_execution.astimezone(
utc).replace(microsecond=0
) if next_execution is not None else None
if next_execution_utc is not None:
if next_executed_task is None or next_execution_utc < next_executed_task[
0]:
next_executed_task = (next_execution_utc, task)
if already_ran_this_minute:
continue
# test if task needs te be executed since last run of ops automator
execute_dt_since_last = task_cron_expression.last_since(
localized_last_run, localized_current_dt)
if execute_dt_since_last is None:
if next_execution is not None:
next_execution = next_execution.astimezone(
task_timezone)
self._logger.info(INFO_NEXT_EXECUTION, task_name,
next_execution.isoformat(),
task_timezone)
else:
self._logger.info(INFO_NO_NEXT_WITHIN, task_name)
continue
self._logger.info(INFO_SCHEDULED_TASK, task_name,
execute_dt_since_last, task_timezone,
str(safe_json(task, indent=2)))
# create an event for lambda function that starts execution by selecting for resources for this task
task_group, sub_tasks = self._execute_task(
task, execute_dt_since_last)
started_tasks[task_name] = {
"task-group": task_group,
"sub-tasks": sub_tasks
}
if started_tasks:
self._logger.info(INFO_STARTED_TASKS, enabled_tasks,
",".join(started_tasks))
else:
self._logger.info(INFO_NO_TASKS_STARTED, enabled_tasks)
self._set_next_schedule_event(current_dt, next_executed_task)
running_time = float((datetime.now() - start).total_seconds())
return safe_dict({
"datetime": datetime.now().isoformat(),
"running-time": running_time,
"event-datetime": current_dt.isoformat(),
"enabled_tasks": enabled_tasks,
"started-tasks": started_tasks
})
except ValueError as ex:
self._logger.error(ERR_SCHEDULE_HANDLER, ex,
safe_json(task, indent=2))
def handle_request(self):
"""
Handles the cloudwatch rule timer event
:return: Started tasks, if any, information
"""
try:
started_tasks = []
start = datetime.now()
last_run_dt = self._get_last_run()
self._logger.info("Handler {}", self.__class__.__name__)
self._logger.info(INFO_LAST_SAVED, str(last_run_dt))
if self.configuration_update:
self._logger.info(INFO_CONFIG_RUN, self.updated_task)
# test if we already executed in this minute
current_dt = self._set_last_run()
already_ran_this_minute = last_run_dt == current_dt
if already_ran_this_minute and not self.configuration_update:
self._logger.info(INFO_TASK_SCHEDULER_ALREADY_RAN)
else:
self._logger.info(INFO_CURRENT_SCHEDULING_DT, current_dt)
task = None
enabled_tasks = 0
next_executed_task = None
utc = pytz.timezone("UTC")
try:
for task in [t for t in TaskConfiguration(context=self._context, logger=self._logger).get_tasks() if
t.get(handlers.TASK_INTERVAL) is not None
and t.get(handlers.TASK_ENABLED, True)]:
enabled_tasks += 1
self._logger.debug_enabled = task[handlers.TASK_DEBUG]
task_name = task[handlers.TASK_NAME]
# timezone for specific task
task_timezone = pytz.timezone(task[handlers.TASK_TIMEZONE])
# create cron expression to test if task needs te be executed
task_cron_expression = CronExpression(expression=task[handlers.TASK_INTERVAL])
localized_last_run = last_run_dt.astimezone(task_timezone)
localized_current_dt = current_dt.astimezone(task_timezone)
next_execution = task_cron_expression.first_within_next(timedelta(hours=24), localized_current_dt)
next_execution_utc = next_execution.astimezone(utc) if next_execution else None
if next_execution_utc is not None:
if next_executed_task is None or next_execution_utc < next_executed_task[0]:
next_executed_task = (next_execution_utc, task)
if already_ran_this_minute:
continue
# test if task needs te be executed since last run of ops automator
execute_dt_since_last = task_cron_expression.last_since(localized_last_run, localized_current_dt)
if execute_dt_since_last is None:
if next_execution is not None:
next_execution = next_execution.astimezone(task_timezone)
self._logger.info(INFO_NEXT_EXECUTION, task_name, next_execution.isoformat(), task_timezone)
else:
self._logger.info(INFO_NO_NEXT_WITHIN, task_name)
continue
started_tasks.append(task_name)
self._logger.debug(INFO_SCHEDULED_TASK, task_name, execute_dt_since_last, task_timezone,
str(safe_json(task, indent=2)))
# create an event for lambda function that starts execution by selecting for resources for this task
self._execute_task(task, execute_dt_since_last)
if started_tasks:
self._logger.info(INFO_STARTED_TASKS, enabled_tasks, ",".join(started_tasks))
else:
self._logger.info(INFO_NO_TASKS_STARTED, enabled_tasks)
self._set_next_schedule_event(current_dt, next_executed_task)
running_time = float((datetime.now() - start).total_seconds())
self._logger.info(INFO_RESULT, running_time)
return safe_dict({
"datetime": datetime.now().isoformat(),
"running-time": running_time,
"event-datetime": current_dt.isoformat(),
"enabled_tasks": enabled_tasks,
"started-tasks": started_tasks
})
except ValueError as ex:
self._logger.error("{}\n{}".format(ex, safe_json(task, indent=2)))
finally:
self._logger.flush()