def _set_scheduler_cloudwatch_rule_expression(expression, context=None): """ Sets a new expression for a CloudWatch rule :param expression: new cloudwatch expression in the syntax cron(x x x x x x). Note that this format has an additional year field that is not used by the scheduler. :param context: Lambda execution context :return: """ events_client = _get_cloudwatch_events_client(context) event_rule = _get_cloudwatch_rule(SCHEDULE_RULE, events_client) try: # get the con expression from the expression cron_str = " ".join( expression[expression.index("(") + 1:expression.index(")")].split(" ")[0:5]) cron = CronExpression(cron_str) next_execution_time = cron.first_within_next( start_dt=datetime.utcnow(), timespan=timedelta(hours=24)) if next_execution_time is not None: description = DESC_EXPRESSION_SET.format( expression, next_execution_time.isoformat()) else: description = DESC_NO_EXECUTIONS_FOR_EXPR except ValueError: description = "" if event_rule["ScheduleExpression"] != expression: args = { "Name": event_rule["Name"], "ScheduleExpression": expression, "Description": description } events_client.put_rule_with_retries(**args)
def _set_scheduler_cloudwatch_rule_expression(expression, context=None, task=None, logger=None): """ Sets a new expression for a CloudWatch rule :param expression: new cloudwatch expression in the syntax cron(x x x x x x). Note that this format has an additional year field that is not used by the scheduler. :param context: Lambda execution context :return: """ events_client = _get_cloudwatch_events_client(context) event_rule = _get_cloudwatch_rule(os.getenv(ENV_OPS_AUTOMATOR_RULE), events_client) try: # get the con expression from the expression cron_str = " ".join( expression[expression.index("(") + 1:expression.index(")")].split(" ")[0:5]) if logger is not None: logger.info(INF_NEW_CRON, expression) cron = CronExpression(cron_str) next_execution_time = cron.first_within_next( start_dt=datetime.utcnow(), timespan=timedelta(hours=24)) if next_execution_time is not None: description = DESC_EXPRESSION_SET.format(expression) if task is not None: description += " for task {} scheduled at {}".format( task.get(TASK_NAME, ""), CronExpression(task[TASK_INTERVAL]).first_within_next( start_dt=datetime.utcnow(), timespan=timedelta(hours=24)).isoformat()) else: description = DESC_NO_EXECUTIONS_FOR_EXPR except ValueError: description = "" if event_rule["ScheduleExpression"] != expression or event_rule.get( "Description", "") != description: args = { "Name": event_rule["Name"], "ScheduleExpression": expression, "Description": description } events_client.put_rule_with_retries(**args)
def _get_next_task_execution(name, context=None, logger=None, days=None, hours=None, minutes=None, include_disabled=False): def get_period(): period = 0 if days is not None: period += days * 60 * 24 if hours is not None: period += 60 * hours if minutes is not None: period += minutes return timedelta(minutes=period if period != 0 else 24 * 60) result = {"Name": name} task = _get_task(name, context=context, logger=logger) enabled = task[configuration.CONFIG_ENABLED] result["Enabled"] = enabled if not enabled and include_disabled: return safe_json(result) task_interval = task.get(configuration.CONFIG_INTERVAL, None) result["Interval"] = task_interval if task_interval is None: return safe_json(result) task_cron_expression = CronExpression(expression=task_interval) task_timezone = task.get(configuration.CONFIG_TIMEZONE, "UTC") result["Timezone"] = task_timezone now = datetime.now(tz=pytz.timezone(task_timezone)) next_execution = task_cron_expression.first_within_next(get_period(), now) if next_execution is None: return safe_json(result) result["NextExecution"] = next_execution return result
def handle_scheduler_tasks(self, task_config): started_tasks = {} start = datetime.now() last_run_dt = self._get_last_run() self._logger.info(INFO_LAST_SAVED, last_run_dt.isoformat()) if self.configuration_update: self._logger.info(INFO_CONFIG_RUN, self.updated_task) current_dt = self._set_last_run() already_ran_this_minute = last_run_dt == current_dt if already_ran_this_minute and not (self.configuration_update or self.execute_task_request): self._logger.info(INFO_TASK_SCHEDULER_ALREADY_RAN) else: self._logger.info(INFO_CURRENT_SCHEDULING_DT, current_dt) task = None enabled_tasks = 0 next_executed_task = None utc = pytz.timezone("UTC") tasks = [ t for t in task_config.get_tasks() if t.get(handlers.TASK_INTERVAL) is not None and t.get(handlers.TASK_ENABLED, True) ] try: for task in tasks: enabled_tasks += 1 self._logger.debug_enabled = task[handlers.TASK_DEBUG] task_name = task[handlers.TASK_NAME] # timezone for specific task task_timezone = pytz.timezone(task[handlers.TASK_TIMEZONE]) # create cron expression to test if task needs te be executed task_cron_expression = CronExpression( expression=task[handlers.TASK_INTERVAL]) localized_last_run = last_run_dt.astimezone(task_timezone) localized_current_dt = current_dt.astimezone(task_timezone) next_execution = task_cron_expression.first_within_next( timedelta(hours=24), localized_current_dt) next_execution_utc = next_execution.astimezone( utc).replace(microsecond=0 ) if next_execution is not None else None if next_execution_utc is not None: if next_executed_task is None or next_execution_utc < next_executed_task[ 0]: next_executed_task = (next_execution_utc, task) if already_ran_this_minute: continue # test if task needs te be executed since last run of ops automator execute_dt_since_last = task_cron_expression.last_since( localized_last_run, localized_current_dt) if execute_dt_since_last is None: if next_execution is not None: next_execution = next_execution.astimezone( task_timezone) self._logger.info(INFO_NEXT_EXECUTION, task_name, next_execution.isoformat(), task_timezone) else: self._logger.info(INFO_NO_NEXT_WITHIN, task_name) continue self._logger.info(INFO_SCHEDULED_TASK, task_name, execute_dt_since_last, task_timezone, str(safe_json(task, indent=2))) # create an event for lambda function that starts execution by selecting for resources for this task task_group, sub_tasks = self._execute_task( task, execute_dt_since_last) started_tasks[task_name] = { "task-group": task_group, "sub-tasks": sub_tasks } if started_tasks: self._logger.info(INFO_STARTED_TASKS, enabled_tasks, ",".join(started_tasks)) else: self._logger.info(INFO_NO_TASKS_STARTED, enabled_tasks) self._set_next_schedule_event(current_dt, next_executed_task) running_time = float((datetime.now() - start).total_seconds()) return safe_dict({ "datetime": datetime.now().isoformat(), "running-time": running_time, "event-datetime": current_dt.isoformat(), "enabled_tasks": enabled_tasks, "started-tasks": started_tasks }) except ValueError as ex: self._logger.error(ERR_SCHEDULE_HANDLER, ex, safe_json(task, indent=2))
def handle_request(self): """ Handles the cloudwatch rule timer event :return: Started tasks, if any, information """ try: started_tasks = [] start = datetime.now() last_run_dt = self._get_last_run() self._logger.info("Handler {}", self.__class__.__name__) self._logger.info(INFO_LAST_SAVED, str(last_run_dt)) if self.configuration_update: self._logger.info(INFO_CONFIG_RUN, self.updated_task) # test if we already executed in this minute current_dt = self._set_last_run() already_ran_this_minute = last_run_dt == current_dt if already_ran_this_minute and not self.configuration_update: self._logger.info(INFO_TASK_SCHEDULER_ALREADY_RAN) else: self._logger.info(INFO_CURRENT_SCHEDULING_DT, current_dt) task = None enabled_tasks = 0 next_executed_task = None utc = pytz.timezone("UTC") try: for task in [t for t in TaskConfiguration(context=self._context, logger=self._logger).get_tasks() if t.get(handlers.TASK_INTERVAL) is not None and t.get(handlers.TASK_ENABLED, True)]: enabled_tasks += 1 self._logger.debug_enabled = task[handlers.TASK_DEBUG] task_name = task[handlers.TASK_NAME] # timezone for specific task task_timezone = pytz.timezone(task[handlers.TASK_TIMEZONE]) # create cron expression to test if task needs te be executed task_cron_expression = CronExpression(expression=task[handlers.TASK_INTERVAL]) localized_last_run = last_run_dt.astimezone(task_timezone) localized_current_dt = current_dt.astimezone(task_timezone) next_execution = task_cron_expression.first_within_next(timedelta(hours=24), localized_current_dt) next_execution_utc = next_execution.astimezone(utc) if next_execution else None if next_execution_utc is not None: if next_executed_task is None or next_execution_utc < next_executed_task[0]: next_executed_task = (next_execution_utc, task) if already_ran_this_minute: continue # test if task needs te be executed since last run of ops automator execute_dt_since_last = task_cron_expression.last_since(localized_last_run, localized_current_dt) if execute_dt_since_last is None: if next_execution is not None: next_execution = next_execution.astimezone(task_timezone) self._logger.info(INFO_NEXT_EXECUTION, task_name, next_execution.isoformat(), task_timezone) else: self._logger.info(INFO_NO_NEXT_WITHIN, task_name) continue started_tasks.append(task_name) self._logger.debug(INFO_SCHEDULED_TASK, task_name, execute_dt_since_last, task_timezone, str(safe_json(task, indent=2))) # create an event for lambda function that starts execution by selecting for resources for this task self._execute_task(task, execute_dt_since_last) if started_tasks: self._logger.info(INFO_STARTED_TASKS, enabled_tasks, ",".join(started_tasks)) else: self._logger.info(INFO_NO_TASKS_STARTED, enabled_tasks) self._set_next_schedule_event(current_dt, next_executed_task) running_time = float((datetime.now() - start).total_seconds()) self._logger.info(INFO_RESULT, running_time) return safe_dict({ "datetime": datetime.now().isoformat(), "running-time": running_time, "event-datetime": current_dt.isoformat(), "enabled_tasks": enabled_tasks, "started-tasks": started_tasks }) except ValueError as ex: self._logger.error("{}\n{}".format(ex, safe_json(task, indent=2))) finally: self._logger.flush()