def build_schedule_from_maintenance_window(period_str): """ Builds a Instance running schedule based on an RDS preferred maintenance windows string in format ddd:hh:mm-ddd:hh:mm :param period_str: rds maintenance windows string :return: Instance running schedule with timezone UTC """ # get elements of period start_string, stop_string = period_str.split("-") start_day_string, start_hhmm_string = start_string.split(":", 1) stop_day_string, stop_hhmm_string = stop_string.split(":", 1) # weekday set builder weekdays_builder = WeekdaySetBuilder() start_weekday = weekdays_builder.build(start_day_string) start_time = SchedulerConfigBuilder.get_time_from_string(start_hhmm_string) end_time = SchedulerConfigBuilder.get_time_from_string(stop_hhmm_string) # windows with now day overlap, can do with one period for schedule if start_day_string == stop_day_string: periods = [ { "period": RunningPeriod(name=MAINTENANCE_PERIOD_NAME, begintime=start_time, endtime=end_time, weekdays=start_weekday) }] else: # window with day overlap, need two periods for schedule end_time_day1 = SchedulerConfigBuilder.get_time_from_string("23:59") begin_time_day2 = SchedulerConfigBuilder.get_time_from_string("00:00") stop_weekday = weekdays_builder.build(stop_day_string) periods = [ { "period": RunningPeriod(name=MAINTENANCE_PERIOD_NAME + "-{}".format(start_day_string), begintime=start_time, endtime=end_time_day1, weekdays=start_weekday), "instancetype": None }, { "period": RunningPeriod(name=MAINTENANCE_PERIOD_NAME + "-{}".format(stop_day_string), begintime=begin_time_day2, endtime=end_time, weekdays=stop_weekday), "instancetype": None }] # create schedule with period(s) and timezone UTC schedule = InstanceSchedule(name=MAINTENANCE_SCHEDULE_NAME, periods=periods, timezone="UTC", enforced=True) return schedule
def configuration(self): """ Returns and cached configuration :return: scheduler configuration """ if self._configuration is None: configdata = ConfigDynamodbAdapter(self._table.name).config self._configuration = SchedulerConfigBuilder(logger=self._logger).build(configdata) return self._configuration
def _execute_as_lambda(self, conf): # runs a service/account/region subset of the configuration as a new lambda function self._logger.info(INF_STARTING_LAMBDA, "-".join(conf.scheduled_services), "-".join(self.account_names(conf)), "-".join(conf.regions)) self._logger.info('#############################################') self._logger.info('boto3.__version__ : {}', boto3.__version__) self._logger.info('#############################################\n\n') # need to convert configuration to dictionary to allow it to be passed in event config = SchedulerConfigBuilder.configuration_as_dict(conf) self._logger.info('#############################################') self._logger.info('config : {}', str(config)) self._logger.info('#############################################\n\n') payload = str.encode( json.dumps({ "action": "scheduler:run", "configuration": config, "dispatch_time": str(datetime.now()) })) if len(payload) > 200000: config["schedules"] = {} config["periods"] = {} payload = str.encode( json.dumps({ "action": "scheduler:run", "configuration": config, "dispatch_time": str(datetime.now()) })) # start the lambda function resp = self.lambda_client.invoke_with_retries( FunctionName=self._context.function_name, InvocationType="Event", LogType="None", Payload=payload) if resp["StatusCode"] != 202: self._logger.error(ERR_STARTING_LAMBDA, self._context.function_name, self._context.function_version, config) result = { "services": list(conf.scheduled_services), "accounts": list(self.account_names(conf)), "regions": list(conf.regions), "lambda_invoke_result": resp["StatusCode"], "lambda_request_id": resp["ResponseMetadata"]["RequestId"] } return result
def get_scheduler_configuration(logger): """ Returns the scheduler configuration :return: scheduler configuration """ global __configuration if __configuration is None: configdata = ConfigDynamodbAdapter(os.getenv(ENV_CONFIG)).config __configuration = SchedulerConfigBuilder(logger=logger).build(configdata) if logger is not None: logger.debug("Configuration loaded\n{}", str(__configuration)) return __configuration
def configuration(self): """ Gets the configuration passed in the event :return: scheduler configuration """ if self._configuration is None: # need to reconstruct configuration from dictionary in event self._configuration = SchedulerConfigBuilder.configuration_from_dict( self._event["configuration"]) # for large configurations the schedules are not passed in the event, need to reload these here if len(self._configuration.schedules) == 0: loaded_config = configuration.get_scheduler_configuration( self._logger) self._configuration.schedules = loaded_config.schedules return self._configuration
def calculate_schedule_usage_for_period(self, schedule_name, start_dt, stop_dt=None, logger=None): result = {} def running_seconds(startdt, stopdt): return max(int((stopdt - startdt).total_seconds()), 60) def running_hours(startdt, stopdt): return int(((stopdt - startdt).total_seconds() - 1) / 3600) + 1 def make_period(started_dt, stopped_dt): running_period = ({ "begin": started_dt, "end": stopped_dt, "billing_hours": running_hours(started_dt, stopped_dt), "billing_seconds": running_seconds(started_dt, stopped_dt) }) return running_period self._logger = logger stop = stop_dt or start_dt if start_dt > stop: raise ValueError(ERR_STOP_MUST_BE_LATER_OR_EQUAL_TO_START) dt = start_dt if isinstance(start_dt, datetime) else datetime( start_dt.year, start_dt.month, start_dt.day) config_data = ConfigDynamodbAdapter(self._table.name).config while dt <= stop: self._configuration = SchedulerConfigBuilder( logger=self._logger).build(config_data) conf = configuration.SchedulerConfigBuilder(self._logger).build( config=config_data, dt=dt) schedule = conf.get_schedule(schedule_name) timeline = {dt.replace(hour=0, minute=0)} for p in schedule.periods: begintime = p["period"].begintime endtime = p["period"].endtime if begintime is None and endtime is None: timeline.add(dt.replace(hour=0, minute=0)) timeline.add(dt.replace(hour=23, minute=59)) else: if begintime: timeline.add( dt.replace(hour=begintime.hour, minute=begintime.minute)) if endtime: timeline.add( dt.replace(hour=endtime.hour, minute=endtime.minute)) running_periods = {} started = None starting_period = None current_state = None inst = as_namedtuple("Instance", { "instance_str": "instance", "allow_resize": False }) for tm in sorted(list(timeline)): desired_state, instance_type, period = schedule.get_desired_state( inst, self._logger, tm, False) if current_state != desired_state: if desired_state == InstanceSchedule.STATE_RUNNING: started = tm current_state = InstanceSchedule.STATE_RUNNING starting_period = period elif desired_state == InstanceSchedule.STATE_STOPPED: stopped = tm desired_state_with_adj_check, _, __ = schedule.get_desired_state( inst, self._logger, tm, True) if desired_state_with_adj_check == InstanceSchedule.STATE_RUNNING: stopped += timedelta(minutes=1) if current_state == InstanceSchedule.STATE_RUNNING: current_state = InstanceSchedule.STATE_STOPPED running_periods[starting_period] = (make_period( started, stopped)) if current_state == InstanceSchedule.STATE_RUNNING: stopped = dt.replace(hour=23, minute=59) + timedelta(minutes=1) running_periods[starting_period] = (make_period( started, stopped)) result[str(dt.date())] = { "running_periods": running_periods, "billing_seconds": sum([ running_periods[ps]["billing_seconds"] for ps in running_periods ]), "billing_hours": sum([ running_periods[ph]["billing_hours"] for ph in running_periods ]) } dt += timedelta(days=1) return {"schedule": schedule_name, "usage": result}
def _validate_schedule(self, **schedule): result = {} # allowed parameters valid_parameters = [ configuration.TIMEZONE, configuration.PERIODS, configuration.NAME, configuration.DESCRIPTION, configuration.OVERWRITE, configuration.METRICS, configuration.STOP_NEW_INSTANCES, configuration.USE_MAINTENANCE_WINDOW, configuration.SSM_MAINTENANCE_WINDOW, configuration.RETAINED_RUNNING, configuration.ENFORCED, configuration.HIBERNATE, configuration.OVERRIDE_STATUS, configuration.SCHEDULE_CONFIG_STACK ] for attr in schedule: if attr == ConfigAdmin.TYPE_ATTR: continue if attr not in valid_parameters: raise ValueError( ERR_SCHEDULE_UNKNOWN_PARAMETER.format( attr, valid_parameters)) # skip None values if schedule[attr] is None or len(str(schedule[attr])) == 0: continue # check periods set if attr == configuration.PERIODS: temp = self._ensure_set(schedule[attr]) if len(temp) > 0: result[attr] = temp continue if attr in [ configuration.NAME, configuration.SSM_MAINTENANCE_WINDOW ]: result[attr] = schedule[attr] continue # make sure these fields are valid booleans if attr in [ configuration.METRICS, configuration.STOP_NEW_INSTANCES, configuration.USE_MAINTENANCE_WINDOW, configuration.RETAINED_RUNNING, configuration.HIBERNATE, configuration.ENFORCED ]: bool_value = self._ensure_bool(schedule[attr]) if bool_value is None: raise ValueError( ERR_SCHEDULE_INVALID_BOOLEAN.format( schedule[attr], attr)) result[attr] = bool_value continue # overwrite status, now deprecated, use PROP_OVERRIDE_STATUS instead if attr == configuration.OVERWRITE: if configuration.OVERRIDE_STATUS in schedule: raise ValueError( ERR_SCHEDULE_OVERWRITE_OVERRIDE_EXCLUSIVE.format( configuration.OVERWRITE, configuration.OVERRIDE_STATUS)) bool_value = self._ensure_bool(schedule[attr]) if bool_value is None: raise ValueError( ERR_SCHEDULE_INVALID_BOOLEAN.format( schedule[attr], attr)) result[ configuration.OVERRIDE_STATUS] = configuration.OVERRIDE_STATUS_RUNNING if bool_value \ else configuration.OVERRIDE_STATUS_STOPPED continue if attr == configuration.OVERRIDE_STATUS: if configuration.OVERWRITE in schedule: raise ValueError( ERR_SCHEDULE_OVERWRITE_OVERRIDE_EXCLUSIVE.format( configuration.OVERWRITE, configuration.OVERRIDE_STATUS)) if schedule[attr] not in configuration.OVERRIDE_STATUS_VALUES: raise ValueError( ERR_SCHEDULE_INVALID_OVERRIDE.format( schedule[attr], attr, ",".join(configuration.OVERRIDE_STATUS_VALUES))) result[attr] = schedule[attr] continue # description if attr in [ configuration.DESCRIPTION, configuration.SCHEDULE_CONFIG_STACK ]: result[attr] = schedule[attr] continue # validate timezone if attr == configuration.TIMEZONE: timezone = schedule[configuration.TIMEZONE] if not SchedulerConfigBuilder.is_valid_timezone(timezone): raise ValueError( ERR_SCHEDULE_INVALID_TIMEZONE.format( timezone, configuration.TIMEZONE)) result[attr] = timezone # name is mandatory if configuration.NAME not in result: raise ValueError(ERR_SCHEDULE_NAME_MISSING) # if there is no overwrite there must be at least one period if configuration.OVERRIDE_STATUS not in schedule: if configuration.PERIODS not in schedule or len( schedule[configuration.PERIODS]) == 0: raise ValueError(ERR_SCHEDULE_NO_PERIOD) # validate if periods are in configuration if configuration.PERIODS in result: # get list of all configured periods periods = [p[configuration.NAME] for p in self._list_periods()] for period in result[configuration.PERIODS]: if period.split( configuration.INSTANCE_TYPE_SEP)[0] not in periods: raise ValueError( ERR_SCHEDULE_PERIOD_DOES_NOT_EXISTS.format(period)) # indicates this s a schedule result[ConfigAdmin.TYPE_ATTR] = "schedule" return result
def update_config(self, **settings): """ Updates configuration, validates new values :param settings: settings values :return: updated values """ valid_attributes = [ configuration.METRICS, configuration.CROSS_ACCOUNT_ROLES, configuration.DEFAULT_TIMEZONE, configuration.REGIONS, configuration.SCHEDULE_LAMBDA_ACCOUNT, configuration.TAGNAME, configuration.TRACE, ConfigAdmin.TYPE_ATTR, configuration.SCHEDULED_SERVICES, configuration.SCHEDULE_CLUSTERS, configuration.CREATE_RDS_SNAPSHOT, configuration.STARTED_TAGS, configuration.STOPPED_TAGS ] checked_settings = {} for attr in settings: if attr in [ConfigAdmin.TYPE_ATTR, configuration.NAME]: continue # only valid fields if attr not in valid_attributes: raise ValueError(ERR_UPDATE_UNKNOWN_PARAMETER.format(attr)) # remove None fields if settings[attr] is None: continue # remove empty strings if len(str(settings[attr])) == 0: continue # make sure these fields are set as sets if attr in [ configuration.REGIONS, configuration.CROSS_ACCOUNT_ROLES, configuration.SCHEDULED_SERVICES ]: temp = self._ensure_set(settings[attr]) if len(settings[attr]) > 0: checked_settings[attr] = temp continue # make sure these fields are valid booleans if attr in [ configuration.METRICS, configuration.TRACE, configuration.SCHEDULE_LAMBDA_ACCOUNT, configuration.CREATE_RDS_SNAPSHOT, configuration.SCHEDULE_CLUSTERS ]: bool_value = self._ensure_bool(settings[attr]) if bool_value is None: raise ValueError( ERR_UPDATE_INVALID_BOOL_PARAM.format( settings[attr], attr)) checked_settings[attr] = bool_value continue # validate timezone if attr == configuration.DEFAULT_TIMEZONE: default_tz = settings[configuration.DEFAULT_TIMEZONE] if not SchedulerConfigBuilder.is_valid_timezone(default_tz): raise ValueError( ERR_UPDATE_INVALID_TZ_PARAMETER.format( default_tz, configuration.DEFAULT_TIMEZONE)) checked_settings[attr] = default_tz continue checked_settings[attr] = settings[attr] if configuration.TAGNAME not in settings: raise ValueError(ERR_UPDATE_TAGNAME_EMPTY) for service in settings.get(configuration.SCHEDULED_SERVICES, []): if service not in ConfigAdmin.SUPPORTED_SERVICES: raise ValueError( ERR_UPDATE_UNKNOWN_SERVICE.format(service)) # keys for config item checked_settings[ConfigAdmin.TYPE_ATTR] = "config" checked_settings[configuration.NAME] = "scheduler" self._table.put_item_with_retries(Item=checked_settings) return ConfigAdmin._for_output(checked_settings)