def _get_next_run_time(self, scheduler_task_name, scheduler_task, current_time): interval = parse_timetable(scheduler_task['schedule']) if not interval: return timestamp_to_datetime(0) scheduled_task_history = self.scheduler_tasks_history[scheduler_task_name] next_run = scheduled_task_history.get('last_run', 0) + interval return timestamp_to_datetime(next_run if next_run > current_time else current_time)
def _get_next_run_time(self, scheduler_task_name, scheduler_task, current_time): interval = parse_timetable(scheduler_task['schedule']) if not interval: return timestamp_to_datetime(0) scheduled_task_history = self.scheduler_tasks_history[ scheduler_task_name] next_run = scheduled_task_history.get('last_run', 0) + interval return timestamp_to_datetime( next_run if next_run > current_time else current_time)
def get_scheduled_actions(self): """ get_scheduled_actions -- fetch list of scheduled actions :returns: (dict) id -> parsed action with schedule """ actions = self.get_config('action') actions_dict = {action.get('_id'): action for action in actions} # Filter list of regular tasks scheduled = filter(lambda action: parse_timetable(action.get('schedule', '')), actions) scheduled_dict = {} for action in scheduled: try: parsed = self._parse_action(actions_dict, action) scheduled_dict[action.get('_id')] = parsed except ActionResursion as ex: self.log.error(ex.value) return scheduled_dict
def get_scheduled_actions(self): """ get_scheduled_actions -- fetch list of scheduled actions :returns: (dict) id -> parsed action with schedule """ actions = self.get_config('action') actions_dict = {action.get('_id'): action for action in actions} # Filter list of regular tasks scheduled = filter( lambda action: parse_timetable(action.get('schedule', '')), actions) scheduled_dict = {} for action in scheduled: try: parsed = self._parse_action(actions_dict, action) scheduled_dict[action.get('_id')] = parsed except ActionResursion as ex: self.log.error(ex.value) return scheduled_dict
def store_metric_value(self, metric_id, object_id, task, values): log.debug( 'store_metric_value {} for action/connection {} by task {}'.format( metric_id, object_id, task['id'])) exit_codes = values.get('exit_codes') stdout = values.get('stdout') metric = self.metrics.get(metric_id) value = self.parse_value(metric, stdout) log.debug('Metric (id={}) parsed value: {}'.format(metric_id, value)) if value is None: logging.error( "No parser match for metric {}, nothing to store".format( metric_id)) self.db_log.error("Пустое значение после фильтрации", stdout, "metric", metric_id) return converter = lambda x: x # Convert metric type if metric['type'] == 'boolean': value = self.cast_to_boolean(metric_id, metric, value) else: converter = SETTINGS.METRICS_TYPES_MAP[metric['type']] try: value = converter(value) except ValueError: log.error( "Wrong value for metric '{}', cannot convert to {}".format( metric_id, metric['type']), exc_info=True) self.db_log.error( "Не удалось привести тип значения к {}".format( metric['type']), str(value), "metric", metric_id) return # Trim strings if isinstance(value, str): value = value[:SETTINGS.METRIC_STRING_LIMIT] # Apply multiplier multiplier = metric.get('multiplier', None) try: if multiplier and metric['type'] in SETTINGS.METRIC_NUMERICAL_TYPES: multiplier = float(multiplier) value = value * multiplier # If it is int, convert to int value = converter(value) except: log.error('Cannot apply multiplier', exc_info=True) self.db_log.error("Не удалось применить множитель", str(value), "metric", metric_id) return timestamp = datetime_to_timestamp(task['run_at']) skip_interval = parse_timetable(metric.get('limit_duplicate_save', '')) if skip_interval: prev_val, prev_timestamp = self._lcache.get(metric_id, (None, 0)) if (prev_val == value) and (timestamp - prev_timestamp) < skip_interval: return True else: self._lcache[metric_id] = (value, datetime_to_timestamp( task['run_at'])) log.info('Store value="{}" for metric {}'.format(value, metric_id)) try: self.metrics_storage.store_metric(metric_id, value, time=task['run_at']) yield from self.connection.hset( SETTINGS.LAST_VALUES_HASH, metric_id.encode('utf-8'), ujson.dumps({ 'value': value, 'timestamp': timestamp }).encode('utf-8')) except: log.error('Cannot store metric value, storage exception', exc_info=True) return # Publish message about finish yield from self.connection.publish( SETTINGS.METRICS_CHANNEL.format(metric_id).encode('utf-8'), b'') return True
def store_metric_value(self, metric_id, object_id, task, values): log.debug('store_metric_value {} for action/connection {} by task {}'.format(metric_id, object_id, task['id'])) exit_codes = values.get('exit_codes') stdout = values.get('stdout') metric = self.metrics.get(metric_id) value = self.parse_value(metric, stdout) log.debug('Metric (id={}) parsed value: {}'.format(metric_id, value)) if value is None: logging.error("No parser match for metric {}, nothing to store".format(metric_id)) self.db_log.error("Пустое значение после фильтрации", stdout, "metric", metric_id) return converter = lambda x: x # Convert metric type if metric['type'] == 'boolean': value = self.cast_to_boolean(metric_id, metric, value) else: converter = SETTINGS.METRICS_TYPES_MAP[metric['type']] try: value = converter(value) except ValueError: log.error("Wrong value for metric '{}', cannot convert to {}".format(metric_id, metric['type']), exc_info=True) self.db_log.error("Не удалось привести тип значения к {}".format(metric['type']), str(value), "metric", metric_id) return # Trim strings if isinstance(value, str): value = value[:SETTINGS.METRIC_STRING_LIMIT] # Apply multiplier multiplier = metric.get('multiplier', None) try: if multiplier and metric['type'] in SETTINGS.METRIC_NUMERICAL_TYPES: multiplier = float(multiplier) value = value * multiplier # If it is int, convert to int value = converter(value) except: log.error('Cannot apply multiplier', exc_info=True) self.db_log.error("Не удалось применить множитель", str(value), "metric", metric_id) return timestamp = datetime_to_timestamp(task['run_at']) skip_interval = parse_timetable(metric.get('limit_duplicate_save', '')) if skip_interval: prev_val, prev_timestamp = self._lcache.get(metric_id, (None, 0)) if (prev_val == value) and (timestamp - prev_timestamp) < skip_interval: return True else: self._lcache[metric_id] = (value, datetime_to_timestamp(task['run_at'])) log.info('Store value="{}" for metric {}'.format(value, metric_id)) try: self.metrics_storage.store_metric(metric_id, value, time=task['run_at']) yield from self.connection.hset(SETTINGS.LAST_VALUES_HASH, metric_id.encode('utf-8'), ujson.dumps({'value': value, 'timestamp': timestamp}).encode('utf-8')) except: log.error('Cannot store metric value, storage exception', exc_info=True) return # Publish message about finish yield from self.connection.publish(SETTINGS.METRICS_CHANNEL.format(metric_id).encode('utf-8'), b'') return True