def _load_managed_entries(self):
""" loads scheduler managed entries. no start-up procedures are performed """
scheduler_entries = self.se_managed_dao.get_all()
for scheduler_entry_obj in scheduler_entries:
process_name = scheduler_entry_obj.process_name
if scheduler_entry_obj.process_name not in ProcessContext.CONTEXT:
self.logger.error('Process %r is not known to the system. Skipping it.' % process_name)
continue
process_type = ProcessContext.get_process_type(process_name)
if process_type in [TYPE_BLOCKING_DEPENDENCIES, TYPE_BLOCKING_CHILDREN, TYPE_MANAGED]:
function = self.fire_managed_worker
handler_type = TYPE_MANAGED
elif process_type == TYPE_GARBAGE_COLLECTOR:
function = self.fire_garbage_collector
handler_type = TYPE_MANAGED
elif process_type == TYPE_FREERUN:
self.logger.error('TYPE_FREERUN process %s was found in scheduler_managed_entry table. '
'Move the process to the scheduler_freerun_entry table. Skipping the process.'
% process_type)
continue
else:
self.logger.error('Process type %s is not known to the system. Skipping it.' % process_type)
continue
self._activate_handler(scheduler_entry_obj, process_name, 'NA', function, handler_type)
def entries(self):
list_of_trees = []
try:
sorter_keys = sorted(context.timetable_context.keys())
for tree_name in sorter_keys:
tree_obj = self.mbean.timetable.trees[tree_name]
tree_row = list()
tree_row.append(tree_name) # index 0
tree_row.append(tree_obj.mx_page) # index 1
tree_row.append(tree_obj.mx_name) # index 2
processes = dict() # index 3
context_entry = context.timetable_context[tree_name]
for process_name in context_entry.enclosed_processes:
process_details = [process_name, # index x0
ProcessContext.get_time_qualifier(process_name), # index x1
self._state_machine_name(process_name), # index x2
ProcessContext.get_process_type(process_name), # index x3
ProcessContext.run_on_active_timeperiod(process_name), # index x4
context_entry.dependent_on, # index x5
self._list_of_dependant_trees(tree_obj)] # index x6
processes[process_name] = process_details
tree_row.append(processes)
list_of_trees.append(tree_row)
except Exception as e:
self.logger.error('MX Exception %s' % str(e), exc_info=True)
return list_of_trees
def _get_nodes_details(self, tree):
timetable = self.mbean.timetable
description = dict()
description['reprocessing_queues'] = dict()
description['processes'] = dict()
description['next_timeperiods'] = dict()
try:
if isinstance(tree, FourLevelTree):
description['number_of_levels'] = 4
description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly)
description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly)
description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily)
description['reprocessing_queues']['hourly'] = self._get_reprocessing_details(tree.process_hourly)
description['processes']['yearly'] = tree.process_yearly
description['processes']['monthly'] = tree.process_monthly
description['processes']['daily'] = tree.process_daily
description['processes']['hourly'] = tree.process_hourly
description['next_timeperiods']['yearly'] = timetable.get_next_job_record(tree.process_yearly).timeperiod
description['next_timeperiods']['monthly'] = timetable.get_next_job_record(tree.process_monthly).timeperiod
description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_daily).timeperiod
description['next_timeperiods']['hourly'] = timetable.get_next_job_record(tree.process_hourly).timeperiod
description['type'] = ProcessContext.get_process_type(tree.process_yearly)
elif isinstance(tree, ThreeLevelTree):
description['number_of_levels'] = 3
description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly)
description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly)
description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily)
description['processes']['yearly'] = tree.process_yearly
description['processes']['monthly'] = tree.process_monthly
description['processes']['daily'] = tree.process_daily
description['next_timeperiods']['yearly'] = timetable.get_next_job_record(tree.process_yearly).timeperiod
description['next_timeperiods']['monthly'] = timetable.get_next_job_record(tree.process_monthly).timeperiod
description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_daily).timeperiod
description['type'] = ProcessContext.get_process_type(tree.process_yearly)
elif isinstance(tree, TwoLevelTree):
description['number_of_levels'] = 1
description['reprocessing_queues']['linear'] = self._get_reprocessing_details(tree.process_name)
description['processes']['linear'] = tree.process_name
description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_name).timeperiod
description['type'] = ProcessContext.get_process_type(tree.process_name)
else:
raise ValueError('Tree type %s has no support within MX module.' % type(tree).__name__)
except Exception as e:
self.logger.error('MX Exception: ' + str(e), exc_info=True)
finally:
return description
def fire_managed_worker(self, *args):
"""requests vertical aggregator (hourly site, daily variant, etc) to start up"""
try:
process_name = args[0]
scheduler_entry_obj = args[1]
self.logger.info('%s {' % process_name)
timetable_record = self.timetable.get_next_job_record(process_name)
pipeline = self.pipelines[scheduler_entry_obj.state_machine_name]
run_on_active_timeperiod = ProcessContext.run_on_active_timeperiod(scheduler_entry_obj.process_name)
if not run_on_active_timeperiod:
time_qualifier = ProcessContext.get_time_qualifier(process_name)
incremented_timeperiod = time_helper.increment_timeperiod(time_qualifier, timetable_record.timeperiod)
dt_record_timestamp = time_helper.synergy_to_datetime(time_qualifier, incremented_timeperiod)
dt_record_timestamp += timedelta(minutes=LAG_5_MINUTES)
if datetime.utcnow() <= dt_record_timestamp:
self.logger.info('Timetable record %s for timeperiod %s will not be triggered until %s.'
% (timetable_record.document['_id'],
timetable_record.timeperiod,
dt_record_timestamp.strftime('%Y-%m-%d %H:%M:%S')))
return
process_type = ProcessContext.get_process_type(scheduler_entry_obj.process_name)
if process_type == TYPE_BLOCKING_DEPENDENCIES:
pipeline.manage_pipeline_with_blocking_dependencies(process_name, timetable_record)
elif process_type == TYPE_BLOCKING_CHILDREN:
pipeline.manage_pipeline_with_blocking_children(process_name, timetable_record)
elif process_type == TYPE_MANAGED:
pipeline.manage_pipeline_for_process(process_name, timetable_record)
except (AMQPError, IOError) as e:
self.logger.error('AMQPError: %s' % str(e), exc_info=True)
self.publishers.reset_all(suppress_logging=True)
except Exception as e:
self.logger.error('Exception: %s' % str(e), exc_info=True)
finally:
self.logger.info('}')
