def _load_managed_entries(self): """ loads scheduler managed entries. no start-up procedures are performed """ scheduler_entries = self.se_managed_dao.get_all() for scheduler_entry_obj in scheduler_entries: process_name = scheduler_entry_obj.process_name if scheduler_entry_obj.process_name not in ProcessContext.CONTEXT: self.logger.error('Process %r is not known to the system. Skipping it.' % process_name) continue process_type = ProcessContext.get_process_type(process_name) if process_type in [TYPE_BLOCKING_DEPENDENCIES, TYPE_BLOCKING_CHILDREN, TYPE_MANAGED]: function = self.fire_managed_worker handler_type = TYPE_MANAGED elif process_type == TYPE_GARBAGE_COLLECTOR: function = self.fire_garbage_collector handler_type = TYPE_MANAGED elif process_type == TYPE_FREERUN: self.logger.error('TYPE_FREERUN process %s was found in scheduler_managed_entry table. ' 'Move the process to the scheduler_freerun_entry table. Skipping the process.' % process_type) continue else: self.logger.error('Process type %s is not known to the system. Skipping it.' % process_type) continue self._activate_handler(scheduler_entry_obj, process_name, 'NA', function, handler_type)
def entries(self): list_of_trees = [] try: sorter_keys = sorted(context.timetable_context.keys()) for tree_name in sorter_keys: tree_obj = self.mbean.timetable.trees[tree_name] tree_row = list() tree_row.append(tree_name) # index 0 tree_row.append(tree_obj.mx_page) # index 1 tree_row.append(tree_obj.mx_name) # index 2 processes = dict() # index 3 context_entry = context.timetable_context[tree_name] for process_name in context_entry.enclosed_processes: process_details = [process_name, # index x0 ProcessContext.get_time_qualifier(process_name), # index x1 self._state_machine_name(process_name), # index x2 ProcessContext.get_process_type(process_name), # index x3 ProcessContext.run_on_active_timeperiod(process_name), # index x4 context_entry.dependent_on, # index x5 self._list_of_dependant_trees(tree_obj)] # index x6 processes[process_name] = process_details tree_row.append(processes) list_of_trees.append(tree_row) except Exception as e: self.logger.error('MX Exception %s' % str(e), exc_info=True) return list_of_trees
def _get_nodes_details(self, tree): timetable = self.mbean.timetable description = dict() description['reprocessing_queues'] = dict() description['processes'] = dict() description['next_timeperiods'] = dict() try: if isinstance(tree, FourLevelTree): description['number_of_levels'] = 4 description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly) description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly) description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily) description['reprocessing_queues']['hourly'] = self._get_reprocessing_details(tree.process_hourly) description['processes']['yearly'] = tree.process_yearly description['processes']['monthly'] = tree.process_monthly description['processes']['daily'] = tree.process_daily description['processes']['hourly'] = tree.process_hourly description['next_timeperiods']['yearly'] = timetable.get_next_job_record(tree.process_yearly).timeperiod description['next_timeperiods']['monthly'] = timetable.get_next_job_record(tree.process_monthly).timeperiod description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_daily).timeperiod description['next_timeperiods']['hourly'] = timetable.get_next_job_record(tree.process_hourly).timeperiod description['type'] = ProcessContext.get_process_type(tree.process_yearly) elif isinstance(tree, ThreeLevelTree): description['number_of_levels'] = 3 description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly) description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly) description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily) description['processes']['yearly'] = tree.process_yearly description['processes']['monthly'] = tree.process_monthly description['processes']['daily'] = tree.process_daily description['next_timeperiods']['yearly'] = timetable.get_next_job_record(tree.process_yearly).timeperiod description['next_timeperiods']['monthly'] = timetable.get_next_job_record(tree.process_monthly).timeperiod description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_daily).timeperiod description['type'] = ProcessContext.get_process_type(tree.process_yearly) elif isinstance(tree, TwoLevelTree): description['number_of_levels'] = 1 description['reprocessing_queues']['linear'] = self._get_reprocessing_details(tree.process_name) description['processes']['linear'] = tree.process_name description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_name).timeperiod description['type'] = ProcessContext.get_process_type(tree.process_name) else: raise ValueError('Tree type %s has no support within MX module.' % type(tree).__name__) except Exception as e: self.logger.error('MX Exception: ' + str(e), exc_info=True) finally: return description
def fire_managed_worker(self, *args): """requests vertical aggregator (hourly site, daily variant, etc) to start up""" try: process_name = args[0] scheduler_entry_obj = args[1] self.logger.info('%s {' % process_name) timetable_record = self.timetable.get_next_job_record(process_name) pipeline = self.pipelines[scheduler_entry_obj.state_machine_name] run_on_active_timeperiod = ProcessContext.run_on_active_timeperiod(scheduler_entry_obj.process_name) if not run_on_active_timeperiod: time_qualifier = ProcessContext.get_time_qualifier(process_name) incremented_timeperiod = time_helper.increment_timeperiod(time_qualifier, timetable_record.timeperiod) dt_record_timestamp = time_helper.synergy_to_datetime(time_qualifier, incremented_timeperiod) dt_record_timestamp += timedelta(minutes=LAG_5_MINUTES) if datetime.utcnow() <= dt_record_timestamp: self.logger.info('Timetable record %s for timeperiod %s will not be triggered until %s.' % (timetable_record.document['_id'], timetable_record.timeperiod, dt_record_timestamp.strftime('%Y-%m-%d %H:%M:%S'))) return process_type = ProcessContext.get_process_type(scheduler_entry_obj.process_name) if process_type == TYPE_BLOCKING_DEPENDENCIES: pipeline.manage_pipeline_with_blocking_dependencies(process_name, timetable_record) elif process_type == TYPE_BLOCKING_CHILDREN: pipeline.manage_pipeline_with_blocking_children(process_name, timetable_record) elif process_type == TYPE_MANAGED: pipeline.manage_pipeline_for_process(process_name, timetable_record) except (AMQPError, IOError) as e: self.logger.error('AMQPError: %s' % str(e), exc_info=True) self.publishers.reset_all(suppress_logging=True) except Exception as e: self.logger.error('Exception: %s' % str(e), exc_info=True) finally: self.logger.info('}')