def stop_process(options): """Stop the synergy-data daemons""" import logging from supervisor import supervisor_helper as helper from model.box_configuration_entry import BoxConfigurationEntry if options.app is not None and options.app != process_context.PROCESS_SUPERVISOR: # mark individual process for termination # real work is performed by Supervisor if options.app not in PROCESSES_FOR_XXL: sys.stdout.write("ERROR: requested process must be withing allowed list of: %r \n" % PROCESSES_FOR_XXL) sys.exit(1) box_id = helper.get_box_id(logging) box_configuration = helper.retrieve_configuration(logging, box_id) box_configuration.set_process_state(options.app, BoxConfigurationEntry.STATE_OFF) helper.update_configuration(logging, box_configuration) else: # stop Supervisor try: pid = _get_supervisor_pid() if pid is None: message = "ERROR: Can not find Supervisor pidfile. Supervisor not running?\n" sys.stderr.write(message) sys.exit(1) # Try killing the daemon process sys.stdout.write("INFO: Killing %r \n" % process_context.PROCESS_SUPERVISOR) while 1: os.kill(pid, signal.SIGTERM) time.sleep(0.1) ProcessContext.remove_pid_file(process_context.PROCESS_SUPERVISOR) except Exception as e: sys.stderr.write("Exception on killing %s : %s \n" % (process_context.PROCESS_SUPERVISOR, str(e))) sys.exit(0)
def create_unit_of_work(process_name, start_id, end_id, timeperiod='INVALID_TIMEPERIOD', state=unit_of_work.STATE_REQUESTED, creation_at=datetime.utcnow(), uow_id=None): """ method creates and returns unit_of_work """ try: source_collection = ProcessContext.get_source(process_name) target_collection = ProcessContext.get_sink(process_name) except KeyError: source_collection = None target_collection = None uow = UnitOfWork() uow.timeperiod = timeperiod uow.start_timeperiod = timeperiod uow.end_timeperiod = timeperiod uow.start_id = start_id uow.end_id = end_id uow.source = source_collection uow.sink = target_collection uow.state = state uow.created_at = creation_at uow.process_name = process_name uow.number_of_retries = 0 if uow_id is not None: uow.document['_id'] = uow_id return uow
def compute_scope_of_processing(self, process_name, start_timeperiod, end_timeperiod, job_record): """method reads collection and identify slice for processing""" source_collection_name = ProcessContext.get_source(process_name) target_collection_name = ProcessContext.get_sink(process_name) start_id = self.ds.highest_primary_key(source_collection_name, start_timeperiod, end_timeperiod) end_id = self.ds.lowest_primary_key(source_collection_name, start_timeperiod, end_timeperiod) uow = UnitOfWork() uow.timeperiod = start_timeperiod uow.start_id = str(start_id) uow.end_id = str(end_id) uow.start_timeperiod = start_timeperiod uow.end_timeperiod = end_timeperiod uow.created_at = datetime.utcnow() uow.source = source_collection_name uow.sink = target_collection_name uow.state = unit_of_work.STATE_REQUESTED uow.process_name = process_name uow.number_of_retries = 0 uow_id = self.uow_dao.insert(uow) mq_request = WorkerMqRequest() mq_request.process_name = process_name mq_request.unit_of_work_id = uow_id publisher = self.publishers.get(process_name) publisher.publish(mq_request.document) publisher.release() msg = 'Published: UOW %r for %r in timeperiod %r.' % (uow_id, process_name, start_timeperiod) self._log_message(INFO, process_name, job_record, msg) return uow
def entries(self): list_of_trees = [] try: sorter_keys = sorted(context.timetable_context.keys()) for tree_name in sorter_keys: tree_obj = self.mbean.timetable.trees[tree_name] tree_row = list() tree_row.append(tree_name) # index 0 tree_row.append(tree_obj.mx_page) # index 1 tree_row.append(tree_obj.mx_name) # index 2 processes = dict() # index 3 context_entry = context.timetable_context[tree_name] for process_name in context_entry.enclosed_processes: process_details = [process_name, # index x0 ProcessContext.get_time_qualifier(process_name), # index x1 self._state_machine_name(process_name), # index x2 ProcessContext.get_process_type(process_name), # index x3 ProcessContext.run_on_active_timeperiod(process_name), # index x4 context_entry.dependent_on, # index x5 self._list_of_dependant_trees(tree_obj)] # index x6 processes[process_name] = process_details tree_row.append(processes) list_of_trees.append(tree_row) except Exception as e: self.logger.error('MX Exception %s' % str(e), exc_info=True) return list_of_trees
def test_double_initialization(self): try: ProcessContext.set_current_process(PROCESS_GC) ProcessContext.set_current_process(PROCESS_SUPERVISOR) self.assertTrue(False, 'set_current_process should not allow double initialization') except AttributeError: self.assertTrue(True)
def __init__(self, process_name): """ renames process to SynergyYYY and creates PID file """ self.process_name = process_name self.logger = ProcessContext.get_logger(process_name) # process-related activities setproctitle.setproctitle(settings['process_prefix'] + self.process_name) ProcessContext.create_pid_file(self.process_name)
def run(process_name): """ You should override this method when you subclass Daemon. It will be called after the process has been daemonized by start() or restart(). """ sys.stdout.write('INFO: Starting %s \n' % ProcessContext.get_classname(process_name)) klass = get_class(ProcessContext.get_classname(process_name)) instance = klass(process_name) instance.start()
def __init__(self, process_name, process_id=None): """ renames process to SynergyYYY and creates PID file """ self.process_name = process_name self.process_id = process_id self.logger = ProcessContext.get_logger(process_name, process_id=self.process_id) # process-related activities process_title = settings['process_prefix'] + self.process_name if self.process_id: process_title += str(self.process_id) setproctitle.setproctitle(process_title) ProcessContext.create_pid_file(self.process_name, process_id=self.process_id)
def __init__(self, name): self.name = name if name in ProcessContext.CONTEXT: self.routing_key = ProcessContext.get_routing(name) self.exchange = ProcessContext.get_exchange(name) self.queue = ProcessContext.get_queue(name) elif name in MqQueueContext.CONTEXT: self.routing_key = MqQueueContext.get_routing(name) self.exchange = MqQueueContext.get_exchange(name) self.queue = name else: raise ValueError( 'Unknown name %s. Unable to retrieve amqp settings.' % name)
def kill_process(process_name): """ method is called to kill a running process """ try: sys.stdout.write('killing: %s { \n' % process_name) pid = get_process_pid(process_name) if pid is not None and psutil.pid_exists(int(pid)): p = psutil.Process(pid) p.kill() p.wait() ProcessContext.remove_pid_file(process_name) except Exception as e: sys.stderr.write('Exception on killing %s : %s \n' % (process_name, str(e))) finally: sys.stdout.write('}')
def kill_process(process_name): """ method is called to kill a running process """ try: sys.stdout.write('killing: {0} {{ \n'.format(process_name)) pid = get_process_pid(process_name) if pid is not None and psutil.pid_exists(int(pid)): p = psutil.Process(pid) p.kill() p.wait() ProcessContext.remove_pid_file(process_name) except Exception as e: sys.stderr.write('Exception on killing {0} : {1} \n'.format( process_name, e)) finally: sys.stdout.write('}')
def query_configuration(options): """ Queries process state """ from system import process_helper if not options.supervisor: # reads status of one process only process_helper.poll_process(options.app) else: # reads current box configuration and prints it to the console from db.dao.box_configuration_dao import BoxConfigurationDao from supervisor import supervisor_helper as helper from system.process_context import ProcessContext from constants import PROCESS_LAUNCH_PY logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY) box_id = helper.get_box_id(logger) bc_dao = BoxConfigurationDao(logger) sys.stdout.write('\nConfiguration for BOX_ID=%r:\n' % box_id) box_configuration = bc_dao.get_one(box_id) process_list = box_configuration.get_process_list() i = 1 for process in process_list: sys.stdout.write('%d\t%r:%r \n' % (i, process, process_list[process])) i += 1 sys.stdout.write('\n')
def _start_process(self, start_timeperiod, end_timeperiod, arguments): try: start_dt = time_helper.synergy_to_datetime(QUALIFIER_HOURLY, start_timeperiod) sqoop_slice_starttime = start_dt.strftime(SqoopDriver.SQOOP_DATE_FORMAT) end_dt = time_helper.synergy_to_datetime(QUALIFIER_HOURLY, end_timeperiod) sqoop_slice_endtime = end_dt.strftime(SqoopDriver.SQOOP_DATE_FORMAT) sink_path = ProcessContext.get_sink(self.process_name) self.logger.info('start: %s {' % self.process_name) p = psutil.Popen([settings['bash_shell'], settings['sqoop_command'], str(sqoop_slice_starttime), str(sqoop_slice_endtime), sink_path + '/' + start_timeperiod], close_fds=True, cwd=settings['process_cwd'], stdin=PIPE, stdout=PIPE, stderr=PIPE) self.cli_process = p self.logger.info('Started %s with pid = %r' % (self.process_name, p.pid)) except Exception: self.logger.error('Exception on starting: %s' % self.process_name, exc_info=True) finally: self.logger.info('}')
def get_reprocessing_candidates(self, since=None): """ method queries Unit Of Work whose <start_timeperiod> is younger than <since> and who could be candidates for re-processing """ collection = self.ds.connection(COLLECTION_UNIT_OF_WORK) query = {unit_of_work.STATE: {'$in': [unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED]}} if since is None: cursor = collection.find(query).sort('_id', ASCENDING) candidates = [UnitOfWork(document) for document in cursor] else: candidates = [] yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, since) query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod} cursor = collection.find(query).sort('_id', ASCENDING) for document in cursor: uow = UnitOfWork(document) if uow.process_name not in ProcessContext.CONTEXT: # this is a decommissioned process continue time_qualifier = ProcessContext.get_time_qualifier(uow.process_name) if time_qualifier == QUALIFIER_REAL_TIME: time_qualifier = QUALIFIER_HOURLY process_specific_since = time_helper.cast_to_time_qualifier(time_qualifier, since) if process_specific_since <= uow.start_timeperiod: candidates.append(uow) if len(candidates) == 0: raise LookupError('MongoDB has no reprocessing candidates units of work') return candidates
def cast_to_time_qualifier(process_name, timestamp): """ method is used to cast synergy_time accordingly to process time qualifier. For example: for QUALIFIER_HOURLY, it can be either 20100101_19 or 20100101_193412 """ date_format = None qualifier = ProcessContext.get_time_qualifier(process_name) if qualifier == ProcessContext.QUALIFIER_HOURLY: if len(timestamp) > 10: t = datetime.strptime(timestamp, SYNERGY_SESSION_PATTERN) return t.strftime(SYNERGY_DATE_PATTERN) else: return timestamp elif qualifier == ProcessContext.QUALIFIER_DAILY : date_format = '%Y%m%d00' elif qualifier == ProcessContext.QUALIFIER_MONTHLY: date_format = '%Y%m0000' elif qualifier == ProcessContext.QUALIFIER_YEARLY: date_format = '%Y000000' pattern = define_pattern(timestamp) t = datetime.strptime(timestamp, pattern) if date_format is not None: return t.strftime(date_format) else: raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
def stop_process(options): """Stop specific daemon""" from system import process_helper from supervisor import supervisor_helper as helper from system.process_context import ProcessContext from supervisor.supervisor_constants import PROCESS_SUPERVISOR from constants import PROCESS_LAUNCH_PY logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY) box_id = helper.get_box_id(logger) if options.supervisor is True and options.app != PROCESS_SUPERVISOR: from db.model import box_configuration from db.dao.box_configuration_dao import BoxConfigurationDao message = 'INFO: Marking %r to be managed by Supervisor \n' % options.app sys.stdout.write(message) bc_dao = BoxConfigurationDao(logger) box_config = bc_dao.get_one(box_id) box_config.set_process_state(options.app, box_configuration.STATE_OFF) bc_dao.update(box_config) return try: pid = process_helper.get_process_pid(options.app) if pid is None or process_helper.poll_process(options.app) is False: message = 'ERROR: Process %r is already terminated %r\n' % ( options.app, pid) sys.stderr.write(message) sys.exit(1) process_helper.kill_process(options.app) except Exception as e: sys.stderr.write('Exception on killing %s : %s \n' % (options.app, str(e))) traceback.print_exc(file=sys.stderr)
def _load_managed_entries(self): """ loads scheduler managed entries. no start-up procedures are performed """ scheduler_entries = self.se_managed_dao.get_all() for scheduler_entry_obj in scheduler_entries: process_name = scheduler_entry_obj.process_name if scheduler_entry_obj.process_name not in ProcessContext.CONTEXT: self.logger.error('Process %r is not known to the system. Skipping it.' % process_name) continue process_type = ProcessContext.get_process_type(process_name) if process_type in [TYPE_BLOCKING_DEPENDENCIES, TYPE_BLOCKING_CHILDREN, TYPE_MANAGED]: function = self.fire_managed_worker handler_type = TYPE_MANAGED elif process_type == TYPE_GARBAGE_COLLECTOR: function = self.fire_garbage_collector handler_type = TYPE_MANAGED elif process_type == TYPE_FREERUN: self.logger.error('TYPE_FREERUN process %s was found in scheduler_managed_entry table. ' 'Move the process to the scheduler_freerun_entry table. Skipping the process.' % process_type) continue else: self.logger.error('Process type %s is not known to the system. Skipping it.' % process_type) continue self._activate_handler(scheduler_entry_obj, process_name, 'NA', function, handler_type)
def __init__(self, process_name): """@param process_name: id of the process, the worker will be performing """ super(AbstractMqWorker, self).__init__(process_name) self.queue_source = ProcessContext.get_source(self.process_name) self.queue_sink = ProcessContext.get_sink(self.process_name) self.consumer = None self._init_mq_consumer() self.main_thread = None self.performance_ticker = None self._init_performance_ticker(self.logger) msg_suffix = 'in Production Mode' if settings['under_test']: msg_suffix = 'in Testing Mode' self.logger.info('Started %s %s' % (self.process_name, msg_suffix))
def _kill_process(self, box_config, process_name): """ method is called to kill a running process """ try: self.logger.info('kill: %s {' % process_name) pid = box_config.get_process_pid(process_name) if pid is not None and psutil.pid_exists(int(pid)): p = psutil.Process(pid) p.kill() p.wait() box_config.set_process_pid(process_name, None) self.bc_dao.update(box_config) ProcessContext.remove_pid_file(process_name) except Exception: self.logger.error('Exception on killing: %s' % process_name, exc_info=True) finally: self.logger.info('}')
def details(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if self.timeperiod is None and tree is not None: # return list of yearly nodes OR leafs for linear tree resp['children'] = dict() # limit number of children to return, since linear tree can holds thousands of nodes sorted_keys = sorted(tree.root.children.keys(), reverse=True) sorted_keys = sorted_keys[:settings['mx_children_limit']] for key in sorted_keys: child = tree.root.children[key] resp['children'][key] = TreeNodeDetails.get_details(self.logger, child) elif tree is not None: time_qualifier = ProcessContext.get_time_qualifier(self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) resp['node'] = TreeNodeDetails.get_details(self.logger, node) resp['children'] = dict() for key in node.children: child = node.children[key] resp['children'][key] = TreeNodeDetails.get_details(self.logger, child) return resp
def _process_state_in_progress(self, process_name, job_record, start_timeperiod): """ method that takes care of processing job records in STATE_IN_PROGRESS state""" time_qualifier = ProcessContext.get_time_qualifier(process_name) end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod) actual_timeperiod = time_helper.actual_timeperiod(time_qualifier) can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record) uow = self.uow_dao.get_one(job_record.related_unit_of_work) if start_timeperiod == actual_timeperiod or can_finalize_job_record is False: if uow.state in [unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED]: # current uow has not been processed yet. update it self.update_scope_of_processing(process_name, uow, start_timeperiod, end_timeperiod, job_record) else: # cls.STATE_IN_PROGRESS, cls.STATE_PROCESSED, cls.STATE_CANCELED # create new uow to cover new inserts self._compute_and_transfer_to_progress(process_name, start_timeperiod, end_timeperiod, job_record) elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True: # create new uow for FINAL RUN self._compute_and_transfer_to_final_run(process_name, start_timeperiod, end_timeperiod, job_record) else: msg = 'job record %s has timeperiod from future %s vs current time %s' \ % (job_record.document['_id'], start_timeperiod, actual_timeperiod) self._log_message(ERROR, process_name, job_record, msg)
def stop_process(options): """Stop specific daemon""" from system import process_helper from supervisor import supervisor_helper as helper from system.process_context import ProcessContext from supervisor.supervisor_constants import PROCESS_SUPERVISOR from constants import PROCESS_LAUNCH_PY logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY) box_id = helper.get_box_id(logger) if options.supervisor is True and options.app != PROCESS_SUPERVISOR: from db.model import box_configuration from db.dao.box_configuration_dao import BoxConfigurationDao message = 'INFO: Marking %r to be managed by Supervisor \n' % options.app sys.stdout.write(message) bc_dao = BoxConfigurationDao(logger) box_config = bc_dao.get_one(box_id) box_config.set_process_state(options.app, box_configuration.STATE_OFF) bc_dao.update(box_config) return try: pid = process_helper.get_process_pid(options.app) if pid is None or process_helper.poll_process(options.app) is False: message = 'ERROR: Process %r is already terminated %r\n' % (options.app, pid) sys.stderr.write(message) sys.exit(1) process_helper.kill_process(options.app) except Exception as e: sys.stderr.write('Exception on killing %s : %s \n' % (options.app, str(e))) traceback.print_exc(file=sys.stderr)
def increment_time(process_name, timestamp): """ method is used by Scheduler to define <<next>> time period. For hourly, it is next hour: 20100101_19 -> 20100101_20 For month - next month: 201001 -> 201002, etc""" qualifier = ProcessContext.get_time_qualifier(process_name) pattern = define_pattern(timestamp) t = datetime.strptime(timestamp, pattern) if qualifier == ProcessContext.QUALIFIER_HOURLY: t = t + timedelta(hours=1) return t.strftime('%Y%m%d%H') elif qualifier == ProcessContext.QUALIFIER_DAILY: t = t + timedelta(days=1) return t.strftime('%Y%m%d00') elif qualifier == ProcessContext.QUALIFIER_MONTHLY: if t.month + 1 > 12: new_month = 1 new_year = t.year + 1 t = t.replace(year = new_year, month = new_month) else: t = t.replace(month = t.month + 1) return t.strftime('%Y%m0000') elif qualifier == ProcessContext.QUALIFIER_YEARLY: t = t.replace(year = t.year + 1) return t.strftime('%Y000000') else: raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
def _process_state_in_progress(self, process_name, job_record, start_timeperiod): """ method that takes care of processing job records in STATE_IN_PROGRESS state""" time_qualifier = ProcessContext.get_time_qualifier(process_name) end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod) actual_timeperiod = time_helper.actual_timeperiod(time_qualifier) can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record) uow = self.uow_dao.get_one(job_record.related_unit_of_work) iteration = int(uow.end_id) try: if start_timeperiod == actual_timeperiod or can_finalize_job_record is False: if uow.state in [unit_of_work.STATE_REQUESTED, unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID]: # Large Job processing takes more than 1 tick of Scheduler # Let the Large Job processing complete - do no updates to Scheduler records pass elif uow.state in [unit_of_work.STATE_PROCESSED, unit_of_work.STATE_CANCELED]: # create new uow to cover new inserts uow = self.insert_uow(process_name, start_timeperiod, end_timeperiod, iteration + 1, job_record) self.timetable.update_job_record(process_name, job_record, uow, job.STATE_IN_PROGRESS) elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True: if uow.state in [unit_of_work.STATE_REQUESTED, unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID]: # Job processing has not started yet # Let the processing complete - do no updates to Scheduler records msg = 'Suppressed creating uow for %s in timeperiod %s; job record is in %s; uow is in %s' \ % (process_name, job_record.timeperiod, job_record.state, uow.state) elif uow.state == unit_of_work.STATE_PROCESSED: self.timetable.update_job_record(process_name, job_record, uow, job.STATE_PROCESSED) timetable_tree = self.timetable.get_tree(process_name) timetable_tree.build_tree() msg = 'Transferred job record %s in timeperiod %s to STATE_PROCESSED for %s' \ % (job_record.document['_id'], job_record.timeperiod, process_name) elif uow.state == unit_of_work.STATE_CANCELED: self.timetable.update_job_record(process_name, job_record, uow, job.STATE_SKIPPED) msg = 'Transferred job record %s in timeperiod %s to STATE_SKIPPED for %s' \ % (job_record.document['_id'], job_record.timeperiod, process_name) else: msg = 'Unknown state %s for job record %s in timeperiod %s for %s' \ % (uow.state, job_record.document['_id'], job_record.timeperiod, process_name) self._log_message(INFO, process_name, job_record, msg) else: msg = 'Job record %s has timeperiod from future %s vs current time %s' \ % (job_record.document['_id'], start_timeperiod, actual_timeperiod) self._log_message(ERROR, process_name, job_record, msg) except DuplicateKeyError as e: uow = self.recover_from_duplicatekeyerror(e) if uow is not None: self.timetable.update_job_record(process_name, job_record, uow, job_record.state) else: msg = 'MANUAL INTERVENTION REQUIRED! Unable to identify unit_of_work for %s in %s' \ % (process_name, job_record.timeperiod) self._log_message(ERROR, process_name, job_record, msg)
def _get_nodes_details(self, tree): timetable = self.mbean.timetable description = dict() description['reprocessing_queues'] = dict() description['processes'] = dict() description['next_timeperiods'] = dict() try: if isinstance(tree, FourLevelTree): description['number_of_levels'] = 4 description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly) description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly) description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily) description['reprocessing_queues']['hourly'] = self._get_reprocessing_details(tree.process_hourly) description['processes']['yearly'] = tree.process_yearly description['processes']['monthly'] = tree.process_monthly description['processes']['daily'] = tree.process_daily description['processes']['hourly'] = tree.process_hourly description['next_timeperiods']['yearly'] = timetable.get_next_job_record(tree.process_yearly).timeperiod description['next_timeperiods']['monthly'] = timetable.get_next_job_record(tree.process_monthly).timeperiod description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_daily).timeperiod description['next_timeperiods']['hourly'] = timetable.get_next_job_record(tree.process_hourly).timeperiod description['type'] = ProcessContext.get_process_type(tree.process_yearly) elif isinstance(tree, ThreeLevelTree): description['number_of_levels'] = 3 description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly) description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly) description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily) description['processes']['yearly'] = tree.process_yearly description['processes']['monthly'] = tree.process_monthly description['processes']['daily'] = tree.process_daily description['next_timeperiods']['yearly'] = timetable.get_next_job_record(tree.process_yearly).timeperiod description['next_timeperiods']['monthly'] = timetable.get_next_job_record(tree.process_monthly).timeperiod description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_daily).timeperiod description['type'] = ProcessContext.get_process_type(tree.process_yearly) elif isinstance(tree, TwoLevelTree): description['number_of_levels'] = 1 description['reprocessing_queues']['linear'] = self._get_reprocessing_details(tree.process_name) description['processes']['linear'] = tree.process_name description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_name).timeperiod description['type'] = ProcessContext.get_process_type(tree.process_name) else: raise ValueError('Tree type %s has no support within MX module.' % type(tree).__name__) except Exception as e: self.logger.error('MX Exception: ' + str(e), exc_info=True) finally: return description
def _get_nodes_details(self, tree): timetable = self.mbean.timetable description = dict() description['reprocessing_queues'] = dict() description['processes'] = dict() description['next_timeperiods'] = dict() try: # workaround for importing "scheduler" package if type(tree).__name__ == 'FourLevelTree': description['number_of_levels'] = 4 description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly) description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly) description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily) description['reprocessing_queues']['hourly'] = self._get_reprocessing_details(tree.process_hourly) description['processes']['yearly'] = tree.process_yearly description['processes']['monthly'] = tree.process_monthly description['processes']['daily'] = tree.process_daily description['processes']['hourly'] = tree.process_hourly description['next_timeperiods']['yearly'] = timetable.get_next_timetable_record(tree.process_yearly).get_timestamp() description['next_timeperiods']['monthly'] = timetable.get_next_timetable_record(tree.process_monthly).get_timestamp() description['next_timeperiods']['daily'] = timetable.get_next_timetable_record(tree.process_daily).get_timestamp() description['next_timeperiods']['hourly'] = timetable.get_next_timetable_record(tree.process_hourly).get_timestamp() description['type'] = ProcessContext.get_type(tree.process_yearly) elif type(tree).__name__ == 'ThreeLevelTree': description['number_of_levels'] = 3 description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly) description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly) description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily) description['processes']['yearly'] = tree.process_yearly description['processes']['monthly'] = tree.process_monthly description['processes']['daily'] = tree.process_daily description['next_timeperiods']['yearly'] = timetable.get_next_timetable_record(tree.process_yearly).get_timestamp() description['next_timeperiods']['monthly'] = timetable.get_next_timetable_record(tree.process_monthly).get_timestamp() description['next_timeperiods']['daily'] = timetable.get_next_timetable_record(tree.process_daily).get_timestamp() description['type'] = ProcessContext.get_type(tree.process_yearly) elif type(tree).__name__ == 'TwoLevelTree': description['number_of_levels'] = 1 description['reprocessing_queues']['linear'] = self._get_reprocessing_details(tree.process_name) description['processes']['linear'] = tree.process_name description['next_timeperiods']['daily'] = timetable.get_next_timetable_record(tree.process_name).get_timestamp() description['type'] = ProcessContext.get_type(tree.process_name) except Exception as e: self.logger.error('MX Exception: ' + str(e), exc_info=True) finally: return description
def get_process_pid(process_name): """ check for process' pid file and returns pid from there """ try: pid_filename = ProcessContext.get_pid_filename(process_name) with open(pid_filename, mode='r') as pid_file: pid = int(pid_file.read().strip()) except IOError: pid = None return pid
def create_and_insert_unit_of_work(process_name, start_id, end_id, state=unit_of_work.STATE_REQUESTED, timeperiod='INVALID_TIMEPERIOD'): """ method creates and inserts a unit_of_work into DB :return id of the created object in the db""" uow = create_unit_of_work(process_name, start_id, end_id, timeperiod, state) logger = ProcessContext.get_logger(process_name) uow_dao = UnitOfWorkDao(logger) uow_id = uow_dao.insert(uow) return uow_id
def create_unit_of_work(process_name, first_object_id, last_object_id): """ method is used to insert unit_of_work """ source_collection = ProcessContext.get_source_collection(process_name) target_collection = ProcessContext.get_target_collection(process_name) logger = ProcessContext.get_logger(process_name) unit_of_work = UnitOfWorkEntry() unit_of_work.set_timestamp('UNIT_TEST') unit_of_work.set_start_id(first_object_id) unit_of_work.set_end_id(last_object_id) unit_of_work.set_source_collection(source_collection) unit_of_work.set_target_collection(target_collection) unit_of_work.set_state(UnitOfWorkEntry.STATE_REQUESTED) unit_of_work.set_process_name(process_name) unit_of_work.set_number_of_retries(0) uow_id = unit_of_work_helper.insert(logger, unit_of_work) return uow_id
def _get_supervisor_pid(): """ check for supervisor's pid file and returns pid from there """ try: pid_filename = ProcessContext.get_pid_filename(process_context.PROCESS_SUPERVISOR) pf = file(pid_filename, "r") pid = int(pf.read().strip()) pf.close() except IOError: pid = None return pid
def clean_session_entries(): logger = ProcessContext.get_logger(PROCESS_UNIT_TEST) ds = ds_manager.ds_factory(logger) connection = ds.connection(COLLECTION_SINGLE_SESSION) for i in range(base_fixtures.TOTAL_ENTRIES): key = generate_session_composite_key(i, base_fixtures.TOTAL_ENTRIES) connection.remove({ raw_data.KEY: key[0], raw_data.TIMEPERIOD: key[1], raw_data.FAMILY_USER_PROFILE + '.' + raw_data.SESSION_ID: 'session_id_%s' % str(i)})
def get_process_pid(process_name): """ check for process' pid file and returns pid from there """ try: pid_filename = ProcessContext.get_pid_filename(process_name) pf = file(pid_filename, 'r') pid = int(pf.read().strip()) pf.close() except IOError: pid = None return pid
def compute_scope_of_processing(self, process_name, start_time, end_time, time_record): """method reads collection and identify slice for processing""" source_collection_name = ProcessContext.get_source_collection(process_name) target_collection_name = ProcessContext.get_target_collection(process_name) source_collection = CollectionContext.get_collection(self.logger, source_collection_name) query = { AbstractModel.TIMESTAMP : { '$gte' : start_time, '$lt' : end_time } } asc_search = source_collection.find(spec=query, fields='_id').sort('_id', ASCENDING).limit(1) if asc_search.count() == 0: raise LookupError('No messages in timeperiod: %s:%s in collection %s' % (start_time, end_time, source_collection_name)) first_object_id = asc_search[0]['_id'] dec_search = source_collection.find(spec=query, fields='_id').sort('_id', DESCENDING).limit(1) last_object_id = dec_search[0]['_id'] unit_of_work = UnitOfWorkEntry() unit_of_work.set_timestamp(start_time) unit_of_work.set_start_id(str(first_object_id)) unit_of_work.set_end_id(str(last_object_id)) unit_of_work.set_start_timestamp(start_time) unit_of_work.set_end_timestamp(end_time) unit_of_work.set_created_at(datetime.utcnow()) unit_of_work.set_source_collection(source_collection_name) unit_of_work.set_target_collection(target_collection_name) unit_of_work.set_state(unit_of_work.STATE_REQUESTED) unit_of_work.set_process_name(process_name) unit_of_work.set_number_of_retries(0) try: uow_id = unit_of_work_helper.insert(self.logger, unit_of_work) except DuplicateKeyError as e: e.first_object_id = str(first_object_id) e.last_object_id = str(last_object_id) e.process_name = process_name e.timestamp = start_time raise e self.publishers.get_publisher(process_name).publish(str(uow_id)) msg = 'Published: UOW %r for %r in timeperiod %r.' % (uow_id, process_name, start_time) self._log_message(INFO, process_name, time_record, msg) return unit_of_work
def start_by_process_name(process_name, *args): """ Function starts the process by: 1. retrieving its fully specified path name 2. if the path name ends with starter method - then creates an instance of the wrapping class and calls <code>starter(*args)</code> method on it 3. if the path name ends with starter function - then retrieves its module and calls <code>starter(*args)</code> function on it """ sys.stdout.write('INFO: Starter path {0} \n'.format(ProcessContext.get_classname(process_name))) t, m, starter = get_class(ProcessContext.get_classname(process_name)) if isinstance(m, class_types): sys.stdout.write('INFO: Starting process by calling starter method {0} \n'.format(starter)) instance = m(process_name) method = getattr(instance, starter) method(*args) elif isinstance(m, types.FunctionType): sys.stdout.write('INFO: Starting module.\n') function = m function(*args) else: raise ValueError('Improper starter path {0}'.format(ProcessContext.get_classname(process_name)))
def action_get_log(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier( self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier( time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) resp['log'] = node.job_record.log return resp
def _get_job_collection(self, process_name): """jobs are stored in 4 collections: hourly, daily, monthly and yearly; method looks for the proper job_collection base on process TIME_QUALIFIER""" qualifier = ProcessContext.get_time_qualifier(process_name) if qualifier == QUALIFIER_HOURLY: collection = self.ds.connection(COLLECTION_JOB_HOURLY) elif qualifier == QUALIFIER_DAILY: collection = self.ds.connection(COLLECTION_JOB_DAILY) elif qualifier == QUALIFIER_MONTHLY: collection = self.ds.connection(COLLECTION_JOB_MONTHLY) elif qualifier == QUALIFIER_YEARLY: collection = self.ds.connection(COLLECTION_JOB_YEARLY) else: raise ValueError('Unknown time qualifier: %s for %s' % (qualifier, process_name)) return collection
def get_reprocessing_candidates(self, since=None): """ method queries Unit Of Work whose <start_timeperiod> is younger than <since> and who could be candidates for re-processing """ collection = self.ds.connection(COLLECTION_UNIT_OF_WORK) query = { unit_of_work.STATE: { '$in': [ unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED ] } } if since is None: cursor = collection.find(query).sort('_id', ASCENDING) candidates = [UnitOfWork(document) for document in cursor] else: candidates = [] yearly_timeperiod = time_helper.cast_to_time_qualifier( QUALIFIER_YEARLY, since) query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod} cursor = collection.find(query).sort('_id', ASCENDING) for document in cursor: uow = UnitOfWork(document) if uow.process_name not in ProcessContext.CONTEXT: # this is a decommissioned process continue time_qualifier = ProcessContext.get_time_qualifier( uow.process_name) if time_qualifier == QUALIFIER_REAL_TIME: time_qualifier = QUALIFIER_HOURLY process_specific_since = time_helper.cast_to_time_qualifier( time_qualifier, since) if process_specific_since <= uow.start_timeperiod: candidates.append(uow) if len(candidates) == 0: raise LookupError( 'MongoDB has no reprocessing candidates units of work') return candidates
def start_process(options, args): """Start up specific daemon """ import psutil from system import process_helper from supervisor import supervisor_helper as helper from system.process_context import ProcessContext from supervisor.supervisor_constants import PROCESS_SUPERVISOR from constants import PROCESS_LAUNCH_PY logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY) box_id = helper.get_box_id(logger) if options.supervisor is True and options.app != PROCESS_SUPERVISOR: from db.model import box_configuration from db.dao.box_configuration_dao import BoxConfigurationDao message = 'INFO: Marking %r to be managed by Supervisor \n' % options.app sys.stdout.write(message) bc_dao = BoxConfigurationDao(logger) box_config = bc_dao.get_one(box_id) box_config.set_process_state(options.app, box_configuration.STATE_ON) bc_dao.update(box_config) return try: pid = process_helper.get_process_pid(options.app) if pid is not None: if psutil.pid_exists(pid): message = 'ERROR: Process %r is already running with pid %r\n' % ( options.app, pid) sys.stderr.write(message) sys.exit(1) if not options.interactive: # this block triggers if the options.interactive is not defined or is False process_helper.start_process(options.app, args) else: process_starter.start_by_process_name(options.app, args) except Exception as e: sys.stderr.write('Exception on starting %s : %s \n' % (options.app, str(e))) traceback.print_exc(file=sys.stderr)
def action_skip(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier( self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier( time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) self.logger.info('MX (requesting skip timeperiod %r for %r) { ' % (self.timeperiod, self.process_name)) effected_nodes = node.request_skip() for node in effected_nodes: resp[node.timeperiod] = TreeNodeDetails.get_details( self.logger, node) self.logger.info('}') return resp
def run_tests(options): import unittest import settings settings.enable_test_mode() argv = [sys.argv[0]] + args try: unittest.main(module=None, defaultTest='__main__.load_all_tests', argv=argv) except SystemExit as e: from system.process_context import ProcessContext from constants import PROCESS_LAUNCH_PY logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY) if e.code == 0: logger.info('PASS') else: logger.error('FAIL') raise
def action_get_uow(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier( self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier( time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) uow_id = node.job_record.related_unit_of_work if uow_id is None: resp = {'response': 'no related unit_of_work'} else: resp = self.uow_dao.get_one(uow_id).document for key in resp: resp[key] = str(resp[key]) return resp
def write(self, msg, level=logging.INFO): """ method implements stream write interface, allowing to redirect stdout to logger """ if msg is not None and len(msg.strip()) > 0: self.logger.log(level, msg) def flush(self): """ method implements stream flush interface, allowing to redirect stdout to logger """ for handler in self.logger.handlers: handler.flush() def isatty(self): """ is the sys.stdout attached to the terminal? python -c "import sys; print(sys.stdout.isatty())" (should write True) python -c "import sys; print(sys.stdout.isatty())" | grep . (should write False). :return: False, indicating that the output is pipped or redirected """ return False if __name__ == '__main__': from system.process_context import ProcessContext from tests.ut_process_context import PROCESS_UNIT_TEST, register_unit_test_context register_unit_test_context() logger = ProcessContext.get_logger(PROCESS_UNIT_TEST) logger.info('test_message') print('regular print message') sys.stdout.flush()
def __del__(self): """ removes PID file """ ProcessContext.remove_pid_file(self.process_name, process_id=self.process_id) self.logger.info('Shutdown {0}'.format(self.process_name))