def cast_to_time_qualifier(process_name, timestamp): """ method is used to cast synergy_time accordingly to process time qualifier. For example: for QUALIFIER_HOURLY, it can be either 20100101_19 or 20100101_193412 """ date_format = None qualifier = ProcessContext.get_time_qualifier(process_name) if qualifier == ProcessContext.QUALIFIER_HOURLY: if len(timestamp) > 10: t = datetime.strptime(timestamp, SYNERGY_SESSION_PATTERN) return t.strftime(SYNERGY_DATE_PATTERN) else: return timestamp elif qualifier == ProcessContext.QUALIFIER_DAILY : date_format = '%Y%m%d00' elif qualifier == ProcessContext.QUALIFIER_MONTHLY: date_format = '%Y%m0000' elif qualifier == ProcessContext.QUALIFIER_YEARLY: date_format = '%Y000000' pattern = define_pattern(timestamp) t = datetime.strptime(timestamp, pattern) if date_format is not None: return t.strftime(date_format) else: raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
def get_reprocessing_candidates(self, since=None): """ method queries Unit Of Work whose <start_timeperiod> is younger than <since> and who could be candidates for re-processing """ collection = self.ds.connection(COLLECTION_UNIT_OF_WORK) query = {unit_of_work.STATE: {'$in': [unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED]}} if since is None: cursor = collection.find(query).sort('_id', ASCENDING) candidates = [UnitOfWork(document) for document in cursor] else: candidates = [] yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, since) query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod} cursor = collection.find(query).sort('_id', ASCENDING) for document in cursor: uow = UnitOfWork(document) if uow.process_name not in ProcessContext.CONTEXT: # this is a decommissioned process continue time_qualifier = ProcessContext.get_time_qualifier(uow.process_name) if time_qualifier == QUALIFIER_REAL_TIME: time_qualifier = QUALIFIER_HOURLY process_specific_since = time_helper.cast_to_time_qualifier(time_qualifier, since) if process_specific_since <= uow.start_timeperiod: candidates.append(uow) if len(candidates) == 0: raise LookupError('MongoDB has no reprocessing candidates units of work') return candidates
def increment_time(process_name, timestamp): """ method is used by Scheduler to define <<next>> time period. For hourly, it is next hour: 20100101_19 -> 20100101_20 For month - next month: 201001 -> 201002, etc""" qualifier = ProcessContext.get_time_qualifier(process_name) pattern = define_pattern(timestamp) t = datetime.strptime(timestamp, pattern) if qualifier == ProcessContext.QUALIFIER_HOURLY: t = t + timedelta(hours=1) return t.strftime('%Y%m%d%H') elif qualifier == ProcessContext.QUALIFIER_DAILY: t = t + timedelta(days=1) return t.strftime('%Y%m%d00') elif qualifier == ProcessContext.QUALIFIER_MONTHLY: if t.month + 1 > 12: new_month = 1 new_year = t.year + 1 t = t.replace(year = new_year, month = new_month) else: t = t.replace(month = t.month + 1) return t.strftime('%Y%m0000') elif qualifier == ProcessContext.QUALIFIER_YEARLY: t = t.replace(year = t.year + 1) return t.strftime('%Y000000') else: raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
def details(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if self.timeperiod is None and tree is not None: # return list of yearly nodes OR leafs for linear tree resp['children'] = dict() # limit number of children to return, since linear tree can holds thousands of nodes sorted_keys = sorted(tree.root.children.keys(), reverse=True) sorted_keys = sorted_keys[:settings['mx_children_limit']] for key in sorted_keys: child = tree.root.children[key] resp['children'][key] = TreeNodeDetails.get_details(self.logger, child) elif tree is not None: time_qualifier = ProcessContext.get_time_qualifier(self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) resp['node'] = TreeNodeDetails.get_details(self.logger, node) resp['children'] = dict() for key in node.children: child = node.children[key] resp['children'][key] = TreeNodeDetails.get_details(self.logger, child) return resp
def _process_state_in_progress(self, process_name, job_record, start_timeperiod): """ method that takes care of processing job records in STATE_IN_PROGRESS state""" time_qualifier = ProcessContext.get_time_qualifier(process_name) end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod) actual_timeperiod = time_helper.actual_timeperiod(time_qualifier) can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record) uow = self.uow_dao.get_one(job_record.related_unit_of_work) if start_timeperiod == actual_timeperiod or can_finalize_job_record is False: if uow.state in [unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED]: # current uow has not been processed yet. update it self.update_scope_of_processing(process_name, uow, start_timeperiod, end_timeperiod, job_record) else: # cls.STATE_IN_PROGRESS, cls.STATE_PROCESSED, cls.STATE_CANCELED # create new uow to cover new inserts self._compute_and_transfer_to_progress(process_name, start_timeperiod, end_timeperiod, job_record) elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True: # create new uow for FINAL RUN self._compute_and_transfer_to_final_run(process_name, start_timeperiod, end_timeperiod, job_record) else: msg = 'job record %s has timeperiod from future %s vs current time %s' \ % (job_record.document['_id'], start_timeperiod, actual_timeperiod) self._log_message(ERROR, process_name, job_record, msg)
def entries(self): list_of_trees = [] try: sorter_keys = sorted(context.timetable_context.keys()) for tree_name in sorter_keys: tree_obj = self.mbean.timetable.trees[tree_name] tree_row = list() tree_row.append(tree_name) # index 0 tree_row.append(tree_obj.mx_page) # index 1 tree_row.append(tree_obj.mx_name) # index 2 processes = dict() # index 3 context_entry = context.timetable_context[tree_name] for process_name in context_entry.enclosed_processes: process_details = [process_name, # index x0 ProcessContext.get_time_qualifier(process_name), # index x1 self._state_machine_name(process_name), # index x2 ProcessContext.get_process_type(process_name), # index x3 ProcessContext.run_on_active_timeperiod(process_name), # index x4 context_entry.dependent_on, # index x5 self._list_of_dependant_trees(tree_obj)] # index x6 processes[process_name] = process_details tree_row.append(processes) list_of_trees.append(tree_row) except Exception as e: self.logger.error('MX Exception %s' % str(e), exc_info=True) return list_of_trees
def _process_state_in_progress(self, process_name, job_record, start_timeperiod): """ method that takes care of processing job records in STATE_IN_PROGRESS state""" time_qualifier = ProcessContext.get_time_qualifier(process_name) end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod) actual_timeperiod = time_helper.actual_timeperiod(time_qualifier) can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record) uow = self.uow_dao.get_one(job_record.related_unit_of_work) iteration = int(uow.end_id) try: if start_timeperiod == actual_timeperiod or can_finalize_job_record is False: if uow.state in [unit_of_work.STATE_REQUESTED, unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID]: # Large Job processing takes more than 1 tick of Scheduler # Let the Large Job processing complete - do no updates to Scheduler records pass elif uow.state in [unit_of_work.STATE_PROCESSED, unit_of_work.STATE_CANCELED]: # create new uow to cover new inserts uow = self.insert_uow(process_name, start_timeperiod, end_timeperiod, iteration + 1, job_record) self.timetable.update_job_record(process_name, job_record, uow, job.STATE_IN_PROGRESS) elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True: if uow.state in [unit_of_work.STATE_REQUESTED, unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID]: # Job processing has not started yet # Let the processing complete - do no updates to Scheduler records msg = 'Suppressed creating uow for %s in timeperiod %s; job record is in %s; uow is in %s' \ % (process_name, job_record.timeperiod, job_record.state, uow.state) elif uow.state == unit_of_work.STATE_PROCESSED: self.timetable.update_job_record(process_name, job_record, uow, job.STATE_PROCESSED) timetable_tree = self.timetable.get_tree(process_name) timetable_tree.build_tree() msg = 'Transferred job record %s in timeperiod %s to STATE_PROCESSED for %s' \ % (job_record.document['_id'], job_record.timeperiod, process_name) elif uow.state == unit_of_work.STATE_CANCELED: self.timetable.update_job_record(process_name, job_record, uow, job.STATE_SKIPPED) msg = 'Transferred job record %s in timeperiod %s to STATE_SKIPPED for %s' \ % (job_record.document['_id'], job_record.timeperiod, process_name) else: msg = 'Unknown state %s for job record %s in timeperiod %s for %s' \ % (uow.state, job_record.document['_id'], job_record.timeperiod, process_name) self._log_message(INFO, process_name, job_record, msg) else: msg = 'Job record %s has timeperiod from future %s vs current time %s' \ % (job_record.document['_id'], start_timeperiod, actual_timeperiod) self._log_message(ERROR, process_name, job_record, msg) except DuplicateKeyError as e: uow = self.recover_from_duplicatekeyerror(e) if uow is not None: self.timetable.update_job_record(process_name, job_record, uow, job_record.state) else: msg = 'MANUAL INTERVENTION REQUIRED! Unable to identify unit_of_work for %s in %s' \ % (process_name, job_record.timeperiod) self._log_message(ERROR, process_name, job_record, msg)
def action_get_log(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier(self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) resp['log'] = node.job_record.log return resp
def __init__(self, tree, parent, process_name, timeperiod, job_record): # initializes the data members self.children = dict() self.tree = tree self.parent = parent self.process_name = process_name self.timeperiod = timeperiod self.job_record = job_record if parent is None and process_name is None and timeperiod is None and job_record is None: # special case - node is TREE ROOT self.time_qualifier = None else: self.time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
def action_get_log(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier( self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier( time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) resp['log'] = node.job_record.log return resp
def _get_timetable_collection(self, process_name): """timetable stores timeperiod in 4 collections: hourly, daily, monthly and yearly; method looks for the proper timetable_collection base on process TIME_QUALIFIER""" qualifier = ProcessContext.get_time_qualifier(process_name) if qualifier == ProcessContext.QUALIFIER_HOURLY: collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_HOURLY) elif qualifier == ProcessContext.QUALIFIER_DAILY: collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_DAILY) elif qualifier == ProcessContext.QUALIFIER_MONTHLY: collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_MONTHLY) elif qualifier == ProcessContext.QUALIFIER_YEARLY: collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_YEARLY) else: raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name)) return collection
def action_skip(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier(self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) self.logger.info('MX (requesting skip timeperiod %r for %r) { ' % (self.timeperiod, self.process_name)) effected_nodes = node.request_skip() for node in effected_nodes: resp[node.timeperiod] = TreeNodeDetails.get_details(self.logger, node) self.logger.info('}') return resp
def synergy_to_datetime(process_name, timestamp): """ method receives timestamp in Synergy format YYYYMMDD_HH and convert it to _naive_ datetime""" qualifier = ProcessContext.get_time_qualifier(process_name) if qualifier == ProcessContext.QUALIFIER_HOURLY: date_format = SYNERGY_DATE_PATTERN elif qualifier == ProcessContext.QUALIFIER_DAILY: date_format = '%Y%m%d00' elif qualifier == ProcessContext.QUALIFIER_MONTHLY: date_format = '%Y%m0000' elif qualifier == ProcessContext.QUALIFIER_YEARLY: date_format = '%Y000000' elif qualifier == ProcessContext.QUALIFIER_REAL_TIME: date_format = SYNERGY_SESSION_PATTERN else: raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name)) return datetime.strptime(timestamp, date_format).replace(tzinfo=None)
def _get_job_collection(self, process_name): """jobs are stored in 4 collections: hourly, daily, monthly and yearly; method looks for the proper job_collection base on process TIME_QUALIFIER""" qualifier = ProcessContext.get_time_qualifier(process_name) if qualifier == QUALIFIER_HOURLY: collection = self.ds.connection(COLLECTION_JOB_HOURLY) elif qualifier == QUALIFIER_DAILY: collection = self.ds.connection(COLLECTION_JOB_DAILY) elif qualifier == QUALIFIER_MONTHLY: collection = self.ds.connection(COLLECTION_JOB_MONTHLY) elif qualifier == QUALIFIER_YEARLY: collection = self.ds.connection(COLLECTION_JOB_YEARLY) else: raise ValueError('Unknown time qualifier: %s for %s' % (qualifier, process_name)) return collection
def datetime_to_synergy(process_name, dt): """ method parses datetime and returns Synergy Date""" date_format = None qualifier = ProcessContext.get_time_qualifier(process_name) if qualifier == ProcessContext.QUALIFIER_HOURLY: date_format = SYNERGY_DATE_PATTERN elif qualifier == ProcessContext.QUALIFIER_DAILY: date_format = '%Y%m%d00' elif qualifier == ProcessContext.QUALIFIER_MONTHLY: date_format = '%Y%m0000' elif qualifier == ProcessContext.QUALIFIER_YEARLY: date_format = '%Y000000' elif qualifier == ProcessContext.QUALIFIER_REAL_TIME: date_format = SYNERGY_SESSION_PATTERN else: raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name)) return dt.strftime(date_format)
def get_reprocessing_candidates(self, since=None): """ method queries Unit Of Work whose <start_timeperiod> is younger than <since> and who could be candidates for re-processing """ collection = self.ds.connection(COLLECTION_UNIT_OF_WORK) query = { unit_of_work.STATE: { '$in': [ unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED ] } } if since is None: cursor = collection.find(query).sort('_id', ASCENDING) candidates = [UnitOfWork(document) for document in cursor] else: candidates = [] yearly_timeperiod = time_helper.cast_to_time_qualifier( QUALIFIER_YEARLY, since) query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod} cursor = collection.find(query).sort('_id', ASCENDING) for document in cursor: uow = UnitOfWork(document) if uow.process_name not in ProcessContext.CONTEXT: # this is a decommissioned process continue time_qualifier = ProcessContext.get_time_qualifier( uow.process_name) if time_qualifier == QUALIFIER_REAL_TIME: time_qualifier = QUALIFIER_HOURLY process_specific_since = time_helper.cast_to_time_qualifier( time_qualifier, since) if process_specific_since <= uow.start_timeperiod: candidates.append(uow) if len(candidates) == 0: raise LookupError( 'MongoDB has no reprocessing candidates units of work') return candidates
def _process_state_embryo(self, process_name, job_record, start_timeperiod): """ method that takes care of processing job records in STATE_EMBRYO state""" time_qualifier = ProcessContext.get_time_qualifier(process_name) end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod) try: uow = self.insert_uow(process_name, start_timeperiod, end_timeperiod, 0, job_record) except DuplicateKeyError as e: msg = 'Catching up with latest unit_of_work %s in timeperiod %s, because of: %r' \ % (process_name, job_record.timeperiod, e) self._log_message(WARNING, process_name, job_record, msg) uow = self.recover_from_duplicatekeyerror(e) if uow is not None: self.timetable.update_job_record(process_name, job_record, uow, job.STATE_IN_PROGRESS) else: msg = 'MANUAL INTERVENTION REQUIRED! Unable to locate unit_of_work for %s in %s' \ % (process_name, job_record.timeperiod) self._log_message(WARNING, process_name, job_record, msg)
def action_get_uow(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier(self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) uow_id = node.job_record.related_unit_of_work if uow_id is None: resp = {'response': 'no related unit_of_work'} else: resp = self.uow_dao.get_one(uow_id).document for key in resp: resp[key] = str(resp[key]) return resp
def action_skip(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier( self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier( time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) self.logger.info('MX (requesting skip timeperiod %r for %r) { ' % (self.timeperiod, self.process_name)) effected_nodes = node.request_skip() for node in effected_nodes: resp[node.timeperiod] = TreeNodeDetails.get_details( self.logger, node) self.logger.info('}') return resp
def _get_nodes_details(cls, logger, node): """method returns { process_name : string, timestamp : string, number_of_children : integer, number_of_failed_calls : integer, state : STATE_SKIPPED, STATE_IN_PROGRESS, STATE_PROCESSED, STATE_FINAL_RUN, STATE_EMBRYO } """ description = dict() try: description['process_name'] = node.process_name description['time_qualifier'] = ProcessContext.get_time_qualifier(node.process_name) description['number_of_children'] = len(node.children) description['number_of_failed_calls'] = node.time_record.get_number_of_failures() description['timestamp'] = node.time_record.get_timestamp() description['state'] = node.time_record.get_state() except Exception as e: logger.error('MX Exception: ' + str(e), exc_info=True) finally: return description
def action_get_uow(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier( self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier( time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) uow_id = node.job_record.related_unit_of_work if uow_id is None: resp = {'response': 'no related unit_of_work'} else: resp = self.uow_dao.get_one(uow_id).document for key in resp: resp[key] = str(resp[key]) return resp
def fire_managed_worker(self, *args): """requests vertical aggregator (hourly site, daily variant, etc) to start up""" try: process_name = args[0] scheduler_entry_obj = args[1] self.logger.info('%s {' % process_name) timetable_record = self.timetable.get_next_job_record(process_name) pipeline = self.pipelines[scheduler_entry_obj.state_machine_name] run_on_active_timeperiod = ProcessContext.run_on_active_timeperiod(scheduler_entry_obj.process_name) if not run_on_active_timeperiod: time_qualifier = ProcessContext.get_time_qualifier(process_name) incremented_timeperiod = time_helper.increment_timeperiod(time_qualifier, timetable_record.timeperiod) dt_record_timestamp = time_helper.synergy_to_datetime(time_qualifier, incremented_timeperiod) dt_record_timestamp += timedelta(minutes=LAG_5_MINUTES) if datetime.utcnow() <= dt_record_timestamp: self.logger.info('Timetable record %s for timeperiod %s will not be triggered until %s.' % (timetable_record.document['_id'], timetable_record.timeperiod, dt_record_timestamp.strftime('%Y-%m-%d %H:%M:%S'))) return process_type = ProcessContext.get_process_type(scheduler_entry_obj.process_name) if process_type == TYPE_BLOCKING_DEPENDENCIES: pipeline.manage_pipeline_with_blocking_dependencies(process_name, timetable_record) elif process_type == TYPE_BLOCKING_CHILDREN: pipeline.manage_pipeline_with_blocking_children(process_name, timetable_record) elif process_type == TYPE_MANAGED: pipeline.manage_pipeline_for_process(process_name, timetable_record) except (AMQPError, IOError) as e: self.logger.error('AMQPError: %s' % str(e), exc_info=True) self.publishers.reset_all(suppress_logging=True) except Exception as e: self.logger.error('Exception: %s' % str(e), exc_info=True) finally: self.logger.info('}')
def match_time_qualifier(actual_process_name, candidate_process_name): if candidate_process_name is None: return False time_qualifier = ProcessContext.get_time_qualifier(actual_process_name) candidate_qualifier = ProcessContext.get_time_qualifier(candidate_process_name) return time_qualifier == candidate_qualifier
def match_time_qualifier(time_qualifier, candidate_process_name): """ :return: True if candidate_process has the same time qualifier as given """ if candidate_process_name is None: return False candidate_qualifier = ProcessContext.get_time_qualifier(candidate_process_name) return time_qualifier == candidate_qualifier
def _process_state_embryo(self, process_name, job_record, start_timeperiod): """ method that takes care of processing job records in STATE_EMBRYO state""" time_qualifier = ProcessContext.get_time_qualifier(process_name) end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod) self._compute_and_transfer_to_progress(process_name, start_timeperiod, end_timeperiod, job_record)