def get_reprocessing_candidates(self, since=None): """ method queries Unit Of Work whose <start_timeperiod> is younger than <since> and who could be candidates for re-processing """ collection = self.ds.connection(COLLECTION_UNIT_OF_WORK) query = {unit_of_work.STATE: {'$in': [unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED]}} if since is None: cursor = collection.find(query).sort('_id', ASCENDING) candidates = [UnitOfWork(document) for document in cursor] else: candidates = [] yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, since) query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod} cursor = collection.find(query).sort('_id', ASCENDING) for document in cursor: uow = UnitOfWork(document) if uow.process_name not in ProcessContext.CONTEXT: # this is a decommissioned process continue time_qualifier = ProcessContext.get_time_qualifier(uow.process_name) if time_qualifier == QUALIFIER_REAL_TIME: time_qualifier = QUALIFIER_HOURLY process_specific_since = time_helper.cast_to_time_qualifier(time_qualifier, since) if process_specific_since <= uow.start_timeperiod: candidates.append(uow) if len(candidates) == 0: raise LookupError('MongoDB has no reprocessing candidates units of work') return candidates
def _perform_assertions(self, start_timeperiod, delta): yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, start_timeperiod) monthly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_MONTHLY, start_timeperiod) daily_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_DAILY, start_timeperiod) number_of_leafs = 0 for yt, yearly_root in sorted(self.tree.root.children.items(), key=lambda x: x[0]): self.assertEqual(yearly_timeperiod, yt) for mt, monthly_root in sorted(yearly_root.children.items(), key=lambda x: x[0]): self.assertEqual(monthly_timeperiod, mt) for dt, daily_root in sorted(monthly_root.children.items(), key=lambda x: x[0]): self.assertEqual(daily_timeperiod, dt) number_of_leafs += 1 daily_timeperiod = time_helper.increment_timeperiod(QUALIFIER_DAILY, daily_timeperiod) monthly_timeperiod = time_helper.increment_timeperiod(QUALIFIER_MONTHLY, monthly_timeperiod) yearly_timeperiod = time_helper.increment_timeperiod(QUALIFIER_YEARLY, yearly_timeperiod) self.assertEqual(number_of_leafs, delta + 1, 'Expected number of daily nodes was %d, while actual is %d' % (delta + 1, number_of_leafs))
def load_tree(self): """ method iterates thru all objects older than synergy_start_timeperiod parameter in job collections and loads them into this timetable""" timeperiod = settings['synergy_start_timeperiod'] yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, timeperiod) monthly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_MONTHLY, timeperiod) daily_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_DAILY, timeperiod) hourly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_HOURLY, timeperiod) self._build_tree_by_level(COLLECTION_JOB_HOURLY, since=hourly_timeperiod) self._build_tree_by_level(COLLECTION_JOB_DAILY, since=daily_timeperiod) self._build_tree_by_level(COLLECTION_JOB_MONTHLY, since=monthly_timeperiod) self._build_tree_by_level(COLLECTION_JOB_YEARLY, since=yearly_timeperiod)
def details(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if self.timeperiod is None and tree is not None: # return list of yearly nodes OR leafs for linear tree resp['children'] = dict() # limit number of children to return, since linear tree can holds thousands of nodes sorted_keys = sorted(tree.root.children.keys(), reverse=True) sorted_keys = sorted_keys[:settings['mx_children_limit']] for key in sorted_keys: child = tree.root.children[key] resp['children'][key] = TreeNodeDetails.get_details(self.logger, child) elif tree is not None: time_qualifier = ProcessContext.get_time_qualifier(self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) resp['node'] = TreeNodeDetails.get_details(self.logger, node) resp['children'] = dict() for key in node.children: child = node.children[key] resp['children'][key] = TreeNodeDetails.get_details(self.logger, child) return resp
def get_reprocessing_candidates(self, since=None): """ method queries Unit Of Work whose <start_timeperiod> is younger than <since> and who could be candidates for re-processing """ collection = self.ds.connection(COLLECTION_UNIT_OF_WORK) query = { unit_of_work.STATE: { '$in': [ unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED ] } } if since is None: cursor = collection.find(query).sort('_id', ASCENDING) candidates = [UnitOfWork(document) for document in cursor] else: candidates = [] yearly_timeperiod = time_helper.cast_to_time_qualifier( QUALIFIER_YEARLY, since) query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod} cursor = collection.find(query).sort('_id', ASCENDING) for document in cursor: uow = UnitOfWork(document) if uow.process_name not in ProcessContext.CONTEXT: # this is a decommissioned process continue time_qualifier = ProcessContext.get_time_qualifier( uow.process_name) if time_qualifier == QUALIFIER_REAL_TIME: time_qualifier = QUALIFIER_HOURLY process_specific_since = time_helper.cast_to_time_qualifier( time_qualifier, since) if process_specific_since <= uow.start_timeperiod: candidates.append(uow) if len(candidates) == 0: raise LookupError( 'MongoDB has no reprocessing candidates units of work') return candidates
def __get_hourly_node(self, timestamp): timestamp_daily = cast_to_time_qualifier(self.process_daily, timestamp) parent = self._ThreeLevelTree__get_daily_node(timestamp_daily) node = parent.children.get(timestamp) if node is None: node = TreeNode(self, parent, self.process_hourly, timestamp, None) parent.children[timestamp] = node return node
def __get_monthly_node(self, timestamp): timestamp_yearly = cast_to_time_qualifier(self.process_yearly, timestamp) parent = self.__get_yearly_node(timestamp_yearly) node = parent.children.get(timestamp) if node is None: node = TreeNode(self, parent, self.process_monthly, timestamp, None) parent.children[timestamp] = node return node
def action_get_log(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: self.timestamp = time_helper.cast_to_time_qualifier(self.process_name, self.timestamp) node = tree.get_node_by_process(self.process_name, self.timestamp) resp['log'] = node.time_record.get_log() return resp
def action_get_log(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier(self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) resp['log'] = node.job_record.log return resp
def action_get_log(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier( self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier( time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) resp['log'] = node.job_record.log return resp
def action_skip(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: self.timestamp = time_helper.cast_to_time_qualifier(self.process_name, self.timestamp) node = tree.get_node_by_process(self.process_name, self.timestamp) self.logger.info('MX (requesting skip timeperiod %r for %r) { ' % (self.timestamp, self.process_name)) effected_nodes = node.request_skip() for node in effected_nodes: resp[node.timestamp] = NodeDetails._get_nodes_details(self.logger, node) self.logger.info('}') return resp
def test_cast_to_time_qualifier(self): processes = [process_context.PROCESS_SITE_HOURLY, process_context.PROCESS_SITE_HOURLY, process_context.PROCESS_SITE_DAILY, process_context.PROCESS_SITE_DAILY, process_context.PROCESS_SITE_MONTHLY, process_context.PROCESS_SITE_MONTHLY, process_context.PROCESS_SITE_YEARLY, process_context.PROCESS_SITE_YEARLY] params = ['2010123123', '20101231231232', '2010123123', '2010123100', '2010123100', '2010120000', '2010120000', '2010000000'] expected = ['2010123123', '2010123123', '2010123100', '2010123100', '2010120000', '2010120000', '2010000000', '2010000000'] for i in range(8): assert time_helper.cast_to_time_qualifier(processes[i], params[i]) == expected[i]
def test_cast_to_time_qualifier(self): qualifiers = [QUALIFIER_HOURLY, QUALIFIER_HOURLY, QUALIFIER_DAILY, QUALIFIER_DAILY, QUALIFIER_MONTHLY, QUALIFIER_MONTHLY, QUALIFIER_YEARLY, QUALIFIER_YEARLY] params = ['2010123123', '20101231231232', '2010123123', '2010123100', '2010123100', '2010120000', '2010120000', '2010000000'] expected = ['2010123123', '2010123123', '2010123100', '2010123100', '2010120000', '2010120000', '2010000000', '2010000000'] for i in range(8): assert time_helper.cast_to_time_qualifier(qualifiers[i], params[i]) == expected[i]
def action_reprocess(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier(self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) self.logger.info('MX (requesting re-process timeperiod %r for %r) {' % (self.timeperiod, self.process_name)) effected_nodes = node.request_reprocess() for node in effected_nodes: resp[node.timeperiod] = TreeNodeDetails.get_details(self.logger, node) self.logger.info('}') return resp
def _build_tree(self, rebuild, process_name, method_get_node): """method builds tree by iterating from the synergy_start_timestamp to current time and inserting corresponding nodes""" if rebuild or self.build_timestamp is None: timestamp = settings['synergy_start_timestamp'] timestamp = cast_to_time_qualifier(process_name, timestamp) else: timestamp = self.build_timestamp now = time_helper.datetime_to_synergy(process_name, datetime.utcnow()) while now >= timestamp: method_get_node(timestamp) timestamp = time_helper.increment_time(process_name, timestamp) self.build_timestamp = now
def action_get_uow(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: self.timestamp = time_helper.cast_to_time_qualifier(self.process_name, self.timestamp) node = tree.get_node_by_process(self.process_name, self.timestamp) uow_id = node.time_record.get_related_unit_of_work() if uow_id is None: resp = {'response' : 'no related unit_of_work'} else: resp = unit_of_work_helper.retrieve_by_id(self.logger, uow_id).get_document() for key in resp: resp[key] = str(resp[key]) return resp
def action_get_uow(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier(self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) uow_id = node.job_record.related_unit_of_work if uow_id is None: resp = {'response': 'no related unit_of_work'} else: resp = self.uow_dao.get_one(uow_id).document for key in resp: resp[key] = str(resp[key]) return resp
def action_skip(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier( self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier( time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) self.logger.info('MX (requesting skip timeperiod %r for %r) { ' % (self.timeperiod, self.process_name)) effected_nodes = node.request_skip() for node in effected_nodes: resp[node.timeperiod] = TreeNodeDetails.get_details( self.logger, node) self.logger.info('}') return resp
def action_get_uow(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if tree is not None: time_qualifier = ProcessContext.get_time_qualifier( self.process_name) self.timeperiod = time_helper.cast_to_time_qualifier( time_qualifier, self.timeperiod) node = tree.get_node_by_process(self.process_name, self.timeperiod) uow_id = node.job_record.related_unit_of_work if uow_id is None: resp = {'response': 'no related unit_of_work'} else: resp = self.uow_dao.get_one(uow_id).document for key in resp: resp[key] = str(resp[key]) return resp
def details(self): resp = dict() timetable = self.mbean.timetable tree = timetable.get_tree(self.process_name) if self.timestamp is None and tree is not None: # return list of yearly nodes resp['children'] = dict() for key in tree.root.children: child = tree.root.children[key] resp['children'][key] = NodeDetails._get_nodes_details(self.logger, child) elif tree is not None: self.timestamp = time_helper.cast_to_time_qualifier(self.process_name, self.timestamp) node = tree.get_node_by_process(self.process_name, self.timestamp) resp['node'] = NodeDetails._get_nodes_details(self.logger, node) resp['children'] = dict() for key in node.children: child = node.children[key] resp['children'][key] = NodeDetails._get_nodes_details(self.logger, child) return resp