def get_reprocessing_candidates(self, since=None):
""" method queries Unit Of Work whose <start_timeperiod> is younger than <since>
and who could be candidates for re-processing """
collection = self.ds.connection(COLLECTION_UNIT_OF_WORK)
query = {unit_of_work.STATE: {'$in': [unit_of_work.STATE_IN_PROGRESS,
unit_of_work.STATE_INVALID,
unit_of_work.STATE_REQUESTED]}}
if since is None:
cursor = collection.find(query).sort('_id', ASCENDING)
candidates = [UnitOfWork(document) for document in cursor]
else:
candidates = []
yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, since)
query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod}
cursor = collection.find(query).sort('_id', ASCENDING)
for document in cursor:
uow = UnitOfWork(document)
if uow.process_name not in ProcessContext.CONTEXT:
# this is a decommissioned process
continue
time_qualifier = ProcessContext.get_time_qualifier(uow.process_name)
if time_qualifier == QUALIFIER_REAL_TIME:
time_qualifier = QUALIFIER_HOURLY
process_specific_since = time_helper.cast_to_time_qualifier(time_qualifier, since)
if process_specific_since <= uow.start_timeperiod:
candidates.append(uow)
if len(candidates) == 0:
raise LookupError('MongoDB has no reprocessing candidates units of work')
return candidates
def _perform_assertions(self, start_timeperiod, delta):
yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY,
start_timeperiod)
monthly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_MONTHLY,
start_timeperiod)
daily_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_DAILY,
start_timeperiod)
number_of_leafs = 0
for yt, yearly_root in sorted(self.tree.root.children.items(), key=lambda x: x[0]):
self.assertEqual(yearly_timeperiod, yt)
for mt, monthly_root in sorted(yearly_root.children.items(), key=lambda x: x[0]):
self.assertEqual(monthly_timeperiod, mt)
for dt, daily_root in sorted(monthly_root.children.items(), key=lambda x: x[0]):
self.assertEqual(daily_timeperiod, dt)
number_of_leafs += 1
daily_timeperiod = time_helper.increment_timeperiod(QUALIFIER_DAILY, daily_timeperiod)
monthly_timeperiod = time_helper.increment_timeperiod(QUALIFIER_MONTHLY, monthly_timeperiod)
yearly_timeperiod = time_helper.increment_timeperiod(QUALIFIER_YEARLY, yearly_timeperiod)
self.assertEqual(number_of_leafs, delta + 1, 'Expected number of daily nodes was %d, while actual is %d'
% (delta + 1, number_of_leafs))
def load_tree(self):
""" method iterates thru all objects older than synergy_start_timeperiod parameter in job collections
and loads them into this timetable"""
timeperiod = settings['synergy_start_timeperiod']
yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, timeperiod)
monthly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_MONTHLY, timeperiod)
daily_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_DAILY, timeperiod)
hourly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_HOURLY, timeperiod)
self._build_tree_by_level(COLLECTION_JOB_HOURLY, since=hourly_timeperiod)
self._build_tree_by_level(COLLECTION_JOB_DAILY, since=daily_timeperiod)
self._build_tree_by_level(COLLECTION_JOB_MONTHLY, since=monthly_timeperiod)
self._build_tree_by_level(COLLECTION_JOB_YEARLY, since=yearly_timeperiod)
def details(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if self.timeperiod is None and tree is not None:
# return list of yearly nodes OR leafs for linear tree
resp['children'] = dict()
# limit number of children to return, since linear tree can holds thousands of nodes
sorted_keys = sorted(tree.root.children.keys(), reverse=True)
sorted_keys = sorted_keys[:settings['mx_children_limit']]
for key in sorted_keys:
child = tree.root.children[key]
resp['children'][key] = TreeNodeDetails.get_details(self.logger, child)
elif tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
resp['node'] = TreeNodeDetails.get_details(self.logger, node)
resp['children'] = dict()
for key in node.children:
child = node.children[key]
resp['children'][key] = TreeNodeDetails.get_details(self.logger, child)
return resp
def get_reprocessing_candidates(self, since=None):
""" method queries Unit Of Work whose <start_timeperiod> is younger than <since>
and who could be candidates for re-processing """
collection = self.ds.connection(COLLECTION_UNIT_OF_WORK)
query = {
unit_of_work.STATE: {
'$in': [
unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID,
unit_of_work.STATE_REQUESTED
]
}
}
if since is None:
cursor = collection.find(query).sort('_id', ASCENDING)
candidates = [UnitOfWork(document) for document in cursor]
else:
candidates = []
yearly_timeperiod = time_helper.cast_to_time_qualifier(
QUALIFIER_YEARLY, since)
query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod}
cursor = collection.find(query).sort('_id', ASCENDING)
for document in cursor:
uow = UnitOfWork(document)
if uow.process_name not in ProcessContext.CONTEXT:
# this is a decommissioned process
continue
time_qualifier = ProcessContext.get_time_qualifier(
uow.process_name)
if time_qualifier == QUALIFIER_REAL_TIME:
time_qualifier = QUALIFIER_HOURLY
process_specific_since = time_helper.cast_to_time_qualifier(
time_qualifier, since)
if process_specific_since <= uow.start_timeperiod:
candidates.append(uow)
if len(candidates) == 0:
raise LookupError(
'MongoDB has no reprocessing candidates units of work')
return candidates
def __get_hourly_node(self, timestamp):
timestamp_daily = cast_to_time_qualifier(self.process_daily, timestamp)
parent = self._ThreeLevelTree__get_daily_node(timestamp_daily)
node = parent.children.get(timestamp)
if node is None:
node = TreeNode(self, parent, self.process_hourly, timestamp, None)
parent.children[timestamp] = node
return node
def __get_monthly_node(self, timestamp):
timestamp_yearly = cast_to_time_qualifier(self.process_yearly, timestamp)
parent = self.__get_yearly_node(timestamp_yearly)
node = parent.children.get(timestamp)
if node is None:
node = TreeNode(self, parent, self.process_monthly, timestamp, None)
parent.children[timestamp] = node
return node
def action_get_log(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
self.timestamp = time_helper.cast_to_time_qualifier(self.process_name, self.timestamp)
node = tree.get_node_by_process(self.process_name, self.timestamp)
resp['log'] = node.time_record.get_log()
return resp
def action_get_log(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
resp['log'] = node.job_record.log
return resp
def action_get_log(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(
self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(
time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
resp['log'] = node.job_record.log
return resp
def action_skip(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
self.timestamp = time_helper.cast_to_time_qualifier(self.process_name, self.timestamp)
node = tree.get_node_by_process(self.process_name, self.timestamp)
self.logger.info('MX (requesting skip timeperiod %r for %r) { ' % (self.timestamp, self.process_name))
effected_nodes = node.request_skip()
for node in effected_nodes:
resp[node.timestamp] = NodeDetails._get_nodes_details(self.logger, node)
self.logger.info('}')
return resp
def test_cast_to_time_qualifier(self):
processes = [process_context.PROCESS_SITE_HOURLY, process_context.PROCESS_SITE_HOURLY,
process_context.PROCESS_SITE_DAILY, process_context.PROCESS_SITE_DAILY,
process_context.PROCESS_SITE_MONTHLY, process_context.PROCESS_SITE_MONTHLY,
process_context.PROCESS_SITE_YEARLY, process_context.PROCESS_SITE_YEARLY]
params = ['2010123123', '20101231231232',
'2010123123', '2010123100',
'2010123100', '2010120000',
'2010120000', '2010000000']
expected = ['2010123123', '2010123123',
'2010123100', '2010123100',
'2010120000', '2010120000',
'2010000000', '2010000000']
for i in range(8):
assert time_helper.cast_to_time_qualifier(processes[i], params[i]) == expected[i]
def test_cast_to_time_qualifier(self):
qualifiers = [QUALIFIER_HOURLY, QUALIFIER_HOURLY,
QUALIFIER_DAILY, QUALIFIER_DAILY,
QUALIFIER_MONTHLY, QUALIFIER_MONTHLY,
QUALIFIER_YEARLY, QUALIFIER_YEARLY]
params = ['2010123123', '20101231231232',
'2010123123', '2010123100',
'2010123100', '2010120000',
'2010120000', '2010000000']
expected = ['2010123123', '2010123123',
'2010123100', '2010123100',
'2010120000', '2010120000',
'2010000000', '2010000000']
for i in range(8):
assert time_helper.cast_to_time_qualifier(qualifiers[i], params[i]) == expected[i]
def action_reprocess(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
self.logger.info('MX (requesting re-process timeperiod %r for %r) {' % (self.timeperiod, self.process_name))
effected_nodes = node.request_reprocess()
for node in effected_nodes:
resp[node.timeperiod] = TreeNodeDetails.get_details(self.logger, node)
self.logger.info('}')
return resp
def _build_tree(self, rebuild, process_name, method_get_node):
"""method builds tree by iterating from the synergy_start_timestamp to current time
and inserting corresponding nodes"""
if rebuild or self.build_timestamp is None:
timestamp = settings['synergy_start_timestamp']
timestamp = cast_to_time_qualifier(process_name, timestamp)
else:
timestamp = self.build_timestamp
now = time_helper.datetime_to_synergy(process_name, datetime.utcnow())
while now >= timestamp:
method_get_node(timestamp)
timestamp = time_helper.increment_time(process_name, timestamp)
self.build_timestamp = now
def action_get_uow(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
self.timestamp = time_helper.cast_to_time_qualifier(self.process_name, self.timestamp)
node = tree.get_node_by_process(self.process_name, self.timestamp)
uow_id = node.time_record.get_related_unit_of_work()
if uow_id is None:
resp = {'response' : 'no related unit_of_work'}
else:
resp = unit_of_work_helper.retrieve_by_id(self.logger, uow_id).get_document()
for key in resp:
resp[key] = str(resp[key])
return resp
def action_get_uow(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
uow_id = node.job_record.related_unit_of_work
if uow_id is None:
resp = {'response': 'no related unit_of_work'}
else:
resp = self.uow_dao.get_one(uow_id).document
for key in resp:
resp[key] = str(resp[key])
return resp
def action_skip(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(
self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(
time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
self.logger.info('MX (requesting skip timeperiod %r for %r) { ' %
(self.timeperiod, self.process_name))
effected_nodes = node.request_skip()
for node in effected_nodes:
resp[node.timeperiod] = TreeNodeDetails.get_details(
self.logger, node)
self.logger.info('}')
return resp
def action_get_uow(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(
self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(
time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
uow_id = node.job_record.related_unit_of_work
if uow_id is None:
resp = {'response': 'no related unit_of_work'}
else:
resp = self.uow_dao.get_one(uow_id).document
for key in resp:
resp[key] = str(resp[key])
return resp
def details(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if self.timestamp is None and tree is not None:
# return list of yearly nodes
resp['children'] = dict()
for key in tree.root.children:
child = tree.root.children[key]
resp['children'][key] = NodeDetails._get_nodes_details(self.logger, child)
elif tree is not None:
self.timestamp = time_helper.cast_to_time_qualifier(self.process_name, self.timestamp)
node = tree.get_node_by_process(self.process_name, self.timestamp)
resp['node'] = NodeDetails._get_nodes_details(self.logger, node)
resp['children'] = dict()
for key in node.children:
child = node.children[key]
resp['children'][key] = NodeDetails._get_nodes_details(self.logger, child)
return resp
