def cast_to_time_qualifier(process_name, timestamp):
""" method is used to cast synergy_time accordingly to process time qualifier.
For example: for QUALIFIER_HOURLY, it can be either 20100101_19 or 20100101_193412 """
date_format = None
qualifier = ProcessContext.get_time_qualifier(process_name)
if qualifier == ProcessContext.QUALIFIER_HOURLY:
if len(timestamp) > 10:
t = datetime.strptime(timestamp, SYNERGY_SESSION_PATTERN)
return t.strftime(SYNERGY_DATE_PATTERN)
else:
return timestamp
elif qualifier == ProcessContext.QUALIFIER_DAILY :
date_format = '%Y%m%d00'
elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
date_format = '%Y%m0000'
elif qualifier == ProcessContext.QUALIFIER_YEARLY:
date_format = '%Y000000'
pattern = define_pattern(timestamp)
t = datetime.strptime(timestamp, pattern)
if date_format is not None:
return t.strftime(date_format)
else:
raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
def get_reprocessing_candidates(self, since=None):
""" method queries Unit Of Work whose <start_timeperiod> is younger than <since>
and who could be candidates for re-processing """
collection = self.ds.connection(COLLECTION_UNIT_OF_WORK)
query = {unit_of_work.STATE: {'$in': [unit_of_work.STATE_IN_PROGRESS,
unit_of_work.STATE_INVALID,
unit_of_work.STATE_REQUESTED]}}
if since is None:
cursor = collection.find(query).sort('_id', ASCENDING)
candidates = [UnitOfWork(document) for document in cursor]
else:
candidates = []
yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, since)
query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod}
cursor = collection.find(query).sort('_id', ASCENDING)
for document in cursor:
uow = UnitOfWork(document)
if uow.process_name not in ProcessContext.CONTEXT:
# this is a decommissioned process
continue
time_qualifier = ProcessContext.get_time_qualifier(uow.process_name)
if time_qualifier == QUALIFIER_REAL_TIME:
time_qualifier = QUALIFIER_HOURLY
process_specific_since = time_helper.cast_to_time_qualifier(time_qualifier, since)
if process_specific_since <= uow.start_timeperiod:
candidates.append(uow)
if len(candidates) == 0:
raise LookupError('MongoDB has no reprocessing candidates units of work')
return candidates
def increment_time(process_name, timestamp):
""" method is used by Scheduler to define <<next>> time period.
For hourly, it is next hour: 20100101_19 -> 20100101_20
For month - next month: 201001 -> 201002, etc"""
qualifier = ProcessContext.get_time_qualifier(process_name)
pattern = define_pattern(timestamp)
t = datetime.strptime(timestamp, pattern)
if qualifier == ProcessContext.QUALIFIER_HOURLY:
t = t + timedelta(hours=1)
return t.strftime('%Y%m%d%H')
elif qualifier == ProcessContext.QUALIFIER_DAILY:
t = t + timedelta(days=1)
return t.strftime('%Y%m%d00')
elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
if t.month + 1 > 12:
new_month = 1
new_year = t.year + 1
t = t.replace(year = new_year, month = new_month)
else:
t = t.replace(month = t.month + 1)
return t.strftime('%Y%m0000')
elif qualifier == ProcessContext.QUALIFIER_YEARLY:
t = t.replace(year = t.year + 1)
return t.strftime('%Y000000')
else:
raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
def details(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if self.timeperiod is None and tree is not None:
# return list of yearly nodes OR leafs for linear tree
resp['children'] = dict()
# limit number of children to return, since linear tree can holds thousands of nodes
sorted_keys = sorted(tree.root.children.keys(), reverse=True)
sorted_keys = sorted_keys[:settings['mx_children_limit']]
for key in sorted_keys:
child = tree.root.children[key]
resp['children'][key] = TreeNodeDetails.get_details(self.logger, child)
elif tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
resp['node'] = TreeNodeDetails.get_details(self.logger, node)
resp['children'] = dict()
for key in node.children:
child = node.children[key]
resp['children'][key] = TreeNodeDetails.get_details(self.logger, child)
return resp
def _process_state_in_progress(self, process_name, job_record, start_timeperiod):
""" method that takes care of processing job records in STATE_IN_PROGRESS state"""
time_qualifier = ProcessContext.get_time_qualifier(process_name)
end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)
actual_timeperiod = time_helper.actual_timeperiod(time_qualifier)
can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record)
uow = self.uow_dao.get_one(job_record.related_unit_of_work)
if start_timeperiod == actual_timeperiod or can_finalize_job_record is False:
if uow.state in [unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED]:
# current uow has not been processed yet. update it
self.update_scope_of_processing(process_name, uow, start_timeperiod, end_timeperiod, job_record)
else:
# cls.STATE_IN_PROGRESS, cls.STATE_PROCESSED, cls.STATE_CANCELED
# create new uow to cover new inserts
self._compute_and_transfer_to_progress(process_name, start_timeperiod, end_timeperiod, job_record)
elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True:
# create new uow for FINAL RUN
self._compute_and_transfer_to_final_run(process_name, start_timeperiod, end_timeperiod, job_record)
else:
msg = 'job record %s has timeperiod from future %s vs current time %s' \
% (job_record.document['_id'], start_timeperiod, actual_timeperiod)
self._log_message(ERROR, process_name, job_record, msg)
def entries(self):
list_of_trees = []
try:
sorter_keys = sorted(context.timetable_context.keys())
for tree_name in sorter_keys:
tree_obj = self.mbean.timetable.trees[tree_name]
tree_row = list()
tree_row.append(tree_name) # index 0
tree_row.append(tree_obj.mx_page) # index 1
tree_row.append(tree_obj.mx_name) # index 2
processes = dict() # index 3
context_entry = context.timetable_context[tree_name]
for process_name in context_entry.enclosed_processes:
process_details = [process_name, # index x0
ProcessContext.get_time_qualifier(process_name), # index x1
self._state_machine_name(process_name), # index x2
ProcessContext.get_process_type(process_name), # index x3
ProcessContext.run_on_active_timeperiod(process_name), # index x4
context_entry.dependent_on, # index x5
self._list_of_dependant_trees(tree_obj)] # index x6
processes[process_name] = process_details
tree_row.append(processes)
list_of_trees.append(tree_row)
except Exception as e:
self.logger.error('MX Exception %s' % str(e), exc_info=True)
return list_of_trees
def _process_state_in_progress(self, process_name, job_record, start_timeperiod):
""" method that takes care of processing job records in STATE_IN_PROGRESS state"""
time_qualifier = ProcessContext.get_time_qualifier(process_name)
end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)
actual_timeperiod = time_helper.actual_timeperiod(time_qualifier)
can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record)
uow = self.uow_dao.get_one(job_record.related_unit_of_work)
iteration = int(uow.end_id)
try:
if start_timeperiod == actual_timeperiod or can_finalize_job_record is False:
if uow.state in [unit_of_work.STATE_REQUESTED,
unit_of_work.STATE_IN_PROGRESS,
unit_of_work.STATE_INVALID]:
# Large Job processing takes more than 1 tick of Scheduler
# Let the Large Job processing complete - do no updates to Scheduler records
pass
elif uow.state in [unit_of_work.STATE_PROCESSED,
unit_of_work.STATE_CANCELED]:
# create new uow to cover new inserts
uow = self.insert_uow(process_name, start_timeperiod, end_timeperiod, iteration + 1, job_record)
self.timetable.update_job_record(process_name, job_record, uow, job.STATE_IN_PROGRESS)
elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True:
if uow.state in [unit_of_work.STATE_REQUESTED,
unit_of_work.STATE_IN_PROGRESS,
unit_of_work.STATE_INVALID]:
# Job processing has not started yet
# Let the processing complete - do no updates to Scheduler records
msg = 'Suppressed creating uow for %s in timeperiod %s; job record is in %s; uow is in %s' \
% (process_name, job_record.timeperiod, job_record.state, uow.state)
elif uow.state == unit_of_work.STATE_PROCESSED:
self.timetable.update_job_record(process_name, job_record, uow, job.STATE_PROCESSED)
timetable_tree = self.timetable.get_tree(process_name)
timetable_tree.build_tree()
msg = 'Transferred job record %s in timeperiod %s to STATE_PROCESSED for %s' \
% (job_record.document['_id'], job_record.timeperiod, process_name)
elif uow.state == unit_of_work.STATE_CANCELED:
self.timetable.update_job_record(process_name, job_record, uow, job.STATE_SKIPPED)
msg = 'Transferred job record %s in timeperiod %s to STATE_SKIPPED for %s' \
% (job_record.document['_id'], job_record.timeperiod, process_name)
else:
msg = 'Unknown state %s for job record %s in timeperiod %s for %s' \
% (uow.state, job_record.document['_id'], job_record.timeperiod, process_name)
self._log_message(INFO, process_name, job_record, msg)
else:
msg = 'Job record %s has timeperiod from future %s vs current time %s' \
% (job_record.document['_id'], start_timeperiod, actual_timeperiod)
self._log_message(ERROR, process_name, job_record, msg)
except DuplicateKeyError as e:
uow = self.recover_from_duplicatekeyerror(e)
if uow is not None:
self.timetable.update_job_record(process_name, job_record, uow, job_record.state)
else:
msg = 'MANUAL INTERVENTION REQUIRED! Unable to identify unit_of_work for %s in %s' \
% (process_name, job_record.timeperiod)
self._log_message(ERROR, process_name, job_record, msg)
def action_get_log(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
resp['log'] = node.job_record.log
return resp
def __init__(self, tree, parent, process_name, timeperiod, job_record):
# initializes the data members
self.children = dict()
self.tree = tree
self.parent = parent
self.process_name = process_name
self.timeperiod = timeperiod
self.job_record = job_record
if parent is None and process_name is None and timeperiod is None and job_record is None:
# special case - node is TREE ROOT
self.time_qualifier = None
else:
self.time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
def action_get_log(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(
self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(
time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
resp['log'] = node.job_record.log
return resp
def _get_timetable_collection(self, process_name):
"""timetable stores timeperiod in 4 collections: hourly, daily, monthly and yearly; method looks for the
proper timetable_collection base on process TIME_QUALIFIER"""
qualifier = ProcessContext.get_time_qualifier(process_name)
if qualifier == ProcessContext.QUALIFIER_HOURLY:
collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_HOURLY)
elif qualifier == ProcessContext.QUALIFIER_DAILY:
collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_DAILY)
elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_MONTHLY)
elif qualifier == ProcessContext.QUALIFIER_YEARLY:
collection = CollectionContext.get_collection(self.logger, COLLECTION_TIMETABLE_YEARLY)
else:
raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
return collection
def action_skip(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
self.logger.info('MX (requesting skip timeperiod %r for %r) { ' % (self.timeperiod, self.process_name))
effected_nodes = node.request_skip()
for node in effected_nodes:
resp[node.timeperiod] = TreeNodeDetails.get_details(self.logger, node)
self.logger.info('}')
return resp
def synergy_to_datetime(process_name, timestamp):
""" method receives timestamp in Synergy format YYYYMMDD_HH and convert it to _naive_ datetime"""
qualifier = ProcessContext.get_time_qualifier(process_name)
if qualifier == ProcessContext.QUALIFIER_HOURLY:
date_format = SYNERGY_DATE_PATTERN
elif qualifier == ProcessContext.QUALIFIER_DAILY:
date_format = '%Y%m%d00'
elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
date_format = '%Y%m0000'
elif qualifier == ProcessContext.QUALIFIER_YEARLY:
date_format = '%Y000000'
elif qualifier == ProcessContext.QUALIFIER_REAL_TIME:
date_format = SYNERGY_SESSION_PATTERN
else:
raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
return datetime.strptime(timestamp, date_format).replace(tzinfo=None)
def _get_job_collection(self, process_name):
"""jobs are stored in 4 collections: hourly, daily, monthly and yearly;
method looks for the proper job_collection base on process TIME_QUALIFIER"""
qualifier = ProcessContext.get_time_qualifier(process_name)
if qualifier == QUALIFIER_HOURLY:
collection = self.ds.connection(COLLECTION_JOB_HOURLY)
elif qualifier == QUALIFIER_DAILY:
collection = self.ds.connection(COLLECTION_JOB_DAILY)
elif qualifier == QUALIFIER_MONTHLY:
collection = self.ds.connection(COLLECTION_JOB_MONTHLY)
elif qualifier == QUALIFIER_YEARLY:
collection = self.ds.connection(COLLECTION_JOB_YEARLY)
else:
raise ValueError('Unknown time qualifier: %s for %s' % (qualifier, process_name))
return collection
def datetime_to_synergy(process_name, dt):
""" method parses datetime and returns Synergy Date"""
date_format = None
qualifier = ProcessContext.get_time_qualifier(process_name)
if qualifier == ProcessContext.QUALIFIER_HOURLY:
date_format = SYNERGY_DATE_PATTERN
elif qualifier == ProcessContext.QUALIFIER_DAILY:
date_format = '%Y%m%d00'
elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
date_format = '%Y%m0000'
elif qualifier == ProcessContext.QUALIFIER_YEARLY:
date_format = '%Y000000'
elif qualifier == ProcessContext.QUALIFIER_REAL_TIME:
date_format = SYNERGY_SESSION_PATTERN
else:
raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
return dt.strftime(date_format)
def get_reprocessing_candidates(self, since=None):
""" method queries Unit Of Work whose <start_timeperiod> is younger than <since>
and who could be candidates for re-processing """
collection = self.ds.connection(COLLECTION_UNIT_OF_WORK)
query = {
unit_of_work.STATE: {
'$in': [
unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID,
unit_of_work.STATE_REQUESTED
]
}
}
if since is None:
cursor = collection.find(query).sort('_id', ASCENDING)
candidates = [UnitOfWork(document) for document in cursor]
else:
candidates = []
yearly_timeperiod = time_helper.cast_to_time_qualifier(
QUALIFIER_YEARLY, since)
query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod}
cursor = collection.find(query).sort('_id', ASCENDING)
for document in cursor:
uow = UnitOfWork(document)
if uow.process_name not in ProcessContext.CONTEXT:
# this is a decommissioned process
continue
time_qualifier = ProcessContext.get_time_qualifier(
uow.process_name)
if time_qualifier == QUALIFIER_REAL_TIME:
time_qualifier = QUALIFIER_HOURLY
process_specific_since = time_helper.cast_to_time_qualifier(
time_qualifier, since)
if process_specific_since <= uow.start_timeperiod:
candidates.append(uow)
if len(candidates) == 0:
raise LookupError(
'MongoDB has no reprocessing candidates units of work')
return candidates
def _process_state_embryo(self, process_name, job_record, start_timeperiod):
""" method that takes care of processing job records in STATE_EMBRYO state"""
time_qualifier = ProcessContext.get_time_qualifier(process_name)
end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)
try:
uow = self.insert_uow(process_name, start_timeperiod, end_timeperiod, 0, job_record)
except DuplicateKeyError as e:
msg = 'Catching up with latest unit_of_work %s in timeperiod %s, because of: %r' \
% (process_name, job_record.timeperiod, e)
self._log_message(WARNING, process_name, job_record, msg)
uow = self.recover_from_duplicatekeyerror(e)
if uow is not None:
self.timetable.update_job_record(process_name, job_record, uow, job.STATE_IN_PROGRESS)
else:
msg = 'MANUAL INTERVENTION REQUIRED! Unable to locate unit_of_work for %s in %s' \
% (process_name, job_record.timeperiod)
self._log_message(WARNING, process_name, job_record, msg)
def action_get_uow(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
uow_id = node.job_record.related_unit_of_work
if uow_id is None:
resp = {'response': 'no related unit_of_work'}
else:
resp = self.uow_dao.get_one(uow_id).document
for key in resp:
resp[key] = str(resp[key])
return resp
def action_skip(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(
self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(
time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
self.logger.info('MX (requesting skip timeperiod %r for %r) { ' %
(self.timeperiod, self.process_name))
effected_nodes = node.request_skip()
for node in effected_nodes:
resp[node.timeperiod] = TreeNodeDetails.get_details(
self.logger, node)
self.logger.info('}')
return resp
def _get_nodes_details(cls, logger, node):
"""method returns {
process_name : string,
timestamp : string,
number_of_children : integer,
number_of_failed_calls : integer,
state : STATE_SKIPPED, STATE_IN_PROGRESS, STATE_PROCESSED, STATE_FINAL_RUN, STATE_EMBRYO
}
"""
description = dict()
try:
description['process_name'] = node.process_name
description['time_qualifier'] = ProcessContext.get_time_qualifier(node.process_name)
description['number_of_children'] = len(node.children)
description['number_of_failed_calls'] = node.time_record.get_number_of_failures()
description['timestamp'] = node.time_record.get_timestamp()
description['state'] = node.time_record.get_state()
except Exception as e:
logger.error('MX Exception: ' + str(e), exc_info=True)
finally:
return description
def action_get_uow(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(
self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(
time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
uow_id = node.job_record.related_unit_of_work
if uow_id is None:
resp = {'response': 'no related unit_of_work'}
else:
resp = self.uow_dao.get_one(uow_id).document
for key in resp:
resp[key] = str(resp[key])
return resp
def fire_managed_worker(self, *args):
"""requests vertical aggregator (hourly site, daily variant, etc) to start up"""
try:
process_name = args[0]
scheduler_entry_obj = args[1]
self.logger.info('%s {' % process_name)
timetable_record = self.timetable.get_next_job_record(process_name)
pipeline = self.pipelines[scheduler_entry_obj.state_machine_name]
run_on_active_timeperiod = ProcessContext.run_on_active_timeperiod(scheduler_entry_obj.process_name)
if not run_on_active_timeperiod:
time_qualifier = ProcessContext.get_time_qualifier(process_name)
incremented_timeperiod = time_helper.increment_timeperiod(time_qualifier, timetable_record.timeperiod)
dt_record_timestamp = time_helper.synergy_to_datetime(time_qualifier, incremented_timeperiod)
dt_record_timestamp += timedelta(minutes=LAG_5_MINUTES)
if datetime.utcnow() <= dt_record_timestamp:
self.logger.info('Timetable record %s for timeperiod %s will not be triggered until %s.'
% (timetable_record.document['_id'],
timetable_record.timeperiod,
dt_record_timestamp.strftime('%Y-%m-%d %H:%M:%S')))
return
process_type = ProcessContext.get_process_type(scheduler_entry_obj.process_name)
if process_type == TYPE_BLOCKING_DEPENDENCIES:
pipeline.manage_pipeline_with_blocking_dependencies(process_name, timetable_record)
elif process_type == TYPE_BLOCKING_CHILDREN:
pipeline.manage_pipeline_with_blocking_children(process_name, timetable_record)
elif process_type == TYPE_MANAGED:
pipeline.manage_pipeline_for_process(process_name, timetable_record)
except (AMQPError, IOError) as e:
self.logger.error('AMQPError: %s' % str(e), exc_info=True)
self.publishers.reset_all(suppress_logging=True)
except Exception as e:
self.logger.error('Exception: %s' % str(e), exc_info=True)
finally:
self.logger.info('}')
def match_time_qualifier(actual_process_name, candidate_process_name):
if candidate_process_name is None:
return False
time_qualifier = ProcessContext.get_time_qualifier(actual_process_name)
candidate_qualifier = ProcessContext.get_time_qualifier(candidate_process_name)
return time_qualifier == candidate_qualifier
def match_time_qualifier(time_qualifier, candidate_process_name):
""" :return: True if candidate_process has the same time qualifier as given """
if candidate_process_name is None:
return False
candidate_qualifier = ProcessContext.get_time_qualifier(candidate_process_name)
return time_qualifier == candidate_qualifier
def _process_state_embryo(self, process_name, job_record, start_timeperiod):
""" method that takes care of processing job records in STATE_EMBRYO state"""
time_qualifier = ProcessContext.get_time_qualifier(process_name)
end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)
self._compute_and_transfer_to_progress(process_name, start_timeperiod, end_timeperiod, job_record)
