def stop_process(options):
"""Stop the synergy-data daemons"""
import logging
from supervisor import supervisor_helper as helper
from model.box_configuration_entry import BoxConfigurationEntry
if options.app is not None and options.app != process_context.PROCESS_SUPERVISOR:
# mark individual process for termination
# real work is performed by Supervisor
if options.app not in PROCESSES_FOR_XXL:
sys.stdout.write("ERROR: requested process must be withing allowed list of: %r \n" % PROCESSES_FOR_XXL)
sys.exit(1)
box_id = helper.get_box_id(logging)
box_configuration = helper.retrieve_configuration(logging, box_id)
box_configuration.set_process_state(options.app, BoxConfigurationEntry.STATE_OFF)
helper.update_configuration(logging, box_configuration)
else:
# stop Supervisor
try:
pid = _get_supervisor_pid()
if pid is None:
message = "ERROR: Can not find Supervisor pidfile. Supervisor not running?\n"
sys.stderr.write(message)
sys.exit(1)
# Try killing the daemon process
sys.stdout.write("INFO: Killing %r \n" % process_context.PROCESS_SUPERVISOR)
while 1:
os.kill(pid, signal.SIGTERM)
time.sleep(0.1)
ProcessContext.remove_pid_file(process_context.PROCESS_SUPERVISOR)
except Exception as e:
sys.stderr.write("Exception on killing %s : %s \n" % (process_context.PROCESS_SUPERVISOR, str(e)))
sys.exit(0)
def create_unit_of_work(process_name,
start_id,
end_id,
timeperiod='INVALID_TIMEPERIOD',
state=unit_of_work.STATE_REQUESTED,
creation_at=datetime.utcnow(),
uow_id=None):
""" method creates and returns unit_of_work """
try:
source_collection = ProcessContext.get_source(process_name)
target_collection = ProcessContext.get_sink(process_name)
except KeyError:
source_collection = None
target_collection = None
uow = UnitOfWork()
uow.timeperiod = timeperiod
uow.start_timeperiod = timeperiod
uow.end_timeperiod = timeperiod
uow.start_id = start_id
uow.end_id = end_id
uow.source = source_collection
uow.sink = target_collection
uow.state = state
uow.created_at = creation_at
uow.process_name = process_name
uow.number_of_retries = 0
if uow_id is not None:
uow.document['_id'] = uow_id
return uow
def compute_scope_of_processing(self, process_name, start_timeperiod, end_timeperiod, job_record):
"""method reads collection and identify slice for processing"""
source_collection_name = ProcessContext.get_source(process_name)
target_collection_name = ProcessContext.get_sink(process_name)
start_id = self.ds.highest_primary_key(source_collection_name, start_timeperiod, end_timeperiod)
end_id = self.ds.lowest_primary_key(source_collection_name, start_timeperiod, end_timeperiod)
uow = UnitOfWork()
uow.timeperiod = start_timeperiod
uow.start_id = str(start_id)
uow.end_id = str(end_id)
uow.start_timeperiod = start_timeperiod
uow.end_timeperiod = end_timeperiod
uow.created_at = datetime.utcnow()
uow.source = source_collection_name
uow.sink = target_collection_name
uow.state = unit_of_work.STATE_REQUESTED
uow.process_name = process_name
uow.number_of_retries = 0
uow_id = self.uow_dao.insert(uow)
mq_request = WorkerMqRequest()
mq_request.process_name = process_name
mq_request.unit_of_work_id = uow_id
publisher = self.publishers.get(process_name)
publisher.publish(mq_request.document)
publisher.release()
msg = 'Published: UOW %r for %r in timeperiod %r.' % (uow_id, process_name, start_timeperiod)
self._log_message(INFO, process_name, job_record, msg)
return uow
def entries(self):
list_of_trees = []
try:
sorter_keys = sorted(context.timetable_context.keys())
for tree_name in sorter_keys:
tree_obj = self.mbean.timetable.trees[tree_name]
tree_row = list()
tree_row.append(tree_name) # index 0
tree_row.append(tree_obj.mx_page) # index 1
tree_row.append(tree_obj.mx_name) # index 2
processes = dict() # index 3
context_entry = context.timetable_context[tree_name]
for process_name in context_entry.enclosed_processes:
process_details = [process_name, # index x0
ProcessContext.get_time_qualifier(process_name), # index x1
self._state_machine_name(process_name), # index x2
ProcessContext.get_process_type(process_name), # index x3
ProcessContext.run_on_active_timeperiod(process_name), # index x4
context_entry.dependent_on, # index x5
self._list_of_dependant_trees(tree_obj)] # index x6
processes[process_name] = process_details
tree_row.append(processes)
list_of_trees.append(tree_row)
except Exception as e:
self.logger.error('MX Exception %s' % str(e), exc_info=True)
return list_of_trees
def test_double_initialization(self):
try:
ProcessContext.set_current_process(PROCESS_GC)
ProcessContext.set_current_process(PROCESS_SUPERVISOR)
self.assertTrue(False, 'set_current_process should not allow double initialization')
except AttributeError:
self.assertTrue(True)
def __init__(self, process_name):
""" renames process to SynergyYYY and creates PID file """
self.process_name = process_name
self.logger = ProcessContext.get_logger(process_name)
# process-related activities
setproctitle.setproctitle(settings['process_prefix'] + self.process_name)
ProcessContext.create_pid_file(self.process_name)
def run(process_name):
"""
You should override this method when you subclass Daemon. It will be
called after the process has been daemonized by start() or restart().
"""
sys.stdout.write('INFO: Starting %s \n' % ProcessContext.get_classname(process_name))
klass = get_class(ProcessContext.get_classname(process_name))
instance = klass(process_name)
instance.start()
def __init__(self, process_name, process_id=None):
""" renames process to SynergyYYY and creates PID file """
self.process_name = process_name
self.process_id = process_id
self.logger = ProcessContext.get_logger(process_name, process_id=self.process_id)
# process-related activities
process_title = settings['process_prefix'] + self.process_name
if self.process_id:
process_title += str(self.process_id)
setproctitle.setproctitle(process_title)
ProcessContext.create_pid_file(self.process_name, process_id=self.process_id)
def __init__(self, name):
self.name = name
if name in ProcessContext.CONTEXT:
self.routing_key = ProcessContext.get_routing(name)
self.exchange = ProcessContext.get_exchange(name)
self.queue = ProcessContext.get_queue(name)
elif name in MqQueueContext.CONTEXT:
self.routing_key = MqQueueContext.get_routing(name)
self.exchange = MqQueueContext.get_exchange(name)
self.queue = name
else:
raise ValueError(
'Unknown name %s. Unable to retrieve amqp settings.' % name)
def __init__(self, process_name, process_id=None):
""" renames process to SynergyYYY and creates PID file """
self.process_name = process_name
self.process_id = process_id
self.logger = ProcessContext.get_logger(process_name,
process_id=self.process_id)
# process-related activities
process_title = settings['process_prefix'] + self.process_name
if self.process_id:
process_title += str(self.process_id)
setproctitle.setproctitle(process_title)
ProcessContext.create_pid_file(self.process_name,
process_id=self.process_id)
def kill_process(process_name):
""" method is called to kill a running process """
try:
sys.stdout.write('killing: %s { \n' % process_name)
pid = get_process_pid(process_name)
if pid is not None and psutil.pid_exists(int(pid)):
p = psutil.Process(pid)
p.kill()
p.wait()
ProcessContext.remove_pid_file(process_name)
except Exception as e:
sys.stderr.write('Exception on killing %s : %s \n' % (process_name, str(e)))
finally:
sys.stdout.write('}')
def kill_process(process_name):
""" method is called to kill a running process """
try:
sys.stdout.write('killing: {0} {{ \n'.format(process_name))
pid = get_process_pid(process_name)
if pid is not None and psutil.pid_exists(int(pid)):
p = psutil.Process(pid)
p.kill()
p.wait()
ProcessContext.remove_pid_file(process_name)
except Exception as e:
sys.stderr.write('Exception on killing {0} : {1} \n'.format(
process_name, e))
finally:
sys.stdout.write('}')
def query_configuration(options):
""" Queries process state """
from system import process_helper
if not options.supervisor:
# reads status of one process only
process_helper.poll_process(options.app)
else:
# reads current box configuration and prints it to the console
from db.dao.box_configuration_dao import BoxConfigurationDao
from supervisor import supervisor_helper as helper
from system.process_context import ProcessContext
from constants import PROCESS_LAUNCH_PY
logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY)
box_id = helper.get_box_id(logger)
bc_dao = BoxConfigurationDao(logger)
sys.stdout.write('\nConfiguration for BOX_ID=%r:\n' % box_id)
box_configuration = bc_dao.get_one(box_id)
process_list = box_configuration.get_process_list()
i = 1
for process in process_list:
sys.stdout.write('%d\t%r:%r \n' % (i, process, process_list[process]))
i += 1
sys.stdout.write('\n')
def _start_process(self, start_timeperiod, end_timeperiod, arguments):
try:
start_dt = time_helper.synergy_to_datetime(QUALIFIER_HOURLY, start_timeperiod)
sqoop_slice_starttime = start_dt.strftime(SqoopDriver.SQOOP_DATE_FORMAT)
end_dt = time_helper.synergy_to_datetime(QUALIFIER_HOURLY, end_timeperiod)
sqoop_slice_endtime = end_dt.strftime(SqoopDriver.SQOOP_DATE_FORMAT)
sink_path = ProcessContext.get_sink(self.process_name)
self.logger.info('start: %s {' % self.process_name)
p = psutil.Popen([settings['bash_shell'],
settings['sqoop_command'],
str(sqoop_slice_starttime),
str(sqoop_slice_endtime),
sink_path + '/' + start_timeperiod],
close_fds=True,
cwd=settings['process_cwd'],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
self.cli_process = p
self.logger.info('Started %s with pid = %r' % (self.process_name, p.pid))
except Exception:
self.logger.error('Exception on starting: %s' % self.process_name, exc_info=True)
finally:
self.logger.info('}')
def get_reprocessing_candidates(self, since=None):
""" method queries Unit Of Work whose <start_timeperiod> is younger than <since>
and who could be candidates for re-processing """
collection = self.ds.connection(COLLECTION_UNIT_OF_WORK)
query = {unit_of_work.STATE: {'$in': [unit_of_work.STATE_IN_PROGRESS,
unit_of_work.STATE_INVALID,
unit_of_work.STATE_REQUESTED]}}
if since is None:
cursor = collection.find(query).sort('_id', ASCENDING)
candidates = [UnitOfWork(document) for document in cursor]
else:
candidates = []
yearly_timeperiod = time_helper.cast_to_time_qualifier(QUALIFIER_YEARLY, since)
query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod}
cursor = collection.find(query).sort('_id', ASCENDING)
for document in cursor:
uow = UnitOfWork(document)
if uow.process_name not in ProcessContext.CONTEXT:
# this is a decommissioned process
continue
time_qualifier = ProcessContext.get_time_qualifier(uow.process_name)
if time_qualifier == QUALIFIER_REAL_TIME:
time_qualifier = QUALIFIER_HOURLY
process_specific_since = time_helper.cast_to_time_qualifier(time_qualifier, since)
if process_specific_since <= uow.start_timeperiod:
candidates.append(uow)
if len(candidates) == 0:
raise LookupError('MongoDB has no reprocessing candidates units of work')
return candidates
def cast_to_time_qualifier(process_name, timestamp):
""" method is used to cast synergy_time accordingly to process time qualifier.
For example: for QUALIFIER_HOURLY, it can be either 20100101_19 or 20100101_193412 """
date_format = None
qualifier = ProcessContext.get_time_qualifier(process_name)
if qualifier == ProcessContext.QUALIFIER_HOURLY:
if len(timestamp) > 10:
t = datetime.strptime(timestamp, SYNERGY_SESSION_PATTERN)
return t.strftime(SYNERGY_DATE_PATTERN)
else:
return timestamp
elif qualifier == ProcessContext.QUALIFIER_DAILY :
date_format = '%Y%m%d00'
elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
date_format = '%Y%m0000'
elif qualifier == ProcessContext.QUALIFIER_YEARLY:
date_format = '%Y000000'
pattern = define_pattern(timestamp)
t = datetime.strptime(timestamp, pattern)
if date_format is not None:
return t.strftime(date_format)
else:
raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
def stop_process(options):
"""Stop specific daemon"""
from system import process_helper
from supervisor import supervisor_helper as helper
from system.process_context import ProcessContext
from supervisor.supervisor_constants import PROCESS_SUPERVISOR
from constants import PROCESS_LAUNCH_PY
logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY)
box_id = helper.get_box_id(logger)
if options.supervisor is True and options.app != PROCESS_SUPERVISOR:
from db.model import box_configuration
from db.dao.box_configuration_dao import BoxConfigurationDao
message = 'INFO: Marking %r to be managed by Supervisor \n' % options.app
sys.stdout.write(message)
bc_dao = BoxConfigurationDao(logger)
box_config = bc_dao.get_one(box_id)
box_config.set_process_state(options.app, box_configuration.STATE_OFF)
bc_dao.update(box_config)
return
try:
pid = process_helper.get_process_pid(options.app)
if pid is None or process_helper.poll_process(options.app) is False:
message = 'ERROR: Process %r is already terminated %r\n' % (
options.app, pid)
sys.stderr.write(message)
sys.exit(1)
process_helper.kill_process(options.app)
except Exception as e:
sys.stderr.write('Exception on killing %s : %s \n' %
(options.app, str(e)))
traceback.print_exc(file=sys.stderr)
def _load_managed_entries(self):
""" loads scheduler managed entries. no start-up procedures are performed """
scheduler_entries = self.se_managed_dao.get_all()
for scheduler_entry_obj in scheduler_entries:
process_name = scheduler_entry_obj.process_name
if scheduler_entry_obj.process_name not in ProcessContext.CONTEXT:
self.logger.error('Process %r is not known to the system. Skipping it.' % process_name)
continue
process_type = ProcessContext.get_process_type(process_name)
if process_type in [TYPE_BLOCKING_DEPENDENCIES, TYPE_BLOCKING_CHILDREN, TYPE_MANAGED]:
function = self.fire_managed_worker
handler_type = TYPE_MANAGED
elif process_type == TYPE_GARBAGE_COLLECTOR:
function = self.fire_garbage_collector
handler_type = TYPE_MANAGED
elif process_type == TYPE_FREERUN:
self.logger.error('TYPE_FREERUN process %s was found in scheduler_managed_entry table. '
'Move the process to the scheduler_freerun_entry table. Skipping the process.'
% process_type)
continue
else:
self.logger.error('Process type %s is not known to the system. Skipping it.' % process_type)
continue
self._activate_handler(scheduler_entry_obj, process_name, 'NA', function, handler_type)
def __init__(self, process_name):
"""@param process_name: id of the process, the worker will be performing """
super(AbstractMqWorker, self).__init__(process_name)
self.queue_source = ProcessContext.get_source(self.process_name)
self.queue_sink = ProcessContext.get_sink(self.process_name)
self.consumer = None
self._init_mq_consumer()
self.main_thread = None
self.performance_ticker = None
self._init_performance_ticker(self.logger)
msg_suffix = 'in Production Mode'
if settings['under_test']:
msg_suffix = 'in Testing Mode'
self.logger.info('Started %s %s' % (self.process_name, msg_suffix))
def query_configuration(options):
""" Queries process state """
from system import process_helper
if not options.supervisor:
# reads status of one process only
process_helper.poll_process(options.app)
else:
# reads current box configuration and prints it to the console
from db.dao.box_configuration_dao import BoxConfigurationDao
from supervisor import supervisor_helper as helper
from system.process_context import ProcessContext
from constants import PROCESS_LAUNCH_PY
logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY)
box_id = helper.get_box_id(logger)
bc_dao = BoxConfigurationDao(logger)
sys.stdout.write('\nConfiguration for BOX_ID=%r:\n' % box_id)
box_configuration = bc_dao.get_one(box_id)
process_list = box_configuration.get_process_list()
i = 1
for process in process_list:
sys.stdout.write('%d\t%r:%r \n' %
(i, process, process_list[process]))
i += 1
sys.stdout.write('\n')
def _kill_process(self, box_config, process_name):
""" method is called to kill a running process """
try:
self.logger.info('kill: %s {' % process_name)
pid = box_config.get_process_pid(process_name)
if pid is not None and psutil.pid_exists(int(pid)):
p = psutil.Process(pid)
p.kill()
p.wait()
box_config.set_process_pid(process_name, None)
self.bc_dao.update(box_config)
ProcessContext.remove_pid_file(process_name)
except Exception:
self.logger.error('Exception on killing: %s' % process_name, exc_info=True)
finally:
self.logger.info('}')
def details(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if self.timeperiod is None and tree is not None:
# return list of yearly nodes OR leafs for linear tree
resp['children'] = dict()
# limit number of children to return, since linear tree can holds thousands of nodes
sorted_keys = sorted(tree.root.children.keys(), reverse=True)
sorted_keys = sorted_keys[:settings['mx_children_limit']]
for key in sorted_keys:
child = tree.root.children[key]
resp['children'][key] = TreeNodeDetails.get_details(self.logger, child)
elif tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
resp['node'] = TreeNodeDetails.get_details(self.logger, node)
resp['children'] = dict()
for key in node.children:
child = node.children[key]
resp['children'][key] = TreeNodeDetails.get_details(self.logger, child)
return resp
def _process_state_in_progress(self, process_name, job_record, start_timeperiod):
""" method that takes care of processing job records in STATE_IN_PROGRESS state"""
time_qualifier = ProcessContext.get_time_qualifier(process_name)
end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)
actual_timeperiod = time_helper.actual_timeperiod(time_qualifier)
can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record)
uow = self.uow_dao.get_one(job_record.related_unit_of_work)
if start_timeperiod == actual_timeperiod or can_finalize_job_record is False:
if uow.state in [unit_of_work.STATE_INVALID, unit_of_work.STATE_REQUESTED]:
# current uow has not been processed yet. update it
self.update_scope_of_processing(process_name, uow, start_timeperiod, end_timeperiod, job_record)
else:
# cls.STATE_IN_PROGRESS, cls.STATE_PROCESSED, cls.STATE_CANCELED
# create new uow to cover new inserts
self._compute_and_transfer_to_progress(process_name, start_timeperiod, end_timeperiod, job_record)
elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True:
# create new uow for FINAL RUN
self._compute_and_transfer_to_final_run(process_name, start_timeperiod, end_timeperiod, job_record)
else:
msg = 'job record %s has timeperiod from future %s vs current time %s' \
% (job_record.document['_id'], start_timeperiod, actual_timeperiod)
self._log_message(ERROR, process_name, job_record, msg)
def stop_process(options):
"""Stop specific daemon"""
from system import process_helper
from supervisor import supervisor_helper as helper
from system.process_context import ProcessContext
from supervisor.supervisor_constants import PROCESS_SUPERVISOR
from constants import PROCESS_LAUNCH_PY
logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY)
box_id = helper.get_box_id(logger)
if options.supervisor is True and options.app != PROCESS_SUPERVISOR:
from db.model import box_configuration
from db.dao.box_configuration_dao import BoxConfigurationDao
message = 'INFO: Marking %r to be managed by Supervisor \n' % options.app
sys.stdout.write(message)
bc_dao = BoxConfigurationDao(logger)
box_config = bc_dao.get_one(box_id)
box_config.set_process_state(options.app, box_configuration.STATE_OFF)
bc_dao.update(box_config)
return
try:
pid = process_helper.get_process_pid(options.app)
if pid is None or process_helper.poll_process(options.app) is False:
message = 'ERROR: Process %r is already terminated %r\n' % (options.app, pid)
sys.stderr.write(message)
sys.exit(1)
process_helper.kill_process(options.app)
except Exception as e:
sys.stderr.write('Exception on killing %s : %s \n' % (options.app, str(e)))
traceback.print_exc(file=sys.stderr)
def increment_time(process_name, timestamp):
""" method is used by Scheduler to define <<next>> time period.
For hourly, it is next hour: 20100101_19 -> 20100101_20
For month - next month: 201001 -> 201002, etc"""
qualifier = ProcessContext.get_time_qualifier(process_name)
pattern = define_pattern(timestamp)
t = datetime.strptime(timestamp, pattern)
if qualifier == ProcessContext.QUALIFIER_HOURLY:
t = t + timedelta(hours=1)
return t.strftime('%Y%m%d%H')
elif qualifier == ProcessContext.QUALIFIER_DAILY:
t = t + timedelta(days=1)
return t.strftime('%Y%m%d00')
elif qualifier == ProcessContext.QUALIFIER_MONTHLY:
if t.month + 1 > 12:
new_month = 1
new_year = t.year + 1
t = t.replace(year = new_year, month = new_month)
else:
t = t.replace(month = t.month + 1)
return t.strftime('%Y%m0000')
elif qualifier == ProcessContext.QUALIFIER_YEARLY:
t = t.replace(year = t.year + 1)
return t.strftime('%Y000000')
else:
raise ValueError('unknown time qualifier: %s for %s' % (qualifier, process_name))
def _process_state_in_progress(self, process_name, job_record, start_timeperiod):
""" method that takes care of processing job records in STATE_IN_PROGRESS state"""
time_qualifier = ProcessContext.get_time_qualifier(process_name)
end_timeperiod = time_helper.increment_timeperiod(time_qualifier, start_timeperiod)
actual_timeperiod = time_helper.actual_timeperiod(time_qualifier)
can_finalize_job_record = self.timetable.can_finalize_job_record(process_name, job_record)
uow = self.uow_dao.get_one(job_record.related_unit_of_work)
iteration = int(uow.end_id)
try:
if start_timeperiod == actual_timeperiod or can_finalize_job_record is False:
if uow.state in [unit_of_work.STATE_REQUESTED,
unit_of_work.STATE_IN_PROGRESS,
unit_of_work.STATE_INVALID]:
# Large Job processing takes more than 1 tick of Scheduler
# Let the Large Job processing complete - do no updates to Scheduler records
pass
elif uow.state in [unit_of_work.STATE_PROCESSED,
unit_of_work.STATE_CANCELED]:
# create new uow to cover new inserts
uow = self.insert_uow(process_name, start_timeperiod, end_timeperiod, iteration + 1, job_record)
self.timetable.update_job_record(process_name, job_record, uow, job.STATE_IN_PROGRESS)
elif start_timeperiod < actual_timeperiod and can_finalize_job_record is True:
if uow.state in [unit_of_work.STATE_REQUESTED,
unit_of_work.STATE_IN_PROGRESS,
unit_of_work.STATE_INVALID]:
# Job processing has not started yet
# Let the processing complete - do no updates to Scheduler records
msg = 'Suppressed creating uow for %s in timeperiod %s; job record is in %s; uow is in %s' \
% (process_name, job_record.timeperiod, job_record.state, uow.state)
elif uow.state == unit_of_work.STATE_PROCESSED:
self.timetable.update_job_record(process_name, job_record, uow, job.STATE_PROCESSED)
timetable_tree = self.timetable.get_tree(process_name)
timetable_tree.build_tree()
msg = 'Transferred job record %s in timeperiod %s to STATE_PROCESSED for %s' \
% (job_record.document['_id'], job_record.timeperiod, process_name)
elif uow.state == unit_of_work.STATE_CANCELED:
self.timetable.update_job_record(process_name, job_record, uow, job.STATE_SKIPPED)
msg = 'Transferred job record %s in timeperiod %s to STATE_SKIPPED for %s' \
% (job_record.document['_id'], job_record.timeperiod, process_name)
else:
msg = 'Unknown state %s for job record %s in timeperiod %s for %s' \
% (uow.state, job_record.document['_id'], job_record.timeperiod, process_name)
self._log_message(INFO, process_name, job_record, msg)
else:
msg = 'Job record %s has timeperiod from future %s vs current time %s' \
% (job_record.document['_id'], start_timeperiod, actual_timeperiod)
self._log_message(ERROR, process_name, job_record, msg)
except DuplicateKeyError as e:
uow = self.recover_from_duplicatekeyerror(e)
if uow is not None:
self.timetable.update_job_record(process_name, job_record, uow, job_record.state)
else:
msg = 'MANUAL INTERVENTION REQUIRED! Unable to identify unit_of_work for %s in %s' \
% (process_name, job_record.timeperiod)
self._log_message(ERROR, process_name, job_record, msg)
def _get_nodes_details(self, tree):
timetable = self.mbean.timetable
description = dict()
description['reprocessing_queues'] = dict()
description['processes'] = dict()
description['next_timeperiods'] = dict()
try:
if isinstance(tree, FourLevelTree):
description['number_of_levels'] = 4
description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly)
description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly)
description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily)
description['reprocessing_queues']['hourly'] = self._get_reprocessing_details(tree.process_hourly)
description['processes']['yearly'] = tree.process_yearly
description['processes']['monthly'] = tree.process_monthly
description['processes']['daily'] = tree.process_daily
description['processes']['hourly'] = tree.process_hourly
description['next_timeperiods']['yearly'] = timetable.get_next_job_record(tree.process_yearly).timeperiod
description['next_timeperiods']['monthly'] = timetable.get_next_job_record(tree.process_monthly).timeperiod
description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_daily).timeperiod
description['next_timeperiods']['hourly'] = timetable.get_next_job_record(tree.process_hourly).timeperiod
description['type'] = ProcessContext.get_process_type(tree.process_yearly)
elif isinstance(tree, ThreeLevelTree):
description['number_of_levels'] = 3
description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly)
description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly)
description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily)
description['processes']['yearly'] = tree.process_yearly
description['processes']['monthly'] = tree.process_monthly
description['processes']['daily'] = tree.process_daily
description['next_timeperiods']['yearly'] = timetable.get_next_job_record(tree.process_yearly).timeperiod
description['next_timeperiods']['monthly'] = timetable.get_next_job_record(tree.process_monthly).timeperiod
description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_daily).timeperiod
description['type'] = ProcessContext.get_process_type(tree.process_yearly)
elif isinstance(tree, TwoLevelTree):
description['number_of_levels'] = 1
description['reprocessing_queues']['linear'] = self._get_reprocessing_details(tree.process_name)
description['processes']['linear'] = tree.process_name
description['next_timeperiods']['daily'] = timetable.get_next_job_record(tree.process_name).timeperiod
description['type'] = ProcessContext.get_process_type(tree.process_name)
else:
raise ValueError('Tree type %s has no support within MX module.' % type(tree).__name__)
except Exception as e:
self.logger.error('MX Exception: ' + str(e), exc_info=True)
finally:
return description
def _get_nodes_details(self, tree):
timetable = self.mbean.timetable
description = dict()
description['reprocessing_queues'] = dict()
description['processes'] = dict()
description['next_timeperiods'] = dict()
try:
# workaround for importing "scheduler" package
if type(tree).__name__ == 'FourLevelTree':
description['number_of_levels'] = 4
description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly)
description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly)
description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily)
description['reprocessing_queues']['hourly'] = self._get_reprocessing_details(tree.process_hourly)
description['processes']['yearly'] = tree.process_yearly
description['processes']['monthly'] = tree.process_monthly
description['processes']['daily'] = tree.process_daily
description['processes']['hourly'] = tree.process_hourly
description['next_timeperiods']['yearly'] = timetable.get_next_timetable_record(tree.process_yearly).get_timestamp()
description['next_timeperiods']['monthly'] = timetable.get_next_timetable_record(tree.process_monthly).get_timestamp()
description['next_timeperiods']['daily'] = timetable.get_next_timetable_record(tree.process_daily).get_timestamp()
description['next_timeperiods']['hourly'] = timetable.get_next_timetable_record(tree.process_hourly).get_timestamp()
description['type'] = ProcessContext.get_type(tree.process_yearly)
elif type(tree).__name__ == 'ThreeLevelTree':
description['number_of_levels'] = 3
description['reprocessing_queues']['yearly'] = self._get_reprocessing_details(tree.process_yearly)
description['reprocessing_queues']['monthly'] = self._get_reprocessing_details(tree.process_monthly)
description['reprocessing_queues']['daily'] = self._get_reprocessing_details(tree.process_daily)
description['processes']['yearly'] = tree.process_yearly
description['processes']['monthly'] = tree.process_monthly
description['processes']['daily'] = tree.process_daily
description['next_timeperiods']['yearly'] = timetable.get_next_timetable_record(tree.process_yearly).get_timestamp()
description['next_timeperiods']['monthly'] = timetable.get_next_timetable_record(tree.process_monthly).get_timestamp()
description['next_timeperiods']['daily'] = timetable.get_next_timetable_record(tree.process_daily).get_timestamp()
description['type'] = ProcessContext.get_type(tree.process_yearly)
elif type(tree).__name__ == 'TwoLevelTree':
description['number_of_levels'] = 1
description['reprocessing_queues']['linear'] = self._get_reprocessing_details(tree.process_name)
description['processes']['linear'] = tree.process_name
description['next_timeperiods']['daily'] = timetable.get_next_timetable_record(tree.process_name).get_timestamp()
description['type'] = ProcessContext.get_type(tree.process_name)
except Exception as e:
self.logger.error('MX Exception: ' + str(e), exc_info=True)
finally:
return description
def get_process_pid(process_name):
""" check for process' pid file and returns pid from there """
try:
pid_filename = ProcessContext.get_pid_filename(process_name)
with open(pid_filename, mode='r') as pid_file:
pid = int(pid_file.read().strip())
except IOError:
pid = None
return pid
def get_process_pid(process_name):
""" check for process' pid file and returns pid from there """
try:
pid_filename = ProcessContext.get_pid_filename(process_name)
with open(pid_filename, mode='r') as pid_file:
pid = int(pid_file.read().strip())
except IOError:
pid = None
return pid
def create_and_insert_unit_of_work(process_name, start_id, end_id, state=unit_of_work.STATE_REQUESTED,
timeperiod='INVALID_TIMEPERIOD'):
""" method creates and inserts a unit_of_work into DB
:return id of the created object in the db"""
uow = create_unit_of_work(process_name, start_id, end_id, timeperiod, state)
logger = ProcessContext.get_logger(process_name)
uow_dao = UnitOfWorkDao(logger)
uow_id = uow_dao.insert(uow)
return uow_id
def create_unit_of_work(process_name, first_object_id, last_object_id):
""" method is used to insert unit_of_work """
source_collection = ProcessContext.get_source_collection(process_name)
target_collection = ProcessContext.get_target_collection(process_name)
logger = ProcessContext.get_logger(process_name)
unit_of_work = UnitOfWorkEntry()
unit_of_work.set_timestamp('UNIT_TEST')
unit_of_work.set_start_id(first_object_id)
unit_of_work.set_end_id(last_object_id)
unit_of_work.set_source_collection(source_collection)
unit_of_work.set_target_collection(target_collection)
unit_of_work.set_state(UnitOfWorkEntry.STATE_REQUESTED)
unit_of_work.set_process_name(process_name)
unit_of_work.set_number_of_retries(0)
uow_id = unit_of_work_helper.insert(logger, unit_of_work)
return uow_id
def _get_supervisor_pid():
""" check for supervisor's pid file and returns pid from there """
try:
pid_filename = ProcessContext.get_pid_filename(process_context.PROCESS_SUPERVISOR)
pf = file(pid_filename, "r")
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
return pid
def clean_session_entries():
logger = ProcessContext.get_logger(PROCESS_UNIT_TEST)
ds = ds_manager.ds_factory(logger)
connection = ds.connection(COLLECTION_SINGLE_SESSION)
for i in range(base_fixtures.TOTAL_ENTRIES):
key = generate_session_composite_key(i, base_fixtures.TOTAL_ENTRIES)
connection.remove({
raw_data.KEY: key[0],
raw_data.TIMEPERIOD: key[1],
raw_data.FAMILY_USER_PROFILE + '.' + raw_data.SESSION_ID: 'session_id_%s' % str(i)})
def get_process_pid(process_name):
""" check for process' pid file and returns pid from there """
try:
pid_filename = ProcessContext.get_pid_filename(process_name)
pf = file(pid_filename, 'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
return pid
def compute_scope_of_processing(self, process_name, start_time, end_time, time_record):
"""method reads collection and identify slice for processing"""
source_collection_name = ProcessContext.get_source_collection(process_name)
target_collection_name = ProcessContext.get_target_collection(process_name)
source_collection = CollectionContext.get_collection(self.logger, source_collection_name)
query = { AbstractModel.TIMESTAMP : { '$gte' : start_time, '$lt' : end_time } }
asc_search = source_collection.find(spec=query, fields='_id').sort('_id', ASCENDING).limit(1)
if asc_search.count() == 0:
raise LookupError('No messages in timeperiod: %s:%s in collection %s'
% (start_time, end_time, source_collection_name))
first_object_id = asc_search[0]['_id']
dec_search = source_collection.find(spec=query, fields='_id').sort('_id', DESCENDING).limit(1)
last_object_id = dec_search[0]['_id']
unit_of_work = UnitOfWorkEntry()
unit_of_work.set_timestamp(start_time)
unit_of_work.set_start_id(str(first_object_id))
unit_of_work.set_end_id(str(last_object_id))
unit_of_work.set_start_timestamp(start_time)
unit_of_work.set_end_timestamp(end_time)
unit_of_work.set_created_at(datetime.utcnow())
unit_of_work.set_source_collection(source_collection_name)
unit_of_work.set_target_collection(target_collection_name)
unit_of_work.set_state(unit_of_work.STATE_REQUESTED)
unit_of_work.set_process_name(process_name)
unit_of_work.set_number_of_retries(0)
try:
uow_id = unit_of_work_helper.insert(self.logger, unit_of_work)
except DuplicateKeyError as e:
e.first_object_id = str(first_object_id)
e.last_object_id = str(last_object_id)
e.process_name = process_name
e.timestamp = start_time
raise e
self.publishers.get_publisher(process_name).publish(str(uow_id))
msg = 'Published: UOW %r for %r in timeperiod %r.' % (uow_id, process_name, start_time)
self._log_message(INFO, process_name, time_record, msg)
return unit_of_work
def start_by_process_name(process_name, *args):
"""
Function starts the process by:
1. retrieving its fully specified path name
2. if the path name ends with starter method - then creates an instance of the wrapping class
and calls <code>starter(*args)</code> method on it
3. if the path name ends with starter function - then retrieves its module
and calls <code>starter(*args)</code> function on it
"""
sys.stdout.write('INFO: Starter path {0} \n'.format(ProcessContext.get_classname(process_name)))
t, m, starter = get_class(ProcessContext.get_classname(process_name))
if isinstance(m, class_types):
sys.stdout.write('INFO: Starting process by calling starter method {0} \n'.format(starter))
instance = m(process_name)
method = getattr(instance, starter)
method(*args)
elif isinstance(m, types.FunctionType):
sys.stdout.write('INFO: Starting module.\n')
function = m
function(*args)
else:
raise ValueError('Improper starter path {0}'.format(ProcessContext.get_classname(process_name)))
def action_get_log(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(
self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(
time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
resp['log'] = node.job_record.log
return resp
def _get_job_collection(self, process_name):
"""jobs are stored in 4 collections: hourly, daily, monthly and yearly;
method looks for the proper job_collection base on process TIME_QUALIFIER"""
qualifier = ProcessContext.get_time_qualifier(process_name)
if qualifier == QUALIFIER_HOURLY:
collection = self.ds.connection(COLLECTION_JOB_HOURLY)
elif qualifier == QUALIFIER_DAILY:
collection = self.ds.connection(COLLECTION_JOB_DAILY)
elif qualifier == QUALIFIER_MONTHLY:
collection = self.ds.connection(COLLECTION_JOB_MONTHLY)
elif qualifier == QUALIFIER_YEARLY:
collection = self.ds.connection(COLLECTION_JOB_YEARLY)
else:
raise ValueError('Unknown time qualifier: %s for %s' % (qualifier, process_name))
return collection
def get_reprocessing_candidates(self, since=None):
""" method queries Unit Of Work whose <start_timeperiod> is younger than <since>
and who could be candidates for re-processing """
collection = self.ds.connection(COLLECTION_UNIT_OF_WORK)
query = {
unit_of_work.STATE: {
'$in': [
unit_of_work.STATE_IN_PROGRESS, unit_of_work.STATE_INVALID,
unit_of_work.STATE_REQUESTED
]
}
}
if since is None:
cursor = collection.find(query).sort('_id', ASCENDING)
candidates = [UnitOfWork(document) for document in cursor]
else:
candidates = []
yearly_timeperiod = time_helper.cast_to_time_qualifier(
QUALIFIER_YEARLY, since)
query[unit_of_work.START_TIMEPERIOD] = {'$gte': yearly_timeperiod}
cursor = collection.find(query).sort('_id', ASCENDING)
for document in cursor:
uow = UnitOfWork(document)
if uow.process_name not in ProcessContext.CONTEXT:
# this is a decommissioned process
continue
time_qualifier = ProcessContext.get_time_qualifier(
uow.process_name)
if time_qualifier == QUALIFIER_REAL_TIME:
time_qualifier = QUALIFIER_HOURLY
process_specific_since = time_helper.cast_to_time_qualifier(
time_qualifier, since)
if process_specific_since <= uow.start_timeperiod:
candidates.append(uow)
if len(candidates) == 0:
raise LookupError(
'MongoDB has no reprocessing candidates units of work')
return candidates
def start_process(options, args):
"""Start up specific daemon """
import psutil
from system import process_helper
from supervisor import supervisor_helper as helper
from system.process_context import ProcessContext
from supervisor.supervisor_constants import PROCESS_SUPERVISOR
from constants import PROCESS_LAUNCH_PY
logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY)
box_id = helper.get_box_id(logger)
if options.supervisor is True and options.app != PROCESS_SUPERVISOR:
from db.model import box_configuration
from db.dao.box_configuration_dao import BoxConfigurationDao
message = 'INFO: Marking %r to be managed by Supervisor \n' % options.app
sys.stdout.write(message)
bc_dao = BoxConfigurationDao(logger)
box_config = bc_dao.get_one(box_id)
box_config.set_process_state(options.app, box_configuration.STATE_ON)
bc_dao.update(box_config)
return
try:
pid = process_helper.get_process_pid(options.app)
if pid is not None:
if psutil.pid_exists(pid):
message = 'ERROR: Process %r is already running with pid %r\n' % (
options.app, pid)
sys.stderr.write(message)
sys.exit(1)
if not options.interactive:
# this block triggers if the options.interactive is not defined or is False
process_helper.start_process(options.app, args)
else:
process_starter.start_by_process_name(options.app, args)
except Exception as e:
sys.stderr.write('Exception on starting %s : %s \n' %
(options.app, str(e)))
traceback.print_exc(file=sys.stderr)
def action_skip(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(
self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(
time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
self.logger.info('MX (requesting skip timeperiod %r for %r) { ' %
(self.timeperiod, self.process_name))
effected_nodes = node.request_skip()
for node in effected_nodes:
resp[node.timeperiod] = TreeNodeDetails.get_details(
self.logger, node)
self.logger.info('}')
return resp
def run_tests(options):
import unittest
import settings
settings.enable_test_mode()
argv = [sys.argv[0]] + args
try:
unittest.main(module=None,
defaultTest='__main__.load_all_tests',
argv=argv)
except SystemExit as e:
from system.process_context import ProcessContext
from constants import PROCESS_LAUNCH_PY
logger = ProcessContext.get_logger(PROCESS_LAUNCH_PY)
if e.code == 0:
logger.info('PASS')
else:
logger.error('FAIL')
raise
def action_get_uow(self):
resp = dict()
timetable = self.mbean.timetable
tree = timetable.get_tree(self.process_name)
if tree is not None:
time_qualifier = ProcessContext.get_time_qualifier(
self.process_name)
self.timeperiod = time_helper.cast_to_time_qualifier(
time_qualifier, self.timeperiod)
node = tree.get_node_by_process(self.process_name, self.timeperiod)
uow_id = node.job_record.related_unit_of_work
if uow_id is None:
resp = {'response': 'no related unit_of_work'}
else:
resp = self.uow_dao.get_one(uow_id).document
for key in resp:
resp[key] = str(resp[key])
return resp
def write(self, msg, level=logging.INFO):
""" method implements stream write interface, allowing to redirect stdout to logger """
if msg is not None and len(msg.strip()) > 0:
self.logger.log(level, msg)
def flush(self):
""" method implements stream flush interface, allowing to redirect stdout to logger """
for handler in self.logger.handlers:
handler.flush()
def isatty(self):
""" is the sys.stdout attached to the terminal?
python -c "import sys; print(sys.stdout.isatty())" (should write True)
python -c "import sys; print(sys.stdout.isatty())" | grep . (should write False).
:return: False, indicating that the output is pipped or redirected
"""
return False
if __name__ == '__main__':
from system.process_context import ProcessContext
from tests.ut_process_context import PROCESS_UNIT_TEST, register_unit_test_context
register_unit_test_context()
logger = ProcessContext.get_logger(PROCESS_UNIT_TEST)
logger.info('test_message')
print('regular print message')
sys.stdout.flush()
def __del__(self):
""" removes PID file """
ProcessContext.remove_pid_file(self.process_name,
process_id=self.process_id)
self.logger.info('Shutdown {0}'.format(self.process_name))
