def create_unit_of_work(process_name, start_id, end_id, timeperiod='INVALID_TIMEPERIOD', state=unit_of_work.STATE_REQUESTED, creation_at=datetime.utcnow(), uow_id=None): """ method creates and returns unit_of_work """ try: source_collection = ProcessContext.get_source(process_name) target_collection = ProcessContext.get_sink(process_name) except KeyError: source_collection = None target_collection = None uow = UnitOfWork() uow.timeperiod = timeperiod uow.start_timeperiod = timeperiod uow.end_timeperiod = timeperiod uow.start_id = start_id uow.end_id = end_id uow.source = source_collection uow.sink = target_collection uow.state = state uow.created_at = creation_at uow.process_name = process_name uow.number_of_retries = 0 if uow_id is not None: uow.document['_id'] = uow_id return uow
def compute_scope_of_processing(self, process_name, start_timeperiod, end_timeperiod, job_record): """method reads collection and identify slice for processing""" source_collection_name = ProcessContext.get_source(process_name) target_collection_name = ProcessContext.get_sink(process_name) start_id = self.ds.highest_primary_key(source_collection_name, start_timeperiod, end_timeperiod) end_id = self.ds.lowest_primary_key(source_collection_name, start_timeperiod, end_timeperiod) uow = UnitOfWork() uow.timeperiod = start_timeperiod uow.start_id = str(start_id) uow.end_id = str(end_id) uow.start_timeperiod = start_timeperiod uow.end_timeperiod = end_timeperiod uow.created_at = datetime.utcnow() uow.source = source_collection_name uow.sink = target_collection_name uow.state = unit_of_work.STATE_REQUESTED uow.process_name = process_name uow.number_of_retries = 0 uow_id = self.uow_dao.insert(uow) mq_request = WorkerMqRequest() mq_request.process_name = process_name mq_request.unit_of_work_id = uow_id publisher = self.publishers.get(process_name) publisher.publish(mq_request.document) publisher.release() msg = 'Published: UOW %r for %r in timeperiod %r.' % (uow_id, process_name, start_timeperiod) self._log_message(INFO, process_name, job_record, msg) return uow
def __init__(self, process_name): """@param process_name: id of the process, the worker will be performing """ super(AbstractMqWorker, self).__init__(process_name) self.queue_source = ProcessContext.get_source(self.process_name) self.queue_sink = ProcessContext.get_sink(self.process_name) self.consumer = None self._init_mq_consumer() self.main_thread = None self.performance_ticker = None self._init_performance_ticker(self.logger) msg_suffix = 'in Production Mode' if settings['under_test']: msg_suffix = 'in Testing Mode' self.logger.info('Started %s %s' % (self.process_name, msg_suffix))
def _start_process(self, start_timeperiod, end_timeperiod, arguments): try: input_file = ProcessContext.get_source(self.process_name) self.logger.info('start: %s {' % self.process_name) p = psutil.Popen([settings['bash_shell'], settings['pig_command'], '-f', '/home/bmushkevych/git/synergy-pig/script.pig', '-p', 'input_file=' + input_file + '/' + start_timeperiod, '-p', 'timeperiod=' + start_timeperiod], close_fds=True, cwd=settings['process_cwd'], stdin=PIPE, stdout=PIPE, stderr=PIPE) self.cli_process = p self.logger.info('Started %s with pid = %r' % (self.process_name, p.pid)) except Exception: self.logger.error('Exception on starting: %s' % self.process_name, exc_info=True) finally: self.logger.info('}')
def _mq_callback(self, message): """ try/except wrapper in case exception breaks the abstract method, this method: - catches the exception - logs the exception - marks unit of work as INVALID""" try: mq_request = WorkerMqRequest(message.body) uow = self.uow_dao.get_one(mq_request.unit_of_work_id) if uow.state in [unit_of_work.STATE_CANCELED, unit_of_work.STATE_PROCESSED]: # garbage collector might have reposted this UOW self.logger.warning('Skipping unit_of_work: id %s; state %s;' % (str(message.body), uow.state), exc_info=False) self.consumer.acknowledge(message.delivery_tag) return except Exception: self.logger.error('Safety fuse. Can not identify unit_of_work %s' % str(message.body), exc_info=True) self.consumer.acknowledge(message.delivery_tag) return try: start_id_obj = uow.start_id end_id_obj = uow.end_id start_timeperiod = uow.start_timeperiod end_timeperiod = uow.end_timeperiod uow.state = unit_of_work.STATE_IN_PROGRESS uow.started_at = datetime.utcnow() self.uow_dao.update(uow) self.performance_ticker.start_uow(uow) bulk_threshold = settings['bulk_threshold'] iteration = 0 while True: collection_name = ProcessContext.get_source(self.process_name) cursor = self.ds.cursor_for(collection_name, start_id_obj, end_id_obj, iteration, start_timeperiod, end_timeperiod, bulk_threshold) count = cursor.count(with_limit_and_skip=True) if count == 0 and iteration == 0: msg = 'No entries in %s at range [%s : %s]' % (collection_name, uow.start_id, uow.end_id) self.logger.warning(msg) break else: shall_continue, new_start_id = self._process_not_empty_cursor(cursor) if shall_continue: start_id_obj = new_start_id iteration += 1 else: break msg = 'Cursor exploited after %s iterations' % str(iteration) self.logger.info(msg) self.perform_post_processing(uow.timeperiod) number_of_aggregated_objects = self._flush_aggregated_objects() uow.number_of_aggregated_documents = number_of_aggregated_objects uow.number_of_processed_documents = self.performance_ticker.per_job uow.finished_at = datetime.utcnow() uow.state = unit_of_work.STATE_PROCESSED self.uow_dao.update(uow) self.performance_ticker.finish_uow() except Exception as e: uow.state = unit_of_work.STATE_INVALID self.uow_dao.update(uow) self.performance_ticker.cancel_uow() del self.aggregated_objects self.aggregated_objects = dict() gc.collect() self.logger.error('Safety fuse while processing unit_of_work %s in timeperiod %s : %r' % (message.body, uow.timeperiod, e), exc_info=True) finally: self.consumer.acknowledge(message.delivery_tag) self.consumer.close()