def elioterize(action_type, master_id=None, worker_id=None): """Eliot action continuer that can log to either worker or master.""" clear_logger_destinations(Logger) if worker_id: to_file(get_eliot_log_file(worker_id=worker_id)) client = RedisWorker(worker_id) elif master_id: to_file(get_eliot_log_file(master_id=master_id)) client = RedisMaster(master_id) with Action.continue_task(task_id=client.eliot_task_id): return start_action(action_type=action_type)
def _pytest_sessionstart(session, worker): """Initialize session-wide variables for record management and caching.""" session.invenio_records = {'original': {}, 'modified': {}, 'temporary': {}} global Session # pylint: disable=global-statement Session = session # Set the eliot log path eliot.to_file(get_eliot_log_file(worker_id=worker.uuid)) session.invenio_eliot_action = eliot.start_action(action_type=u"pytest worker")
def _run_task(rule_name, master_id): # # If you find yourself debugging celery crashes: # redis_master = None # def cleanup_session(): # if redis_master is not None: # redis_master.zap() # def sigint_hook(rcv_signal, frame): # cleanup_session() # def except_hook(type_, value, tback): # cleanup_session() # reraise(type_, value, tback) # signal.signal(signal.SIGINT, sigint_hook) # signal.signal(signal.SIGTERM, sigint_hook) # sys.excepthook = except_hook clear_logger_destinations(Logger) to_file(get_eliot_log_file(master_id=master_id)) with start_task(action_type="invenio_checker:supervisor:_run_task", master_id=master_id) as eliot_task: eliot_task_id = eliot_task.serialize_task_id() # Have the master initialize its presence in redis. Message.log(message_type='creating master') redis_master = RedisMaster.create(master_id, eliot_task_id, rule_name) # Load the rule from its name. `run_task` has already checked that it's # there. rule = CheckerRule.query.filter(CheckerRule.name == rule_name).one() Message.log(message_type='loaded rule', rule_name=rule.name) # Create workers to attach to this master. `record_centric` means that # the task uses the `record` fixture, which causes pytest to loop over # it len(chunk_recids) times. This is important to know now so that we # will spawn multiple workers. subtasks = [] record_centric = _get_record_fixture_presence(rule.filepath) if record_centric: # We wish to spawn multiple workers to split the load. if rule.allow_chunking: recid_chunks = tuple(chunk_recids(rule.modified_requested_recids)) else: recid_chunks = (rule.modified_requested_recids,) Message.log(message_type='creating subtasks', count=len(recid_chunks), mode='record_centric', recid_count=len(rule.modified_requested_recids)) else: # We wish to spawn just one worker than will run the check function # once. recid_chunks = (set(),) Message.log(message_type='creating subtasks', count=1, mode='not_record_centric') # Create the subtasks based on the decisions taken above and inform the # master of its associations with these new workers/tasks. for chunk in recid_chunks: task_id = uuid() redis_master.workers_append(task_id) subtasks.append(create_celery_task(task_id, redis_master.master_id, rule, chunk, eliot_task)) if not subtasks: # Note that if `record-centric` is True, there's the chance that no # records matched our query. This does not imply a problem. redis_master.status = StatusMaster.completed else: redis_master.status = StatusMaster.running # FIXME: handle_all_completion should be called after the callbacks # of all workers have completed. callback = handle_all_completion.subtask() chord(subtasks)(callback)
def _run_task(rule_name, master_id): # # If you find yourself debugging celery crashes: # redis_master = None # def cleanup_session(): # if redis_master is not None: # redis_master.zap() # def sigint_hook(rcv_signal, frame): # cleanup_session() # def except_hook(type_, value, tback): # cleanup_session() # reraise(type_, value, tback) # signal.signal(signal.SIGINT, sigint_hook) # signal.signal(signal.SIGTERM, sigint_hook) # sys.excepthook = except_hook clear_logger_destinations(Logger) to_file(get_eliot_log_file(master_id=master_id)) with start_task(action_type="invenio_checker:supervisor:_run_task", master_id=master_id) as eliot_task: eliot_task_id = eliot_task.serialize_task_id() # Have the master initialize its presence in redis. Message.log(message_type='creating master') redis_master = RedisMaster.create(master_id, eliot_task_id, rule_name) # Load the rule from its name. `run_task` has already checked that it's # there. rule = CheckerRule.query.filter(CheckerRule.name == rule_name).one() Message.log(message_type='loaded rule', rule_name=rule.name) # Create workers to attach to this master. `record_centric` means that # the task uses the `record` fixture, which causes pytest to loop over # it len(chunk_recids) times. This is important to know now so that we # will spawn multiple workers. subtasks = [] record_centric = _get_record_fixture_presence(rule.filepath) if record_centric: # We wish to spawn multiple workers to split the load. if rule.allow_chunking: recid_chunks = tuple( chunk_recids(rule.modified_requested_recids)) else: recid_chunks = (rule.modified_requested_recids, ) Message.log(message_type='creating subtasks', count=len(recid_chunks), mode='record_centric', recid_count=len(rule.modified_requested_recids)) else: # We wish to spawn just one worker than will run the check function # once. recid_chunks = (set(), ) Message.log(message_type='creating subtasks', count=1, mode='not_record_centric') # Create the subtasks based on the decisions taken above and inform the # master of its associations with these new workers/tasks. for chunk in recid_chunks: task_id = uuid() redis_master.workers_append(task_id) subtasks.append( create_celery_task(task_id, redis_master.master_id, rule, chunk, eliot_task)) if not subtasks: # Note that if `record-centric` is True, there's the chance that no # records matched our query. This does not imply a problem. redis_master.status = StatusMaster.completed else: redis_master.status = StatusMaster.running # FIXME: handle_all_completion should be called after the callbacks # of all workers have completed. callback = handle_all_completion.subtask() chord(subtasks)(callback)