def elioterize(action_type, master_id=None, worker_id=None):
"""Eliot action continuer that can log to either worker or master."""
clear_logger_destinations(Logger)
if worker_id:
to_file(get_eliot_log_file(worker_id=worker_id))
client = RedisWorker(worker_id)
elif master_id:
to_file(get_eliot_log_file(master_id=master_id))
client = RedisMaster(master_id)
with Action.continue_task(task_id=client.eliot_task_id):
return start_action(action_type=action_type)
def elioterize(action_type, master_id=None, worker_id=None):
"""Eliot action continuer that can log to either worker or master."""
clear_logger_destinations(Logger)
if worker_id:
to_file(get_eliot_log_file(worker_id=worker_id))
client = RedisWorker(worker_id)
elif master_id:
to_file(get_eliot_log_file(master_id=master_id))
client = RedisMaster(master_id)
with Action.continue_task(task_id=client.eliot_task_id):
return start_action(action_type=action_type)
def _pytest_sessionstart(session, worker):
"""Initialize session-wide variables for record management and caching."""
session.invenio_records = {'original': {}, 'modified': {}, 'temporary': {}}
global Session # pylint: disable=global-statement
Session = session
# Set the eliot log path
eliot.to_file(get_eliot_log_file(worker_id=worker.uuid))
session.invenio_eliot_action = eliot.start_action(action_type=u"pytest worker")
def _run_task(rule_name, master_id):
# # If you find yourself debugging celery crashes:
# redis_master = None
# def cleanup_session():
# if redis_master is not None:
# redis_master.zap()
# def sigint_hook(rcv_signal, frame):
# cleanup_session()
# def except_hook(type_, value, tback):
# cleanup_session()
# reraise(type_, value, tback)
# signal.signal(signal.SIGINT, sigint_hook)
# signal.signal(signal.SIGTERM, sigint_hook)
# sys.excepthook = except_hook
clear_logger_destinations(Logger)
to_file(get_eliot_log_file(master_id=master_id))
with start_task(action_type="invenio_checker:supervisor:_run_task",
master_id=master_id) as eliot_task:
eliot_task_id = eliot_task.serialize_task_id()
# Have the master initialize its presence in redis.
Message.log(message_type='creating master')
redis_master = RedisMaster.create(master_id, eliot_task_id, rule_name)
# Load the rule from its name. `run_task` has already checked that it's
# there.
rule = CheckerRule.query.filter(CheckerRule.name == rule_name).one()
Message.log(message_type='loaded rule', rule_name=rule.name)
# Create workers to attach to this master. `record_centric` means that
# the task uses the `record` fixture, which causes pytest to loop over
# it len(chunk_recids) times. This is important to know now so that we
# will spawn multiple workers.
subtasks = []
record_centric = _get_record_fixture_presence(rule.filepath)
if record_centric:
# We wish to spawn multiple workers to split the load.
if rule.allow_chunking:
recid_chunks = tuple(chunk_recids(rule.modified_requested_recids))
else:
recid_chunks = (rule.modified_requested_recids,)
Message.log(message_type='creating subtasks', count=len(recid_chunks),
mode='record_centric', recid_count=len(rule.modified_requested_recids))
else:
# We wish to spawn just one worker than will run the check function
# once.
recid_chunks = (set(),)
Message.log(message_type='creating subtasks', count=1,
mode='not_record_centric')
# Create the subtasks based on the decisions taken above and inform the
# master of its associations with these new workers/tasks.
for chunk in recid_chunks:
task_id = uuid()
redis_master.workers_append(task_id)
subtasks.append(create_celery_task(task_id, redis_master.master_id,
rule, chunk, eliot_task))
if not subtasks:
# Note that if `record-centric` is True, there's the chance that no
# records matched our query. This does not imply a problem.
redis_master.status = StatusMaster.completed
else:
redis_master.status = StatusMaster.running
# FIXME: handle_all_completion should be called after the callbacks
# of all workers have completed.
callback = handle_all_completion.subtask()
chord(subtasks)(callback)
def _run_task(rule_name, master_id):
# # If you find yourself debugging celery crashes:
# redis_master = None
# def cleanup_session():
# if redis_master is not None:
# redis_master.zap()
# def sigint_hook(rcv_signal, frame):
# cleanup_session()
# def except_hook(type_, value, tback):
# cleanup_session()
# reraise(type_, value, tback)
# signal.signal(signal.SIGINT, sigint_hook)
# signal.signal(signal.SIGTERM, sigint_hook)
# sys.excepthook = except_hook
clear_logger_destinations(Logger)
to_file(get_eliot_log_file(master_id=master_id))
with start_task(action_type="invenio_checker:supervisor:_run_task",
master_id=master_id) as eliot_task:
eliot_task_id = eliot_task.serialize_task_id()
# Have the master initialize its presence in redis.
Message.log(message_type='creating master')
redis_master = RedisMaster.create(master_id, eliot_task_id, rule_name)
# Load the rule from its name. `run_task` has already checked that it's
# there.
rule = CheckerRule.query.filter(CheckerRule.name == rule_name).one()
Message.log(message_type='loaded rule', rule_name=rule.name)
# Create workers to attach to this master. `record_centric` means that
# the task uses the `record` fixture, which causes pytest to loop over
# it len(chunk_recids) times. This is important to know now so that we
# will spawn multiple workers.
subtasks = []
record_centric = _get_record_fixture_presence(rule.filepath)
if record_centric:
# We wish to spawn multiple workers to split the load.
if rule.allow_chunking:
recid_chunks = tuple(
chunk_recids(rule.modified_requested_recids))
else:
recid_chunks = (rule.modified_requested_recids, )
Message.log(message_type='creating subtasks',
count=len(recid_chunks),
mode='record_centric',
recid_count=len(rule.modified_requested_recids))
else:
# We wish to spawn just one worker than will run the check function
# once.
recid_chunks = (set(), )
Message.log(message_type='creating subtasks',
count=1,
mode='not_record_centric')
# Create the subtasks based on the decisions taken above and inform the
# master of its associations with these new workers/tasks.
for chunk in recid_chunks:
task_id = uuid()
redis_master.workers_append(task_id)
subtasks.append(
create_celery_task(task_id, redis_master.master_id, rule,
chunk, eliot_task))
if not subtasks:
# Note that if `record-centric` is True, there's the chance that no
# records matched our query. This does not imply a problem.
redis_master.status = StatusMaster.completed
else:
redis_master.status = StatusMaster.running
# FIXME: handle_all_completion should be called after the callbacks
# of all workers have completed.
callback = handle_all_completion.subtask()
chord(subtasks)(callback)
