def assign_to_author(from_author_recid, to_author_recid, literature_recids):
author_record = AuthorsRecord.get_record_by_pid_value(to_author_recid)
num_workers = count_consumers_for_queue("assign")
for batch in chunker(literature_recids, 10, num_workers):
current_celery_app.send_task(
"inspirehep.assign.tasks.assign_papers",
kwargs={
"from_author_recid": from_author_recid,
"to_author_record": author_record,
"author_papers_recids": batch,
},
)
unstub_author_by_recid(to_author_recid)
def assign_to_new_stub_author(from_author_recid, literature_recids):
# TODO: differentiate from BEARD created stub author
author_papers = get_literature_records_by_recid(literature_recids)
author_signatures = get_author_signatures(from_author_recid, author_papers)
stub_author_data = update_author_names({"name": {}}, author_signatures)
to_author = create_new_stub_author(**stub_author_data)
num_workers = count_consumers_for_queue("assign")
for batch in chunker(literature_recids, 10, num_workers):
current_celery_app.send_task(
"inspirehep.assign.tasks.assign_papers",
kwargs={
"from_author_recid": from_author_recid,
"to_author_record": to_author,
"author_papers_recids": batch,
"is_stub_author": True,
},
)
return to_author["control_number"]
def test_count_consumers_for_queue(mock_inspect):
mock_inspect.return_value.active_queues.return_value = {
"worker-1": [{
"name": "some-queue"
}, {
"name": "other-queue"
}],
"worker-2": [{
"name": "other-queue"
}],
"worker-3": [{
"name": "other-queue"
}],
"worker-4": [{
"name": "some-queue"
}],
}
assert count_consumers_for_queue("some-queue") == 2
def migrate_from_mirror(also_migrate=None,
disable_external_push=True,
date_from=None):
"""Migrate legacy records from the local mirror.
By default, only the records that have not been migrated yet are migrated.
Args:
also_migrate(Optional[string]): if set to ``'broken'``, also broken
records will be migrated. If set to ``'all'``, all records will be
migrated.
disable_external_push (bool): flag indicating whether the orcid_push
and hal push should be disabled (if True) or executed at the end
of migrations (if False).
"""
disable_references_processing = False
query = LegacyRecordsMirror.query.with_entities(LegacyRecordsMirror.recid)
if also_migrate is None:
query = query.filter(LegacyRecordsMirror.valid.is_(None))
elif also_migrate == "broken":
query = query.filter(LegacyRecordsMirror.valid.isnot(True))
elif also_migrate == "all":
disable_references_processing = True
else:
raise ValueError(
'"also_migrate" should be either None, "all" or "broken"')
if date_from:
date = parse(date_from, ignoretz=True)
query = query.filter(LegacyRecordsMirror.last_updated >= date)
num_workers = count_consumers_for_queue("migrator")
recids_chunked = chunker(
[res.recid for res in query.yield_per(CHUNK_SIZE)], CHUNK_SIZE,
num_workers)
task = migrate_recids_from_mirror(
list(recids_chunked),
disable_external_push=disable_external_push,
disable_references_processing=disable_references_processing,
)
LOGGER.info("All migration tasks have been scheduled.")
return task
def migrate_from_mirror_run_step(disable_external_push=True,
disable_references_processing=True,
step_no=1):
"""Allows to easily run step by step migration only for valid records """
num_workers = count_consumers_for_queue("migrator")
if step_no == 0:
query = LegacyRecordsMirror.query.with_entities(
LegacyRecordsMirror.recid).filter(
LegacyRecordsMirror.valid.is_(True))
recids_chunked = chunker(
[str(res.recid) for res in query.yield_per(CHUNK_SIZE)],
CHUNK_SIZE,
num_workers,
)
elif 0 < step_no < 3:
query = (PersistentIdentifier.query.with_entities(
PersistentIdentifier.object_uuid).filter_by(
pid_provider="recid").distinct())
recids_chunked = chunker(
[str(res.object_uuid) for res in query.yield_per(CHUNK_SIZE)],
CHUNK_SIZE,
num_workers,
)
else:
echo("Wrong step number!")
return
task = migrate_recids_from_mirror(
list(recids_chunked),
disable_external_push=disable_external_push,
disable_references_processing=disable_references_processing,
step_no=step_no,
one_step=True,
)
echo("All migration tasks have been scheduled.")
return task