def get_orcids_for_push(record):
"""Obtain the ORCIDs associated to the list of authors in the Literature record.
The ORCIDs are looked up both in the ``ids`` of the ``authors`` and in the
Author records that have claimed the paper.
Args:
record(dict): metadata from a Literature record
Returns:
Iterator[str]: all ORCIDs associated to these authors
"""
orcids_on_record = []
author_recids_with_claims = []
for author in record.get('authors', []):
orcids_in_author = get_values_for_schema(author.get('ids', []),
'ORCID')
if orcids_in_author:
orcids_on_record.extend(orcids_in_author)
elif author.get('curated_relation') is True and 'record' in author:
author_recids_with_claims.append(
get_recid_from_ref(author['record']))
author_records = get_db_records('aut', author_recids_with_claims)
all_ids = (author.get('ids', []) for author in author_records)
orcids_in_authors = chain.from_iterable(
get_values_for_schema(ids, 'ORCID') for ids in all_ids)
return chain(orcids_on_record, orcids_in_authors)
def get_linked_records_in_field(record, field_path):
"""Get all linked records in a given field.
Args:
record (dict): the record containing the links
field_path (string): a dotted field path specification understandable
by ``get_value``, containing a json reference to another record.
Returns:
Iterator[dict]: an iterator on the linked record.
Warning:
Currently, the order in which the linked records are yielded is
different from the order in which they appear in the record.
Example:
>>> record = {'references': [
... {'record': {'$ref': 'https://labs.inspirehep.net/api/literature/1234'}},
... {'record': {'$ref': 'https://labs.inspirehep.net/api/data/421'}},
... ]}
>>> get_linked_record_in_field(record, 'references.record')
[...]
"""
full_path = '.'.join([field_path, '$ref'])
pids = force_list([
get_pid_from_record_uri(rec)
for rec in get_value(record, full_path, [])
])
return get_db_records(pids)
def get_orcids_for_push(record):
"""Obtain the ORCIDs associated to the list of authors in the Literature record.
The ORCIDs are looked up both in the ``ids`` of the ``authors`` and in the
Author records that have claimed the paper.
Args:
record(dict): metadata from a Literature record
Returns:
Iterator[str]: all ORCIDs associated to these authors
"""
orcids_on_record = []
author_recids_with_claims = []
for author in record.get('authors', []):
orcids_in_author = get_values_for_schema(author.get('ids', []), 'ORCID')
if orcids_in_author:
orcids_on_record.extend(orcids_in_author)
elif author.get('curated_relation') is True and 'record' in author:
author_recids_with_claims.append(get_recid_from_ref(author['record']))
author_records = get_db_records(('aut', recid) for recid in author_recids_with_claims)
all_ids = (author.get('ids', []) for author in author_records)
orcids_in_authors = chain.from_iterable(get_values_for_schema(ids, 'ORCID') for ids in all_ids)
return chain(orcids_on_record, orcids_in_authors)
def get_linked_records_in_field(record, field_path):
"""Get all linked records in a given field.
Args:
record (dict): the record containing the links
field_path (string): a dotted field path specification understandable
by ``get_value``, containing a json reference to another record.
Returns:
Iterator[dict]: an iterator on the linked record.
Warning:
Currently, the order in which the linked records are yielded is
different from the order in which they appear in the record.
Example:
>>> record = {'references': [
... {'record': {'$ref': 'https://labs.inspirehep.net/api/literature/1234'}},
... {'record': {'$ref': 'https://labs.inspirehep.net/api/data/421'}},
... ]}
>>> get_linked_record_in_field(record, 'references.record')
[...]
"""
full_path = '.'.join([field_path, '$ref'])
pids = force_list([get_pid_from_record_uri(rec) for rec in get_value(record, full_path, [])])
return get_db_records(pids)
def test_get_db_records_finds_right_results(app):
literature = [1498175, 1090628]
authors = [983059]
results = list(get_db_records('lit', literature + authors))
recids = {result['control_number'] for result in results}
assert len(results) == len(literature)
assert recids == set(literature)
def test_get_db_records_accept_multiple_pid_types(app):
records = [('lit', 1498175), ('lit', 1090628), ('aut', 983059)]
results = list(get_db_records(records))
assert len(results) == 3
def test_get_db_records_accepts_string_pid_values(app):
records = list(get_db_records([('lit', '4328')]))
assert len(records) == 1
def test_get_db_records_accepts_integer_pid_values(app):
records = list(get_db_records([('lit', 4328)]))
assert len(records) == 1
def test_get_db_records_handles_empty_lists(app):
assert list(get_db_records([])) == []
def find_arxiv_duplicates(starting_date=None):
"""Finds arXiv paper created more than once.
After duplicates are detected, merged/deleted records are filtered out.
Example:
>> from datetime import datetime
>> date = datetime(2019, 01, 10)
>> dups = find_arxiv_duplicates(date)
>> dups
{'1812.10562': {'recids': [1711898, 1711919], 'wf_ids': [1387687, 1388915]}}
Args:
starting_date (datetime): date from which starting to search workflows
Returns:
(dict): a dictionary having as key the arXiv id and as value a
dictionary having two keys: `recids`, which is a list of control
numbers of the paper refering to the arXiv id key, and `wf_ids`,
which contains the list od ids of the workflows that created the
related records.
"""
query = WorkflowObjectModel.query.\
with_entities(WorkflowObjectModel.id).\
filter(WorkflowObjectModel.status == ObjectStatus.COMPLETED).\
filter(cast(WorkflowObjectModel.data, JSONB).has_key('arxiv_eprints')).\
filter(cast(WorkflowObjectModel.data, JSONB).has_key('control_number')) # noqa: W601
if starting_date:
query = query.filter(WorkflowObjectModel.created > starting_date)
workflows_ids = [wf_id for (wf_id,) in query.all()]
duplicates = dict()
with click.progressbar(workflows_ids, label='Processing workflows') as wf_iterator:
for wf_id in wf_iterator:
wf = workflow_object_class.get(wf_id)
recid = wf.data.get('control_number')
arxiv_id = wf.data['arxiv_eprints'][0]['value']
if arxiv_id not in duplicates:
duplicates[arxiv_id] = {'recids': [], 'wf_ids': []}
if recid not in duplicates[arxiv_id]['recids']:
duplicates[arxiv_id]['recids'].append(recid)
duplicates[arxiv_id]['wf_ids'].append(wf_id)
duplicates = {k: v for (k, v) in duplicates.items() if len(v['recids']) > 1}
with click.progressbar(duplicates, label='Removing merged records') as rec_iterator:
for arxiv_id in rec_iterator:
pids = [('lit', _id) for _id in duplicates[arxiv_id]['recids']]
records = get_db_records(pids)
for rec in records:
if rec.get('deleted'):
idx = duplicates[arxiv_id]['recids'].index(rec['control_number'])
del duplicates[arxiv_id]['recids'][idx]
del duplicates[arxiv_id]['wf_ids'][idx]
click.secho('Found %d arXiv duplicates' % len(duplicates))
click.secho(json.dumps(duplicates))
def find_arxiv_duplicates(starting_date=None):
"""Finds arXiv paper created more than once.
After duplicates are detected, merged/deleted records are filtered out.
Example:
>> from datetime import datetime
>> date = datetime(2019, 01, 10)
>> dups = find_arxiv_duplicates(date)
>> dups
{'1812.10562': {'recids': [1711898, 1711919], 'wf_ids': [1387687, 1388915]}}
Args:
starting_date (datetime): date from which starting to search workflows
Returns:
(dict): a dictionary having as key the arXiv id and as value a
dictionary having two keys: `recids`, which is a list of control
numbers of the paper refering to the arXiv id key, and `wf_ids`,
which contains the list od ids of the workflows that created the
related records.
"""
query = WorkflowObjectModel.query.\
with_entities(WorkflowObjectModel.id).\
filter(WorkflowObjectModel.status == ObjectStatus.COMPLETED).\
filter(cast(WorkflowObjectModel.data, JSONB).has_key('arxiv_eprints')).\
filter(cast(WorkflowObjectModel.data, JSONB).has_key('control_number')) # noqa: W601
if starting_date:
query = query.filter(WorkflowObjectModel.created > starting_date)
workflows_ids = [wf_id for (wf_id, ) in query.all()]
duplicates = dict()
with click.progressbar(workflows_ids,
label='Processing workflows') as wf_iterator:
for wf_id in wf_iterator:
wf = workflow_object_class.get(wf_id)
recid = wf.data.get('control_number')
arxiv_id = wf.data['arxiv_eprints'][0]['value']
if arxiv_id not in duplicates:
duplicates[arxiv_id] = {'recids': [], 'wf_ids': []}
if recid not in duplicates[arxiv_id]['recids']:
duplicates[arxiv_id]['recids'].append(recid)
duplicates[arxiv_id]['wf_ids'].append(wf_id)
duplicates = {
k: v
for (k, v) in duplicates.items() if len(v['recids']) > 1
}
with click.progressbar(duplicates,
label='Removing merged records') as rec_iterator:
for arxiv_id in rec_iterator:
pids = [('lit', _id) for _id in duplicates[arxiv_id]['recids']]
records = get_db_records(pids)
for rec in records:
if rec.get('deleted'):
idx = duplicates[arxiv_id]['recids'].index(
rec['control_number'])
del duplicates[arxiv_id]['recids'][idx]
del duplicates[arxiv_id]['wf_ids'][idx]
click.secho('Found %d arXiv duplicates' % len(duplicates))
click.secho(json.dumps(duplicates))
def test_get_db_records_accepts_lists_of_strings(app):
records = list(get_db_records('lit', ['4328']))
assert len(records) == 1
def test_get_db_records_accepts_lists_of_integers(app):
records = list(get_db_records('lit', [4328]))
assert len(records) == 1