def map_refextract_to_schema(extracted_references, source=None):
"""Convert refextract output to the schema using the builder."""
result = []
for reference in extracted_references:
rb = ReferenceBuilder()
mapping = [
('author', rb.add_refextract_authors_str),
('collaboration', rb.add_collaboration),
('doi', rb.add_uid),
('hdl', rb.add_uid),
('isbn', rb.add_uid),
('journal_reference', rb.set_pubnote),
('linemarker', rb.set_label),
('misc', rb.add_misc),
('publisher', rb.set_publisher),
('raw_ref', lambda raw_ref: rb.add_raw_reference(raw_ref, source=source)),
('reportnumber', rb.add_report_number),
('texkey', rb.set_texkey),
('title', rb.add_title),
('url', rb.add_url),
('year', rb.set_year),
]
for field, method in mapping:
for el in force_list(reference.get(field)):
if el:
method(el)
if get_value(rb.obj, 'reference.urls'):
rb.obj['reference']['urls'] = dedupe_list_of_dicts(rb.obj['reference']['urls'])
result.append(rb.obj)
return result
def dedupe_all_lists(obj, exclude_keys=()):
"""Recursively remove duplucates from all lists.
Args:
obj: collection to deduplicate
exclude_keys (Container[str]): key names to ignore for deduplication
"""
squared_dedupe_len = 10
if isinstance(obj, dict):
new_obj = {}
for key, value in obj.items():
if key in exclude_keys:
new_obj[key] = value
else:
new_obj[key] = dedupe_all_lists(value)
return new_obj
elif isinstance(obj, (list, tuple, set)):
new_elements = [dedupe_all_lists(v) for v in obj]
if len(new_elements) < squared_dedupe_len:
new_obj = dedupe_list(new_elements)
else:
new_obj = dedupe_list_of_dicts(new_elements)
return type(obj)(new_obj)
else:
return obj
def references(self):
"""Extract a Crossref record into an Inspire HEP references record.
Returns:
List[dict]: an array of reference schema records, representing
the references in the record
"""
ref_keys = self.record.get("reference")
reference_list = list(
itertools.chain.from_iterable(
self.get_reference(key) for key in force_list(ref_keys)))
return dedupe_list_of_dicts(reference_list)
def dedupe_all_lists(obj):
"""Recursively remove duplucates from all lists."""
squared_dedupe_len = 10
if isinstance(obj, dict):
new_obj = {}
for key, value in obj.items():
new_obj[key] = dedupe_all_lists(value)
return new_obj
elif isinstance(obj, (list, tuple, set)):
new_elements = [dedupe_all_lists(v) for v in obj]
if len(new_elements) < squared_dedupe_len:
new_obj = dedupe_list(new_elements)
else:
new_obj = dedupe_list_of_dicts(new_elements)
return type(obj)(new_obj)
else:
return obj
def test_dedupe_list_of_dicts():
list_of_dicts_with_duplicates = [
{
'a': 123,
'b': 1234
},
{
'a': 3222,
'b': 1234
},
{
'a': 123,
'b': 1234
},
]
expected = [{'a': 123, 'b': 1234}, {'a': 3222, 'b': 1234}]
result = dedupe_list_of_dicts(list_of_dicts_with_duplicates)
assert expected == result
def map_refextract_to_schema(extracted_references, source=None):
"""Convert refextract output to the schema using the builder."""
result = []
for reference in extracted_references:
rb = ReferenceBuilder()
mapping = [
("author", rb.add_refextract_authors_str),
("collaboration", rb.add_collaboration),
("doi", rb.add_uid),
("hdl", rb.add_uid),
("isbn", rb.add_uid),
("journal_reference", rb.set_pubnote),
("linemarker", rb.set_label),
("misc", rb.add_misc),
("publisher", rb.set_publisher),
("raw_ref",
lambda raw_ref: rb.add_raw_reference(raw_ref, source=source)),
("reportnumber", rb.add_report_number),
("texkey", rb.set_texkey),
("title", rb.add_title),
("url", rb.add_url),
("year", rb.set_year),
]
for field, method in mapping:
for el in force_list(reference.get(field)):
if el:
method(el)
if get_value(rb.obj, "reference.urls"):
rb.obj["reference"]["urls"] = dedupe_list_of_dicts(
rb.obj["reference"]["urls"])
result.append(rb.obj)
result.extend(rb.pop_additional_pubnotes())
return result
def map_refextract_to_schema(extracted_references, source=None):
"""Convert refextract output to the schema using the builder."""
result = []
for reference in extracted_references:
rb = ReferenceBuilder()
mapping = [
('author', rb.add_refextract_authors_str),
('collaboration', rb.add_collaboration),
('doi', rb.add_uid),
('hdl', rb.add_uid),
('isbn', rb.add_uid),
('journal_reference', rb.set_pubnote),
('linemarker', rb.set_label),
('misc', rb.add_misc),
('publisher', rb.set_publisher),
('raw_ref',
lambda raw_ref: rb.add_raw_reference(raw_ref, source=source)),
('reportnumber', rb.add_report_number),
('texkey', rb.set_texkey),
('title', rb.add_title),
('url', rb.add_url),
('year', rb.set_year),
]
for field, method in mapping:
for el in force_list(reference.get(field)):
if el:
method(el)
if get_value(rb.obj, 'reference.urls'):
rb.obj['reference']['urls'] = dedupe_list_of_dicts(
rb.obj['reference']['urls'])
result.append(rb.obj)
result.extend(rb.pop_additional_pubnotes())
return result