def test_load_csv_column_as_set(self):
oci_set = CSVManager.load_csv_column_as_set(self.citation_path, "oci",
4)
self.assertSetEqual(
oci_set, {
"02001000308362819371213133704040001020809-020010009063615193700006300030306151914",
"02001000002361927283705040000-02001000002361927283705030002",
"02001000002361927283705040000-020010003093612062710020603000720",
"02001000308362819371213133704040001020804-02001000308362819371213133704040000030707",
"020010000023625242110370100030001-02001010009361222251430273701090809370903040403",
"020010103003602000105370205010358000059-02001010304362801000208030304330009000400020107"
})
def extract_citations(idbaseurl,
baseurl,
python,
pclass,
input,
lookup,
data,
prefix,
agent,
source,
service,
verbose,
doi_manager,
rf_handler,
oci_to_do=None):
BASE_URL = idbaseurl
DATASET_URL = baseurl + "/" if not baseurl.endswith("/") else baseurl
oci_manager = OCIManager(lookup_file=lookup)
exi_ocis = CSVManager.load_csv_column_as_set(
data + sep + "data", "oci"
) # TODO: we need to specify carefully the dir, eg by adding an additional flag to distinguish between the files belonging to a particular process, and it should be aligned with the storer.
if oci_to_do is not None:
oci_to_do.difference_update(exi_ocis)
cit_storer = CitationStorer(data, DATASET_URL)
citations_already_present = 0
new_citations_added = 0
error_in_dois_existence = 0
cs = import_citation_source(python, pclass, input)
next_citation = cs.get_next_citation_data()
while next_citation is not None:
citing, cited, created, timespan, journal_sc, author_sc = next_citation
oci = oci_manager.get_oci(citing, cited, prefix)
oci_noprefix = oci.replace("oci:", "")
if oci_noprefix not in exi_ocis and (oci_to_do is None
or oci_noprefix in oci_to_do):
if doi_manager.is_valid(citing) and doi_manager.is_valid(cited):
if created is None:
citing_date = rf_handler.get_date(citing)
else:
citing_date = created
cited_date = rf_handler.get_date(cited)
if journal_sc is None or type(journal_sc) is not bool:
journal_sc = rf_handler.share_issn(citing, cited)
if author_sc is None or type(author_sc) is not bool:
author_sc = rf_handler.share_orcid(citing, cited)
if created is not None and timespan is not None:
cit = Citation(
oci, BASE_URL + quote(citing), None,
BASE_URL + quote(cited), None, created, timespan, 1,
agent, source,
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), service,
"doi", BASE_URL + "([[XXX__decode]])", "reference",
journal_sc, author_sc, None,
"Creation of the citation", None)
else:
cit = Citation(
oci, BASE_URL + quote(citing), citing_date,
BASE_URL + quote(cited), cited_date, None, None, 1,
agent, source,
datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), service,
"doi", BASE_URL + "([[XXX__decode]])", "reference",
journal_sc, author_sc, None,
"Creation of the citation", None)
cit_storer.store_citation(cit)
if verbose:
print(
"Create citation data for '%s' between DOI '%s' and DOI '%s'"
% (oci, citing, cited))
new_citations_added += 1
exi_ocis.add(oci_noprefix)
else:
if verbose:
print(
"WARNING: some DOIs, among '%s' and '%s', do not exist"
% (citing, cited))
error_in_dois_existence += 1
if oci_to_do is not None:
oci_to_do.remove(oci_noprefix)
else:
if verbose:
print(
"WARNING: the citation between DOI '%s' and DOI '%s' has been already processed"
% (citing, cited))
citations_already_present += 1
next_citation = cs.get_next_citation_data()
return new_citations_added, citations_already_present, error_in_dois_existence