def run_elasticsearch_releases(args: argparse.Namespace) -> None:
for line in args.json_input:
line = line.strip()
if not line:
continue
entity = entity_from_json(line,
ReleaseEntity,
api_client=args.api.api_client)
if entity.state != "active":
continue
args.json_output.write(
json.dumps(release_to_elasticsearch(entity)) + "\n")
def want_live_ingest(self, release, ingest_request):
"""
This function looks at ingest requests and decides whether they are
worth enqueing for ingest.
In theory crawling all DOIs to a landing page is valuable. It is
intended to be an operational point of control to reduce load on daily
ingest crawling (via wayback SPN).
"""
link_source = ingest_request.get('ingest_request')
ingest_type = ingest_request.get('ingest_type')
doi = ingest_request.get('ext_ids', {}).get('doi')
is_document = release.release_type in (
'article-journal',
'paper-conference',
'article',
'report',
'chapter',
'manuscript',
'review',
'thesis',
'letter',
'editorial',
'abstract',
'entry',
'patent',
'post',
'review-book',
)
is_not_pdf = release.release_type in (
'dataset',
'stub',
'software',
'figure',
'graphic',
)
# accept list sets a default "crawl it" despite OA metadata for
# known-OA DOI prefixes
in_acceptlist = False
if doi:
for prefix in self.live_pdf_ingest_doi_prefix_acceptlist:
if doi.startswith(prefix):
in_acceptlist = True
if self.ingest_oa_only and link_source not in ('arxiv', 'pmc'):
es = release_to_elasticsearch(release)
# most datacite documents are in IRs and should be crawled
is_datacite_doc = False
if release.extra and ('datacite' in release.extra) and is_document:
is_datacite_doc = True
if not (es['is_oa'] or in_acceptlist or is_datacite_doc):
return False
# if ingest_type is pdf but release_type is almost certainly not a PDF,
# skip it. This is mostly a datacite thing.
if ingest_type == "pdf" and is_not_pdf:
return False
if ingest_type == "pdf" and doi:
for prefix in self.ingest_pdf_doi_prefix_blocklist:
if doi.startswith(prefix):
return False
return True
def test_rich_elasticsearch_convert():
r = ReleaseEntity(
title="something",
release_year=1234,
license_slug="CC-BY-NC",
ext_ids=ReleaseExtIds(),
refs=[
ReleaseRef(),
ReleaseRef(target_release_id="iznnn644szdwva7khyxqzc73bi"),
],
)
r.state = "active"
r.container = ContainerEntity(
name="dummy journal",
extra={
"ia": {
"sim": {
"year_spans": [[1000, 1100]],
},
},
"kbart": {
"lockss": {
"year_spans": [[1200, 1300]],
},
"jstor": {
"year_spans": [[1000, 1300], [1950, 1960], [1980, 2005]],
},
},
"sherpa_romeo": {
"color": "blue"
},
"doaj": {
"as_of": "2010-02-03"
},
},
)
r.files = [
FileEntity(
mimetype="application/pdf",
urls=[
FileUrl(rel="dweb", url="dat://a954329dlk/thingie"),
FileUrl(
rel="webarchive",
url=
"https://web.archive.org/web/20001122030405/http://example.com",
),
FileUrl(rel="web",
url="https://archive.org/details/blah/file.pdf"),
],
extra={
"shadows": {},
},
)
]
es = release_to_elasticsearch(r)
assert es["release_year"] == r.release_year
assert es["file_count"] == 1
assert es["fileset_count"] == 0
assert es["webcapture_count"] == 0
assert es["ref_count"] == 2
assert es["ref_linked_count"] == 1
assert es["preservation"] == "bright"
assert es["is_oa"] is True
assert es["is_longtail_oa"] is False
assert es["is_preserved"] is True
assert es["in_web"] is True
assert es["in_dweb"] is True
assert es["in_ia"] is True
assert es["in_ia_sim"] is False
assert es["in_kbart"] is True
assert es["in_jstor"] is True
def test_elasticsearch_release_from_json():
r = entity_from_json(
open("./tests/files/release_etodop5banbndg3faecnfm6ozi.json",
"r").read(), ReleaseEntity)
es = release_to_elasticsearch(r)
assert es["subtitle"] == "Correpondence"
assert es["ident"] == "etodop5banbndg3faecnfm6ozi"
assert (es["container_name"] ==
"BJOG: an International Journal of Obstetrics and Gynaecology")
assert es["first_page"] == "1404"
assert es["issue"] == "11"
assert es["volume"] == "118"
assert es["number"] is None
assert es["preservation"] == "dark"
assert es["is_oa"] is False
assert es["is_longtail_oa"] is False
assert es["is_preserved"] is True
assert es["in_web"] is False
assert es["in_dweb"] is False
assert es["in_ia"] is False
assert es["in_ia_sim"] is True
assert es["in_kbart"] is True
assert es["in_jstor"] is False
# this release has a fileset, and no file
r = entity_from_json(
open("./tests/files/release_3mssw2qnlnblbk7oqyv2dafgey.json",
"r").read(), ReleaseEntity)
es = release_to_elasticsearch(r)
assert es["title"] == "Jakobshavn Glacier Bed Elevation"
assert es["ident"] == "3mssw2qnlnblbk7oqyv2dafgey"
assert es["file_count"] == 0
assert es["fileset_count"] == 1
assert es["webcapture_count"] == 0
assert es["preservation"] == "dark"
assert es["is_oa"] is True
assert es["is_longtail_oa"] is False
assert es["is_preserved"] is True
assert es["in_web"] is True
assert es["in_dweb"] is True
assert es["in_ia"] is False
assert es["in_ia_sim"] is False
assert es["in_kbart"] is False
assert es["in_jstor"] is False
# this release has a web capture, and no file (edited the JSON to remove file)
r = entity_from_json(
open("./tests/files/release_mjtqtuyhwfdr7j2c3l36uor7uy.json",
"r").read(), ReleaseEntity)
es = release_to_elasticsearch(r)
assert es["title"] == "Rethinking Personal Digital Archiving, Part 1"
assert es["ident"] == "mjtqtuyhwfdr7j2c3l36uor7uy"
assert es["file_count"] == 0
assert es["fileset_count"] == 0
assert es["webcapture_count"] == 1
assert es["preservation"] == "bright"
assert es["is_oa"] is True
assert es["is_longtail_oa"] is False
assert es["is_preserved"] is True
assert es["in_web"] is True
assert es["in_dweb"] is False
assert es["in_ia"] is True
assert es["in_ia_sim"] is False
assert es["in_kbart"] is False
assert es["in_jstor"] is False
def enrich_release_entity(entity: ReleaseEntity) -> ReleaseEntity:
if entity.state in ("redirect", "deleted"):
return entity
if entity.state == "active":
entity._es = release_to_elasticsearch(entity, force_bool=False)
if entity.container and entity.container.state == "active":
entity.container._es = container_to_elasticsearch(entity.container,
force_bool=False)
if entity.files:
# remove shadows-only files with no URLs
entity.files = [
f for f in entity.files
if not (f.extra and f.extra.get("shadows") and not f.urls)
]
if entity.filesets:
for fs in entity.filesets:
fs._total_size = sum([f.size for f in fs.manifest])
if entity.webcaptures:
for wc in entity.webcaptures:
wc._wayback_suffix = wayback_suffix(wc)
for ref in entity.refs:
# this is a UI hack to get rid of XML crud in unstructured refs like:
# LOCKSS (2014) Available: <ext-link
# xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri"
# xlink:href="http://lockss.org/"
# xlink:type="simple">http://lockss.org/</ext-link>. Accessed: 2014
# November 1.
if ref.extra and ref.extra.get("unstructured"):
ref.extra["unstructured"] = strip_extlink_xml(
ref.extra["unstructured"])
# for backwards compatibility, copy extra['subtitle'] to subtitle
if not entity.subtitle and entity.extra and entity.extra.get("subtitle"):
if isinstance(entity.extra["subtitle"], str):
entity.subtitle = entity.extra["subtitle"]
elif isinstance(entity.extra["subtitle"], list):
entity.subtitle = entity.extra["subtitle"][0] or None
# author list to display; ensure it's sorted by index (any othors with
# index=None go to end of list)
authors = [
c for c in entity.contribs if c.role in ("author", None) and (
c.surname or c.raw_name or (c.creator and c.creator.surname))
]
entity._authors = sorted(authors,
key=lambda c:
(c.index is None and 99999999) or c.index)
# need authors, title for citeproc to work
entity._can_citeproc = bool(entity._authors) and bool(entity.title)
if entity.abstracts and entity.abstracts[0].mimetype:
# hack to show plain text instead of latex abstracts
if "latex" in entity.abstracts[0].mimetype:
entity.abstracts.reverse()
# hack to (partially) clean up common JATS abstract display case
if entity.abstracts[0].mimetype == "application/xml+jats":
for tag in ("p", "jats", "jats:p"):
entity.abstracts[0].content = entity.abstracts[
0].content.replace("<{}>".format(tag), "")
entity.abstracts[0].content = entity.abstracts[
0].content.replace("</{}>".format(tag), "")
# ugh, double encoding happens
entity.abstracts[0].content = entity.abstracts[
0].content.replace("</{}>".format(tag), "")
entity.abstracts[0].content = entity.abstracts[
0].content.replace("<{}>".format(tag), "")
return entity
def want_live_ingest(self, release, ingest_request):
"""
This function looks at ingest requests and decides whether they are
worth enqueing for ingest.
In theory crawling all DOIs to a landing page is valuable. It is
intended to be an operational point of control to reduce load on daily
ingest crawling (via wayback SPN).
"""
link_source = ingest_request.get('ingest_request')
ingest_type = ingest_request.get('ingest_type')
doi = ingest_request.get('ext_ids', {}).get('doi')
es = release_to_elasticsearch(release)
is_document = release.release_type in (
'article',
'article-journal',
'article-newspaper',
'book',
'chapter',
'editorial',
'interview',
'legal_case',
'legislation',
'letter',
'manuscript',
'paper-conference',
'patent',
'peer_review',
'post',
'report',
'retraction',
'review',
'review-book',
'thesis',
)
is_not_pdf = release.release_type in (
'component',
'dataset',
'figure',
'graphic',
'software',
'stub',
)
# accept list sets a default "crawl it" despite OA metadata for
# known-OA DOI prefixes
in_acceptlist = False
if doi:
for prefix in self.live_pdf_ingest_doi_prefix_acceptlist:
if doi.startswith(prefix):
in_acceptlist = True
if self.ingest_oa_only and link_source not in ('arxiv', 'pmc'):
# most datacite documents are in IRs and should be crawled
is_datacite_doc = False
if release.extra and ('datacite' in release.extra) and is_document:
is_datacite_doc = True
if not (es['is_oa'] or in_acceptlist or is_datacite_doc):
return False
# big publishers *generally* have accurate OA metadata, use
# preservation networks, and block our crawlers. So unless OA, or
# explicitly on accept list, or not preserved, skip crawling
if es.get('publisher_type') == 'big5' and es.get(
'is_preserved') and not (es['is_oa'] or in_acceptlist):
return False
# if ingest_type is pdf but release_type is almost certainly not a PDF,
# skip it. This is mostly a datacite thing.
if ingest_type == "pdf" and is_not_pdf:
return False
if ingest_type == "pdf" and doi:
for prefix in self.ingest_pdf_doi_prefix_blocklist:
if doi.startswith(prefix):
return False
# figshare
if doi and (doi.startswith('10.6084/') or doi.startswith('10.25384/')):
# don't crawl "most recent version" (aka "group") DOIs
if not release.version:
return False
# zenodo
if doi and doi.startswith('10.5281/'):
# if this is a "grouping" DOI of multiple "version" DOIs, do not crawl (will crawl the versioned DOIs)
if release.extra and release.extra.get('relations'):
for rel in release.extra['relations']:
if (rel.get('relationType') == 'HasVersion' and rel.get(
'relatedIdentifier', '').startswith('10.5281/')):
return False
return True
def want_live_ingest(self, release: ReleaseEntity,
ingest_request: Dict[str, Any]) -> bool:
"""
This function looks at ingest requests and decides whether they are
worth enqueing for ingest.
In theory crawling all DOIs to a landing page is valuable. It is
intended to be an operational point of control to reduce load on daily
ingest crawling (via wayback SPN).
"""
link_source = ingest_request.get("ingest_request")
ingest_type = ingest_request.get("ingest_type")
doi = ingest_request.get("ext_ids", {}).get("doi")
es = release_to_elasticsearch(release)
is_document = release.release_type in (
"article",
"article-journal",
"article-newspaper",
"book",
"chapter",
"editorial",
"interview",
"legal_case",
"legislation",
"letter",
"manuscript",
"paper-conference",
"patent",
"peer_review",
"post",
"report",
"retraction",
"review",
"review-book",
"thesis",
)
is_not_pdf = release.release_type in (
"component",
"dataset",
"figure",
"graphic",
"software",
"stub",
)
# accept list sets a default "crawl it" despite OA metadata for
# known-OA DOI prefixes
in_acceptlist = False
if doi:
for prefix in self.live_pdf_ingest_doi_prefix_acceptlist:
if doi.startswith(prefix):
in_acceptlist = True
if self.ingest_oa_only and link_source not in ("arxiv", "pmc"):
# most datacite documents are in IRs and should be crawled
is_datacite_doc = False
if release.extra and ("datacite" in release.extra) and is_document:
is_datacite_doc = True
if not (es["is_oa"] or in_acceptlist or is_datacite_doc):
return False
# big publishers *generally* have accurate OA metadata, use
# preservation networks, and block our crawlers. So unless OA, or
# explicitly on accept list, or not preserved, skip crawling
if (es.get("publisher_type") == "big5" and es.get("is_preserved")
and not (es["is_oa"] or in_acceptlist)):
return False
# if ingest_type is pdf but release_type is almost certainly not a PDF,
# skip it. This is mostly a datacite thing.
if ingest_type == "pdf" and is_not_pdf:
return False
if ingest_type == "pdf" and doi:
for prefix in self.ingest_pdf_doi_prefix_blocklist:
if doi.startswith(prefix):
return False
# figshare
if doi and (doi.startswith("10.6084/") or doi.startswith("10.25384/")):
# don't crawl "most recent version" (aka "group") DOIs
if not release.version:
return False
# zenodo
if doi and doi.startswith("10.5281/"):
# if this is a "grouping" DOI of multiple "version" DOIs, do not crawl (will crawl the versioned DOIs)
if release.extra and release.extra.get("relations"):
for rel in release.extra["relations"]:
if rel.get("relationType") == "HasVersion" and rel.get(
"relatedIdentifier", "").startswith("10.5281/"):
return False
return True