def run_elasticsearch_containers(args: argparse.Namespace) -> None:
es_client = elasticsearch.Elasticsearch(args.fatcat_elasticsearch_url)
es_release_index = "fatcat_release"
for line in args.json_input:
line = line.strip()
if not line:
continue
entity = entity_from_json(line,
ContainerEntity,
api_client=args.api.api_client)
if entity.state != "active":
continue
if args.query_stats:
es_doc = container_to_elasticsearch(
entity,
stats=query_es_container_stats(
entity.ident,
es_client=es_client,
es_index=es_release_index,
merge_shadows=True,
),
)
else:
es_doc = container_to_elasticsearch(entity)
args.json_output.write(json.dumps(es_doc) + "\n")
def test_elasticsearch_file_transform():
with open("./tests/files/file_bcah4zp5tvdhjl5bqci2c2lgfa.json", "r") as f:
json_str = f.read()
fe = entity_from_json(json_str, FileEntity)
fe.state = "active"
es = file_to_elasticsearch(fe)
# pylint infers type of 'fe' incorrectly for some reason (as str/bytes)
assert es["sha1"] == fe.sha1 # pylint: disable=no-member
assert es["sha256"] == fe.sha256 # pylint: disable=no-member
assert es["md5"] == fe.md5 # pylint: disable=no-member
assert es["size_bytes"] == fe.size # pylint: disable=no-member
assert es["mimetype"] == fe.mimetype # pylint: disable=no-member
assert es["in_ia"] is True
assert "web" in es["rels"]
assert "www.zhros.ru" in es["hosts"]
assert "zhros.ru" in es["domains"]
assert "archive.org" in (es["hosts"] + es["domains"])
assert "web.archive.org" in (es["hosts"] + es["domains"])
# old regression
assert ".archive.org" not in (es["hosts"] + es["domains"])
def run_elasticsearch_changelogs(args: argparse.Namespace) -> None:
for line in args.json_input:
line = line.strip()
if not line:
continue
entity = entity_from_json(line,
ChangelogEntry,
api_client=args.api.api_client)
args.json_output.write(
json.dumps(changelog_to_elasticsearch(entity)) + "\n")
def test_csl_pubmed(crossref_importer: Any) -> None:
with open("tests/files/example_releases_pubmed19n0972.json", "r") as f:
# multiple single lines
for line in f:
r = entity_from_json(line, ReleaseEntity)
csl = release_to_csl(r)
citeproc_csl(csl, "csl-json")
citeproc_csl(csl, "bibtex")
citeproc_csl(csl, "harvard1")
citeproc_csl(csl, "harvard1", html=True)
def run_elasticsearch_files(args: argparse.Namespace) -> None:
for line in args.json_input:
line = line.strip()
if not line:
continue
entity = entity_from_json(line,
FileEntity,
api_client=args.api.api_client)
if entity.state != "active":
continue
args.json_output.write(
json.dumps(file_to_elasticsearch(entity)) + "\n")
def run_citeproc_releases(args: argparse.Namespace) -> None:
for line in args.json_input:
line = line.strip()
if not line:
continue
entity = entity_from_json(line,
ReleaseEntity,
api_client=args.api.api_client)
if entity.state != "active":
continue
csl_json = release_to_csl(entity)
csl_json["id"] = "release:" + (entity.ident or "unknown")
out = citeproc_csl(csl_json, args.style, args.html)
args.json_output.write(out + "\n")
def test_csl_pubmed_bibtex(crossref_importer: Any) -> None:
with open("tests/files/example_releases_pubmed19n0972.json", "r") as f:
r = entity_from_json(f.readline(), ReleaseEntity)
csl = release_to_csl(r)
print(citeproc_csl(csl, "bibtex"))
# TODO: what's with the '`' in volume?
assert (citeproc_csl(csl, "bibtex").strip() == """
@article{mędrela-kuder_szymura_2018,
title={Selected anti-health behaviours among women with osteoporosis},
volume={69`},
ISSN={0035-7715},
DOI={10.32394/rpzh.2018.0046},
abstractNote={In the prevention of osteoporosis and its treatment, it is important to prevent bone loss by reducing the occurrence of factors determining human health, which reduce the risk of osteoporosis, such as health behaviors.},
number={4},
journal={Roczniki Panstwowego Zakladu Higieny},
author={Mędrela-Kuder and Szymura},
year={2018}
}
""".strip())
assert (citeproc_csl(csl, "harvard1", html=True).strip() == """
Mędrela-Kuder and Szymura (2018) ‘Selected anti-health behaviours among women with osteoporosis’, <i>Roczniki Panstwowego Zakladu Higieny</i>, 69`(4). doi: 10.32394/rpzh.2018.0046.
""".strip())
def test_elasticsearch_changelog_transform():
ce = entity_from_json(
open("./tests/files/changelog_3469683.json", "r").read(),
ChangelogEntry)
es = changelog_to_elasticsearch(ce)
assert es["index"] == 3469683
# len("2020-01-30T05:04:39") => 19
assert es["timestamp"][:19] == "2020-01-30T05:04:39.738601Z"[:19]
assert es["editor_id"] == "scmbogxw25evtcesfcab5qaboa"
assert es["username"] == "crawl-bot"
assert es["is_bot"] is True
assert es["is_admin"] is True
assert es["agent"] == "fatcat_tools.IngestFileResultImporter"
assert es["total"] == 50
assert es["files"] == 50
assert es["new_files"] == 50
assert es["created"] == 50
assert es["releases"] == 0
assert es["new_releases"] == 0
assert es["updated"] == 0
assert es["deleted"] == 0
def test_elasticsearch_release_from_json():
r = entity_from_json(
open("./tests/files/release_etodop5banbndg3faecnfm6ozi.json",
"r").read(), ReleaseEntity)
es = release_to_elasticsearch(r)
assert es["subtitle"] == "Correpondence"
assert es["ident"] == "etodop5banbndg3faecnfm6ozi"
assert (es["container_name"] ==
"BJOG: an International Journal of Obstetrics and Gynaecology")
assert es["first_page"] == "1404"
assert es["issue"] == "11"
assert es["volume"] == "118"
assert es["number"] is None
assert es["preservation"] == "dark"
assert es["is_oa"] is False
assert es["is_longtail_oa"] is False
assert es["is_preserved"] is True
assert es["in_web"] is False
assert es["in_dweb"] is False
assert es["in_ia"] is False
assert es["in_ia_sim"] is True
assert es["in_kbart"] is True
assert es["in_jstor"] is False
# this release has a fileset, and no file
r = entity_from_json(
open("./tests/files/release_3mssw2qnlnblbk7oqyv2dafgey.json",
"r").read(), ReleaseEntity)
es = release_to_elasticsearch(r)
assert es["title"] == "Jakobshavn Glacier Bed Elevation"
assert es["ident"] == "3mssw2qnlnblbk7oqyv2dafgey"
assert es["file_count"] == 0
assert es["fileset_count"] == 1
assert es["webcapture_count"] == 0
assert es["preservation"] == "dark"
assert es["is_oa"] is True
assert es["is_longtail_oa"] is False
assert es["is_preserved"] is True
assert es["in_web"] is True
assert es["in_dweb"] is True
assert es["in_ia"] is False
assert es["in_ia_sim"] is False
assert es["in_kbart"] is False
assert es["in_jstor"] is False
# this release has a web capture, and no file (edited the JSON to remove file)
r = entity_from_json(
open("./tests/files/release_mjtqtuyhwfdr7j2c3l36uor7uy.json",
"r").read(), ReleaseEntity)
es = release_to_elasticsearch(r)
assert es["title"] == "Rethinking Personal Digital Archiving, Part 1"
assert es["ident"] == "mjtqtuyhwfdr7j2c3l36uor7uy"
assert es["file_count"] == 0
assert es["fileset_count"] == 0
assert es["webcapture_count"] == 1
assert es["preservation"] == "bright"
assert es["is_oa"] is True
assert es["is_longtail_oa"] is False
assert es["is_preserved"] is True
assert es["in_web"] is True
assert es["in_dweb"] is False
assert es["in_ia"] is True
assert es["in_ia_sim"] is False
assert es["in_kbart"] is False
assert es["in_jstor"] is False