def _author_list(obj, eng):
from inspire.modules.converter.xslt import convert
from invenio_oaiharvester.utils import find_matching_files
model = eng.workflow_definition.model(obj)
record = get_record_from_model(model)
arxiv_id = get_arxiv_id_from_record(record)
existing_file = get_file_by_name(model, "{0}.tar.gz".format(arxiv_id))
if not existing_file:
# We download it
tarball = get_tarball_for_model(eng, arxiv_id)
if tarball is None:
obj.log.error("No tarball found")
return
add_file_by_name(model, tarball)
else:
tarball = existing_file.get_syspath()
sub_dir = os.path.abspath("{0}_files".format(tarball))
try:
file_list = untar(tarball, sub_dir)
except InvalidTarball:
obj.log.error("Invalid tarball {0}".format(tarball))
return
obj.log.info("Extracted tarball to: {0}".format(sub_dir))
xml_files_list = [filename for filename in file_list
if filename.endswith(".xml")]
obj.log.info("Found xmlfiles: {0}".format(xml_files_list))
for xml_file in xml_files_list:
xml_file_fd = open(xml_file, "r")
xml_content = xml_file_fd.read()
xml_file_fd.close()
match = REGEXP_AUTHLIST.findall(xml_content)
if match:
obj.log.info("Found a match for author extraction")
authors_xml = convert(xml_content, stylesheet)
authorlist_record = get_json_from_marcxml(authors_xml)[0]
record.update(authorlist_record)
obj.update_task_results(
"authors",
[{
"name": "authors",
"results": authorlist_record["authors"]
}]
)
obj.update_task_results(
"number_of_authors",
[{
"name": "number_of_authors",
"results": authorlist_record["number_of_authors"]
}]
)
break
model.update()
def _author_list(obj, eng):
from invenio_oaiharvester.utils import find_matching_files
from invenio_utils.plotextractor.cli import get_defaults
from invenio_utils.plotextractor.converter import untar
from invenio_utils.shell import Timeout
from inspire.modules.converter.xslt import convert
model = eng.workflow_definition.model(obj)
record = get_record_from_model(model)
arxiv_id = get_arxiv_id_from_record(record)
existing_file = get_file_by_name(model, arxiv_id)
if not existing_file:
# We download it
tarball = get_tarball_for_model(eng, arxiv_id)
if tarball is None:
obj.log.error("No tarball found")
return
add_file_by_name(model, tarball)
else:
tarball = existing_file.get_syspath()
sub_dir, dummy = get_defaults(str(tarball), cfg["CFG_TMPDIR"], "")
try:
untar(str(tarball), sub_dir)
obj.log.info("Extracted tarball to: {0}".format(sub_dir))
except Timeout:
eng.log.error("Timeout during tarball extraction on {0}".format(tarball))
xml_files_list = find_matching_files(sub_dir, ["xml"])
obj.log.info("Found xmlfiles: {0}".format(xml_files_list))
for xml_file in xml_files_list:
xml_file_fd = open(xml_file, "r")
xml_content = xml_file_fd.read()
xml_file_fd.close()
match = REGEXP_AUTHLIST.findall(xml_content)
if match:
obj.log.info("Found a match for author extraction")
authors_xml = convert(xml_content, stylesheet)
authorlist_record = get_json_from_marcxml(authors_xml)[0]
record.update(authorlist_record)
obj.update_task_results("authors", [{"name": "authors", "results": authorlist_record["authors"]}])
obj.update_task_results(
"number_of_authors",
[{"name": "number_of_authors", "results": authorlist_record["number_of_authors"]}],
)
break
model.update()
