def refextract(obj, eng):
"""Perform the reference extraction step.
:param obj: Bibworkflow Object to process
:param eng: BibWorkflowEngine processing the object
"""
from invenio.legacy.refextract.api import extract_references_from_file_xml
from invenio.utils.plotextractor.getter import harvest_single
from invenio.modules.workflows.utils import convert_marcxml_to_bibfield
if "_result" not in obj.extra_data:
obj.extra_data["_result"] = {}
pdf = None
if "_result" in obj.extra_data and "pdf" in obj.extra_data["_result"]:
pdf = obj.extra_data["_result"]["pdf"]
if not pdf:
extract_path = os.path.join(
cfg['CFG_TMPSHAREDDIR'],
str(eng.uuid)
)
if not os.path.exists(extract_path):
os.makedirs(extract_path)
tarball, pdf = harvest_single(
obj.data["system_control_number"]["value"], extract_path, ["pdf"]
)
obj.extra_data["_result"]["pdf"] = pdf
if pdf and os.path.isfile(obj.extra_data["_result"]["pdf"]):
references_xml = extract_references_from_file_xml(
obj.extra_data["_result"]["pdf"])
if references_xml:
obj.log.info("Found references: {0}".format(references_xml))
updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n' \
'<collection>\n' + references_xml + \
"\n</collection>"
new_dict_representation = convert_marcxml_to_bibfield(updated_xml)
try:
obj.data['reference'].append(
new_dict_representation["reference"])
except KeyError:
if 'reference' in new_dict_representation:
obj.data['reference'] = [
new_dict_representation['reference']]
obj.add_task_result("References",
new_dict_representation['reference'],
"workflows/results/refextract.html")
else:
obj.log.info("No references")
else:
obj.log.error("Not able to download and process the PDF ")
def arxiv_fulltext_download(obj, eng):
"""Perform the fulltext download step for arXiv records.
:param obj: Bibworkflow Object to process
:param eng: BibWorkflowEngine processing the object
"""
from invenio.utils.plotextractor.getter import harvest_single
from invenio.modules.workflows.utils import convert_marcxml_to_bibfield
if "result" not in obj.extra_data:
obj.extra_data["_result"] = {}
if "pdf" not in obj.extra_data["_result"]:
extract_path = os.path.join(
cfg['CFG_TMPSHAREDDIR'],
str(eng.uuid)
)
if not os.path.exists(extract_path):
os.makedirs(extract_path)
tarball, pdf = harvest_single(
obj.data["system_control_number"]["value"],
extract_path, ["pdf"])
arguments = obj.extra_data["repository"]["arguments"]
try:
if not arguments['t_doctype'] == '':
doctype = arguments['t_doctype']
else:
doctype = 'arXiv'
except KeyError:
eng.log.error("WARNING: HASARDOUS BEHAVIOUR EXPECTED, "
"You didn't specified t_doctype in argument"
" for fulltext_download,"
"try to recover by using the default one!")
doctype = 'arXiv'
if pdf:
obj.extra_data["_result"]["pdf"] = pdf
fulltext_xml = (
" <datafield tag=\"FFT\" ind1=\" \" ind2=\" \">\n"
" <subfield code=\"a\">%(url)s</subfield>\n"
" <subfield code=\"t\">%(doctype)s</subfield>\n"
" </datafield>"
) % {'url': obj.extra_data["_result"]["pdf"],
'doctype': doctype}
updated_xml = '<?xml version="1.0"?>\n' \
'<collection>\n<record>\n' + fulltext_xml + \
'</record>\n</collection>'
new_dict_representation = convert_marcxml_to_bibfield(updated_xml)
try:
if isinstance(new_dict_representation["fft"], list):
for element in new_dict_representation["fft"]:
obj.data['fft'].append(element)
else:
obj.data['fft'].append(new_dict_representation["fft"])
except (KeyError, TypeError):
obj.data['fft'] = [new_dict_representation['fft']]
filename = os.path.basename(pdf)
fileinfo = {
"type": "Fulltext",
"filename": filename,
"full_path": pdf,
}
obj.add_task_result(filename,
fileinfo,
"workflows/results/files.html")
else:
obj.log.error("No PDF found.")
else:
eng.log.info("There was already a pdf register for this record,"
"perhaps a duplicate task in you workflow.")
def author_list(obj, eng):
"""Perform the special authorlist extraction step.
:param obj: Bibworkflow Object to process
:param eng: BibWorkflowEngine processing the object
"""
from invenio.legacy.oaiharvest.utils import (translate_fieldvalues_from_latex,
find_matching_files)
from invenio.legacy.bibrecord import create_records, record_xml_output
from invenio.legacy.bibconvert.xslt_engine import convert
from invenio.utils.plotextractor.cli import get_defaults
from invenio.modules.workflows.utils import convert_marcxml_to_bibfield
from invenio.utils.plotextractor.getter import harvest_single
from invenio.modules.workflows.errors import WorkflowError
from invenio.utils.plotextractor.converter import untar
from invenio.utils.shell import Timeout
identifiers = obj.data["system_control_number"]["value"]
if "_result" not in obj.extra_data:
obj.extra_data["_result"] = {}
if "tarball" not in obj.extra_data["_result"]:
extract_path = os.path.join(
cfg['CFG_TMPSHAREDDIR'],
str(eng.uuid)
)
if not os.path.exists(extract_path):
os.makedirs(extract_path)
tarball, pdf = harvest_single(
obj.data["system_control_number"]["value"], extract_path,
["tarball"])
tarball = str(tarball)
if tarball is None:
raise WorkflowError(str(
"Error harvesting tarball from id: %s %s" % (
identifiers, extract_path)), eng.uuid, id_object=obj.id)
obj.extra_data["_result"]["tarball"] = tarball
sub_dir, dummy = get_defaults(obj.extra_data["_result"]["tarball"],
cfg['CFG_TMPDIR'], "")
try:
untar(obj.extra_data["_result"]["tarball"], sub_dir)
obj.log.info("Extracted tarball to: {0}".format(sub_dir))
except Timeout:
eng.log.error('Timeout during tarball extraction on %s' % (
obj.extra_data["_result"]["tarball"]))
xml_files_list = find_matching_files(sub_dir, ["xml"])
obj.log.info("Found xmlfiles: {0}".format(xml_files_list))
authors = ""
for xml_file in xml_files_list:
xml_file_fd = open(xml_file, "r")
xml_content = xml_file_fd.read()
xml_file_fd.close()
match = REGEXP_AUTHLIST.findall(xml_content)
if match:
obj.log.info("Found a match for author extraction")
a_stylesheet = obj.extra_data["repository"]["arguments"].get(
"a_stylesheet"
) or "authorlist2marcxml.xsl"
authors = convert(xml_content, a_stylesheet)
authorlist_record = create_records(authors)
if len(authorlist_record) == 1:
if authorlist_record[0][0] is None:
eng.log.error("Error parsing authorlist record for id: %s" % (
identifiers,))
authorlist_record = authorlist_record[0][0]
# Convert any LaTeX symbols in authornames
translate_fieldvalues_from_latex(authorlist_record, '100', code='a')
translate_fieldvalues_from_latex(authorlist_record, '700', code='a')
updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' \
+ record_xml_output(authorlist_record) + '</collection>'
if not None == updated_xml:
# We store the path to the directory the tarball contents live
# Read and grab MARCXML from plotextractor run
new_dict_representation = convert_marcxml_to_bibfield(updated_xml)
obj.data['authors'] = new_dict_representation["authors"]
obj.data['number_of_authors'] = new_dict_representation[
"number_of_authors"]
obj.add_task_result("authors", new_dict_representation["authors"])
obj.add_task_result("number_of_authors",
new_dict_representation["number_of_authors"])
break
def _plot_extract(obj, eng):
"""Perform the plotextraction step.
Download tarball for each harvested/converted record,
then run plotextrator.
Update converted xml files with generated xml or add it for upload.
"""
from invenio.utils.plotextractor.output_utils import (create_MARC,
create_contextfiles,
prepare_image_data,
remove_dups)
from invenio.utils.plotextractor.cli import (get_defaults, extract_captions,
extract_context)
from invenio.utils.plotextractor.converter import convert_images
from invenio.utils.plotextractor.getter import harvest_single
from invenio.utils.plotextractor.converter import untar
from invenio.modules.workflows.errors import WorkflowError
from invenio.modules.workflows.utils import convert_marcxml_to_bibfield
from invenio.utils.shell import run_shell_command, Timeout
if "_result" not in obj.extra_data:
obj.extra_data["_result"] = {}
repository = obj.extra_data.get("repository", {})
arguments = repository.get("arguments", {})
if 'p_extraction-source' not in arguments:
p_extraction_source = plotextractor_types
else:
p_extraction_source = arguments.get('p_extraction-source', "")
if not isinstance(p_extraction_source, list):
p_extraction_source = [p_extraction_source]
if 'latex' in p_extraction_source:
# Run LaTeX plotextractor
if "tarball" not in obj.extra_data["_result"]:
extract_path = os.path.join(
cfg['CFG_TMPSHAREDDIR'],
str(eng.uuid)
)
if not os.path.exists(extract_path):
os.makedirs(extract_path)
tarball, pdf = harvest_single(
obj.data["system_control_number"]["value"], extract_path,
["tarball"])
tarball = str(tarball)
if tarball is None:
raise WorkflowError(
str("Error harvesting tarball from id: %s %s" %
(obj.data["system_control_number"]["value"],
extract_path)),
eng.uuid,
id_object=obj.id)
obj.extra_data["_result"]["tarball"] = tarball
else:
tarball = obj.extra_data["_result"]["tarball"]
sub_dir, refno = get_defaults(tarball, cfg['CFG_TMPDIR'], "")
tex_files = None
image_list = None
try:
extracted_files_list, image_list, tex_files = untar(tarball,
sub_dir)
except Timeout:
eng.log.error(
'Timeout during tarball extraction on %s' % (tarball,))
converted_image_list = convert_images(image_list)
eng.log.info('converted %d of %d images found for %s' % (
len(converted_image_list),
len(image_list),
os.path.basename(tarball)))
extracted_image_data = []
if tex_files == [] or tex_files is None:
eng.log.error(
'%s is not a tarball' % (os.path.split(tarball)[-1],))
run_shell_command('rm -r %s', (sub_dir,))
else:
for tex_file in tex_files:
# Extract images, captions and labels
partly_extracted_image_data = extract_captions(tex_file,
sub_dir,
converted_image_list)
if partly_extracted_image_data:
# Add proper filepaths and do various cleaning
cleaned_image_data = prepare_image_data(
partly_extracted_image_data,
tex_file, converted_image_list)
# Using prev. extracted info, get contexts for each
# image found
extracted_image_data.extend(
(extract_context(tex_file, cleaned_image_data)))
if extracted_image_data:
extracted_image_data = remove_dups(extracted_image_data)
create_contextfiles(extracted_image_data)
marc_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n'
marc_xml += create_MARC(extracted_image_data, tarball, None)
marc_xml += "\n</collection>"
if marc_xml:
# We store the path to the directory the tarball
# contents live
# Read and grab MARCXML from plotextractor run
new_dict = convert_marcxml_to_bibfield(marc_xml)
try:
if isinstance(new_dict["fft"], list):
for element in new_dict["fft"]:
obj.data['fft'].append(element)
else:
obj.data['fft'].append(new_dict["fft"])
except KeyError:
obj.data['fft'] = [new_dict['fft']]
obj.add_task_result("filesfft", new_dict["fft"])
obj.add_task_result("number_picture_converted",
len(converted_image_list))
obj.add_task_result("number_of_picture_total",
len(image_list))