def _author_list(obj, eng):
from invenio_oaiharvester.utils import find_matching_files
from invenio.utils.plotextractor.cli import get_defaults
from invenio.utils.plotextractor.converter import untar
from invenio.utils.shell import Timeout
from inspire.modules.converter.xslt import convert
model = eng.workflow_definition.model(obj)
record = get_record_from_model(model)
arxiv_id = get_arxiv_id_from_record(record)
existing_file = get_file_by_name(model, arxiv_id)
if not existing_file:
# We download it
tarball = get_tarball_for_model(eng, arxiv_id)
if tarball is None:
obj.log.error("No tarball found")
return
add_file_by_name(model, tarball)
else:
tarball = existing_file.get_syspath()
sub_dir, dummy = get_defaults(str(tarball), cfg["CFG_TMPDIR"], "")
try:
untar(str(tarball), sub_dir)
obj.log.info("Extracted tarball to: {0}".format(sub_dir))
except Timeout:
eng.log.error("Timeout during tarball extraction on {0}".format(tarball))
xml_files_list = find_matching_files(sub_dir, ["xml"])
obj.log.info("Found xmlfiles: {0}".format(xml_files_list))
for xml_file in xml_files_list:
xml_file_fd = open(xml_file, "r")
xml_content = xml_file_fd.read()
xml_file_fd.close()
match = REGEXP_AUTHLIST.findall(xml_content)
if match:
obj.log.info("Found a match for author extraction")
authors_xml = convert(xml_content, stylesheet)
authorlist_record = get_json_from_marcxml(authors_xml)[0]
record.update(authorlist_record)
obj.update_task_results("authors", [{"name": "authors", "results": authorlist_record["authors"]}])
obj.update_task_results(
"number_of_authors",
[{"name": "number_of_authors", "results": authorlist_record["number_of_authors"]}],
)
break
model.update()
def test_get_defaults(self):
"""plotextractor - get defaults"""
from invenio.utils.shell import run_shell_command
from invenio.utils.plotextractor.cli import get_defaults
sdir_should_be = os.path.join(cfg['CFG_TMPDIR'], self.arXiv_id + '_plots')
refno_should_be = "15" # Note: For ATLANTIS DEMO site
sdir, refno = get_defaults(tarball=self.tarball, sdir=None, refno_url=cfg['CFG_SITE_URL'])
if sdir != None:
run_shell_command("rm -rf %s" % (sdir,))
self.assertTrue(sdir == sdir_should_be, \
"didn\'t get correct default scratch dir")
self.assertTrue(refno == refno_should_be, \
'didn\'t get correct default reference number')
def test_get_defaults(self):
"""plotextractor - get defaults"""
from invenio.utils.shell import run_shell_command
from invenio.utils.plotextractor.cli import get_defaults
sdir_should_be = os.path.join(cfg['CFG_TMPDIR'],
self.arXiv_id + '_plots')
refno_should_be = "15" # Note: For ATLANTIS DEMO site
sdir, refno = get_defaults(tarball=self.tarball,
sdir=None,
refno_url=cfg['CFG_SITE_URL'])
if sdir != None:
run_shell_command("rm -rf %s" % (sdir, ))
self.assertTrue(sdir == sdir_should_be, \
"didn\'t get correct default scratch dir")
self.assertTrue(refno == refno_should_be, \
'didn\'t get correct default reference number')
def author_list(obj, eng):
"""
Performs the special authorlist extraction step (Mostly INSPIRE/CERN related).
:param obj: Bibworkflow Object to process
:param eng: BibWorkflowEngine processing the object
"""
from invenio.legacy.oaiharvest.utils import (translate_fieldvalues_from_latex,
find_matching_files)
from invenio.legacy.bibrecord import create_records, record_xml_output
from invenio.legacy.bibconvert.xslt_engine import convert
from invenio.utils.plotextractor.cli import get_defaults
identifiers = obj.data["system_number_external"]["value"]
bibtask.task_sleep_now_if_required()
if "_result" not in obj.extra_data:
obj.extra_data["_result"] = {}
if "tarball" not in obj.extra_data["_result"]:
extract_path = plotextractor_getter.make_single_directory(cfg['CFG_TMPSHAREDDIR'], eng.uuid)
tarball, pdf = plotextractor_getter.harvest_single(obj.data["system_number_external"]["value"], extract_path, ["tarball"])
tarball = str(tarball)
if tarball is None:
raise workflows_error.WorkflowError(str("Error harvesting tarball from id: %s %s" % (identifiers, extract_path)),
eng.uuid,
id_object=obj.id)
obj.extra_data["_result"]["tarball"] = tarball
sub_dir, dummy = get_defaults(obj.extra_data["_result"]["tarball"], cfg['CFG_TMPDIR'], "")
try:
untar(obj.extra_data["_result"]["tarball"], sub_dir)
except Timeout:
eng.log.error('Timeout during tarball extraction on %s' % (obj.extra_data["_result"]["tarball"]))
xml_files_list = find_matching_files(sub_dir, ["xml"])
authors = ""
for xml_file in xml_files_list:
xml_file_fd = open(xml_file, "r")
xml_content = xml_file_fd.read()
xml_file_fd.close()
match = REGEXP_AUTHLIST.findall(xml_content)
if not match == []:
authors += match[0]
# Generate file to store conversion results
if authors is not '':
authors = convert(authors, "authorlist2marcxml.xsl")
authorlist_record = create_records(authors)
if len(authorlist_record) == 1:
if authorlist_record[0][0] is None:
eng.log.error("Error parsing authorlist record for id: %s" % (identifiers,))
authorlist_record = authorlist_record[0][0]
# Convert any LaTeX symbols in authornames
translate_fieldvalues_from_latex(authorlist_record, '100', code='a')
translate_fieldvalues_from_latex(authorlist_record, '700', code='a')
updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' + record_xml_output(authorlist_record) \
+ '</collection>'
if not None == updated_xml:
# We store the path to the directory the tarball contents live
# Read and grab MARCXML from plotextractor run
new_dict_representation = records_api.create_record(updated_xml, master_format="marc").dumps()
obj.data['authors'] = new_dict_representation["authors"]
obj.data['number_of_authors'] = new_dict_representation["number_of_authors"]
obj.add_task_result("authors", new_dict_representation["authors"])
obj.add_task_result("number_of_authors", new_dict_representation["number_of_authors"])
def _plot_extract(obj, eng):
"""
Performs the plotextraction step.
"""
# Download tarball for each harvested/converted record, then run plotextrator.
# Update converted xml files with generated xml or add it for upload
bibtask.task_sleep_now_if_required()
if "_result" not in obj.extra_data:
obj.extra_data["_result"] = {}
if 'p_extraction-source' not in obj.extra_data["_repository"]["arguments"]:
p_extraction_source = plotextractor_types
else:
p_extraction_source = obj.extra_data["_repository"]["arguments"]['p_extraction-source']
if not isinstance(p_extraction_source, list):
p_extraction_source = [p_extraction_source]
if 'latex' in p_extraction_source:
# Run LaTeX plotextractor
if "tarball" not in obj.extra_data["_result"]:
# turn oaiharvest_23_1_20110214161632_converted -> oaiharvest_23_1_material
# to let harvested material in same folder structure
extract_path = plotextractor_getter.make_single_directory(cfg['CFG_TMPSHAREDDIR'], eng.uuid)
tarball, pdf = plotextractor_getter.harvest_single(obj.data["system_number_external"]["value"], extract_path, ["tarball"])
tarball = str(tarball)
if tarball is None:
raise workflows_error.WorkflowError(str("Error harvesting tarball from id: %s %s" %
(obj.data["system_number_external"]["value"], extract_path)),
eng.uuid,
id_object=obj.id)
obj.extra_data["_result"]["tarball"] = tarball
else:
tarball = obj.extra_data["_result"]["tarball"]
sub_dir, refno = get_defaults(tarball, cfg['CFG_TMPDIR'], "")
tex_files = None
image_list = None
try:
extracted_files_list, image_list, tex_files = untar(tarball, sub_dir)
except Timeout:
eng.log.error('Timeout during tarball extraction on %s' % (tarball,))
converted_image_list = convert_images(image_list)
eng.log.info('converted %d of %d images found for %s' % (len(converted_image_list),
len(image_list),
os.path.basename(tarball)))
extracted_image_data = []
if tex_files == [] or tex_files is None:
eng.log.error('%s is not a tarball' % (os.path.split(tarball)[-1],))
run_shell_command('rm -r %s', (sub_dir,))
else:
for tex_file in tex_files:
# Extract images, captions and labels
partly_extracted_image_data = extract_captions(tex_file, sub_dir,
converted_image_list)
if partly_extracted_image_data:
# Add proper filepaths and do various cleaning
cleaned_image_data = prepare_image_data(partly_extracted_image_data,
tex_file, converted_image_list)
# Using prev. extracted info, get contexts for each image found
extracted_image_data.extend((extract_context(tex_file, cleaned_image_data)))
if extracted_image_data:
extracted_image_data = remove_dups(extracted_image_data)
create_contextfiles(extracted_image_data)
marc_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n'
marc_xml += create_MARC(extracted_image_data, tarball, None)
marc_xml += "\n</collection>"
if marc_xml:
# We store the path to the directory the tarball contents live
# Read and grab MARCXML from plotextractor run
new_dict_representation = records_api.create_record(marc_xml,
master_format="marc").dumps()
try:
obj.data['fft'].append(new_dict_representation["fft"])
except KeyError:
obj.data['fft'] = [new_dict_representation['fft']]
obj.add_task_result("filesfft", new_dict_representation["fft"])
obj.add_task_result("number_picture_converted", len(converted_image_list))
obj.add_task_result("number_of_picture_total", len(image_list))
def _author_list(obj, eng):
from invenio.legacy.bibrecord import create_records, record_xml_output
from invenio.legacy.bibconvert.xslt_engine import convert
from invenio.utils.plotextractor.api import get_tarball_from_arxiv
from invenio.utils.plotextractor.cli import get_defaults
from invenio.modules.workflows.utils import convert_marcxml_to_bibfield
from invenio.utils.plotextractor.converter import untar
from invenio.utils.shell import Timeout
from ..utils import find_matching_files
identifiers = obj.data.get(
cfg.get('OAIHARVESTER_RECORD_ARXIV_ID_LOOKUP'), "")
if "_result" not in obj.extra_data:
obj.extra_data["_result"] = {}
if "tarball" not in obj.extra_data["_result"]:
extract_path = os.path.join(
cfg.get('OAIHARVESTER_STORAGEDIR',
cfg.get('CFG_TMPSHAREDDIR')), str(eng.uuid))
tarball = get_tarball_from_arxiv(
obj.data.get(cfg.get('OAIHARVESTER_RECORD_ARXIV_ID_LOOKUP')),
extract_path)
if tarball is None:
obj.log.error("No tarball found")
return
else:
tarball = obj.extra_data["_result"]["tarball"]
# FIXME
tarball = str(tarball)
sub_dir, dummy = get_defaults(tarball, cfg['CFG_TMPDIR'], "")
try:
untar(tarball, sub_dir)
obj.log.info("Extracted tarball to: {0}".format(sub_dir))
except Timeout:
eng.log.error('Timeout during tarball extraction on %s' %
(obj.extra_data["_result"]["tarball"]))
xml_files_list = find_matching_files(sub_dir, ["xml"])
obj.log.info("Found xmlfiles: {0}".format(xml_files_list))
authors = ""
for xml_file in xml_files_list:
xml_file_fd = open(xml_file, "r")
xml_content = xml_file_fd.read()
xml_file_fd.close()
match = REGEXP_AUTHLIST.findall(xml_content)
if match:
obj.log.info("Found a match for author extraction")
authors = convert(xml_content, stylesheet)
authorlist_record = create_records(authors)
if len(authorlist_record) == 1:
if authorlist_record[0][0] is None:
eng.log.error(
"Error parsing authorlist record for id: %s" %
(identifiers, ))
authorlist_record = authorlist_record[0][0]
author_xml = record_xml_output(authorlist_record)
if author_xml:
updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' \
+ record_xml_output(authorlist_record) + '</collection>'
new_dict_representation = convert_marcxml_to_bibfield(
updated_xml)
obj.data["authors"] = new_dict_representation["authors"]
obj.update_task_results(
"authors",
[{
"name": "authors",
"results": new_dict_representation["authors"]
}])
obj.update_task_results("number_of_authors", [{
"name":
"number_of_authors",
"results":
new_dict_representation["number_of_authors"]
}])
break
def _author_list(obj, eng):
from invenio.legacy.bibrecord import create_records, record_xml_output
from invenio.legacy.bibconvert.xslt_engine import convert
from invenio.utils.plotextractor.api import get_tarball_from_arxiv
from invenio.utils.plotextractor.cli import get_defaults
from invenio.modules.workflows.utils import convert_marcxml_to_bibfield
from invenio.utils.plotextractor.converter import untar
from invenio.utils.shell import Timeout
from ..utils import find_matching_files
identifiers = obj.data.get(cfg.get('OAIHARVESTER_RECORD_ARXIV_ID_LOOKUP'), "")
if "_result" not in obj.extra_data:
obj.extra_data["_result"] = {}
if "tarball" not in obj.extra_data["_result"]:
extract_path = os.path.join(
cfg.get('OAIHARVESTER_STORAGEDIR', cfg.get('CFG_TMPSHAREDDIR')),
str(eng.uuid)
)
tarball = get_tarball_from_arxiv(
obj.data.get(cfg.get('OAIHARVESTER_RECORD_ARXIV_ID_LOOKUP')),
extract_path
)
if tarball is None:
obj.log.error("No tarball found")
return
else:
tarball = obj.extra_data["_result"]["tarball"]
# FIXME
tarball = str(tarball)
sub_dir, dummy = get_defaults(tarball,
cfg['CFG_TMPDIR'], "")
try:
untar(tarball, sub_dir)
obj.log.info("Extracted tarball to: {0}".format(sub_dir))
except Timeout:
eng.log.error('Timeout during tarball extraction on %s' % (
obj.extra_data["_result"]["tarball"]))
xml_files_list = find_matching_files(sub_dir, ["xml"])
obj.log.info("Found xmlfiles: {0}".format(xml_files_list))
authors = ""
for xml_file in xml_files_list:
xml_file_fd = open(xml_file, "r")
xml_content = xml_file_fd.read()
xml_file_fd.close()
match = REGEXP_AUTHLIST.findall(xml_content)
if match:
obj.log.info("Found a match for author extraction")
authors = convert(xml_content, stylesheet)
authorlist_record = create_records(authors)
if len(authorlist_record) == 1:
if authorlist_record[0][0] is None:
eng.log.error("Error parsing authorlist record for id: %s" % (
identifiers,))
authorlist_record = authorlist_record[0][0]
author_xml = record_xml_output(authorlist_record)
if author_xml:
updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' \
+ record_xml_output(authorlist_record) + '</collection>'
new_dict_representation = convert_marcxml_to_bibfield(updated_xml)
obj.data["authors"] = new_dict_representation["authors"]
obj.update_task_results(
"authors",
[{
"name": "authors",
"results": new_dict_representation["authors"]
}]
)
obj.update_task_results(
"number_of_authors",
[{
"name": "number_of_authors",
"results": new_dict_representation["number_of_authors"]
}]
)
break
def author_list(obj, eng):
"""Perform the special authorlist extraction step.
:param obj: Bibworkflow Object to process
:param eng: BibWorkflowEngine processing the object
"""
from invenio.legacy.oaiharvest.utils import (translate_fieldvalues_from_latex,
find_matching_files)
from invenio.legacy.bibrecord import create_records, record_xml_output
from invenio.legacy.bibconvert.xslt_engine import convert
from invenio.utils.plotextractor.cli import get_defaults
from invenio.modules.workflows.utils import convert_marcxml_to_bibfield
from invenio.utils.plotextractor.getter import harvest_single
from invenio.modules.workflows.errors import WorkflowError
from invenio.utils.plotextractor.converter import untar
from invenio.utils.shell import Timeout
identifiers = obj.data["system_control_number"]["value"]
if "_result" not in obj.extra_data:
obj.extra_data["_result"] = {}
if "tarball" not in obj.extra_data["_result"]:
extract_path = os.path.join(
cfg['CFG_TMPSHAREDDIR'],
str(eng.uuid)
)
if not os.path.exists(extract_path):
os.makedirs(extract_path)
tarball, pdf = harvest_single(
obj.data["system_control_number"]["value"], extract_path,
["tarball"])
tarball = str(tarball)
if tarball is None:
raise WorkflowError(str(
"Error harvesting tarball from id: %s %s" % (
identifiers, extract_path)), eng.uuid, id_object=obj.id)
obj.extra_data["_result"]["tarball"] = tarball
sub_dir, dummy = get_defaults(obj.extra_data["_result"]["tarball"],
cfg['CFG_TMPDIR'], "")
try:
untar(obj.extra_data["_result"]["tarball"], sub_dir)
obj.log.info("Extracted tarball to: {0}".format(sub_dir))
except Timeout:
eng.log.error('Timeout during tarball extraction on %s' % (
obj.extra_data["_result"]["tarball"]))
xml_files_list = find_matching_files(sub_dir, ["xml"])
obj.log.info("Found xmlfiles: {0}".format(xml_files_list))
authors = ""
for xml_file in xml_files_list:
xml_file_fd = open(xml_file, "r")
xml_content = xml_file_fd.read()
xml_file_fd.close()
match = REGEXP_AUTHLIST.findall(xml_content)
if match:
obj.log.info("Found a match for author extraction")
a_stylesheet = obj.extra_data["repository"]["arguments"].get(
"a_stylesheet"
) or "authorlist2marcxml.xsl"
authors = convert(xml_content, a_stylesheet)
authorlist_record = create_records(authors)
if len(authorlist_record) == 1:
if authorlist_record[0][0] is None:
eng.log.error("Error parsing authorlist record for id: %s" % (
identifiers,))
authorlist_record = authorlist_record[0][0]
# Convert any LaTeX symbols in authornames
translate_fieldvalues_from_latex(authorlist_record, '100', code='a')
translate_fieldvalues_from_latex(authorlist_record, '700', code='a')
updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' \
+ record_xml_output(authorlist_record) + '</collection>'
if not None == updated_xml:
# We store the path to the directory the tarball contents live
# Read and grab MARCXML from plotextractor run
new_dict_representation = convert_marcxml_to_bibfield(updated_xml)
obj.data['authors'] = new_dict_representation["authors"]
obj.data['number_of_authors'] = new_dict_representation[
"number_of_authors"]
obj.add_task_result("authors", new_dict_representation["authors"])
obj.add_task_result("number_of_authors",
new_dict_representation["number_of_authors"])
break
def _plot_extract(obj, eng):
"""Perform the plotextraction step.
Download tarball for each harvested/converted record,
then run plotextrator.
Update converted xml files with generated xml or add it for upload.
"""
from invenio.utils.plotextractor.output_utils import (create_MARC,
create_contextfiles,
prepare_image_data,
remove_dups)
from invenio.utils.plotextractor.cli import (get_defaults, extract_captions,
extract_context)
from invenio.utils.plotextractor.converter import convert_images
from invenio.utils.plotextractor.getter import harvest_single
from invenio.utils.plotextractor.converter import untar
from invenio.modules.workflows.errors import WorkflowError
from invenio.modules.workflows.utils import convert_marcxml_to_bibfield
from invenio.utils.shell import run_shell_command, Timeout
if "_result" not in obj.extra_data:
obj.extra_data["_result"] = {}
repository = obj.extra_data.get("repository", {})
arguments = repository.get("arguments", {})
if 'p_extraction-source' not in arguments:
p_extraction_source = plotextractor_types
else:
p_extraction_source = arguments.get('p_extraction-source', "")
if not isinstance(p_extraction_source, list):
p_extraction_source = [p_extraction_source]
if 'latex' in p_extraction_source:
# Run LaTeX plotextractor
if "tarball" not in obj.extra_data["_result"]:
extract_path = os.path.join(
cfg['CFG_TMPSHAREDDIR'],
str(eng.uuid)
)
if not os.path.exists(extract_path):
os.makedirs(extract_path)
tarball, pdf = harvest_single(
obj.data["system_control_number"]["value"], extract_path,
["tarball"])
tarball = str(tarball)
if tarball is None:
raise WorkflowError(
str("Error harvesting tarball from id: %s %s" %
(obj.data["system_control_number"]["value"],
extract_path)),
eng.uuid,
id_object=obj.id)
obj.extra_data["_result"]["tarball"] = tarball
else:
tarball = obj.extra_data["_result"]["tarball"]
sub_dir, refno = get_defaults(tarball, cfg['CFG_TMPDIR'], "")
tex_files = None
image_list = None
try:
extracted_files_list, image_list, tex_files = untar(tarball,
sub_dir)
except Timeout:
eng.log.error(
'Timeout during tarball extraction on %s' % (tarball,))
converted_image_list = convert_images(image_list)
eng.log.info('converted %d of %d images found for %s' % (
len(converted_image_list),
len(image_list),
os.path.basename(tarball)))
extracted_image_data = []
if tex_files == [] or tex_files is None:
eng.log.error(
'%s is not a tarball' % (os.path.split(tarball)[-1],))
run_shell_command('rm -r %s', (sub_dir,))
else:
for tex_file in tex_files:
# Extract images, captions and labels
partly_extracted_image_data = extract_captions(tex_file,
sub_dir,
converted_image_list)
if partly_extracted_image_data:
# Add proper filepaths and do various cleaning
cleaned_image_data = prepare_image_data(
partly_extracted_image_data,
tex_file, converted_image_list)
# Using prev. extracted info, get contexts for each
# image found
extracted_image_data.extend(
(extract_context(tex_file, cleaned_image_data)))
if extracted_image_data:
extracted_image_data = remove_dups(extracted_image_data)
create_contextfiles(extracted_image_data)
marc_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n'
marc_xml += create_MARC(extracted_image_data, tarball, None)
marc_xml += "\n</collection>"
if marc_xml:
# We store the path to the directory the tarball
# contents live
# Read and grab MARCXML from plotextractor run
new_dict = convert_marcxml_to_bibfield(marc_xml)
try:
if isinstance(new_dict["fft"], list):
for element in new_dict["fft"]:
obj.data['fft'].append(element)
else:
obj.data['fft'].append(new_dict["fft"])
except KeyError:
obj.data['fft'] = [new_dict['fft']]
obj.add_task_result("filesfft", new_dict["fft"])
obj.add_task_result("number_picture_converted",
len(converted_image_list))
obj.add_task_result("number_of_picture_total",
len(image_list))