def _author_list(obj, eng): from invenio_oaiharvester.utils import find_matching_files from invenio.utils.plotextractor.cli import get_defaults from invenio.utils.plotextractor.converter import untar from invenio.utils.shell import Timeout from inspire.modules.converter.xslt import convert model = eng.workflow_definition.model(obj) record = get_record_from_model(model) arxiv_id = get_arxiv_id_from_record(record) existing_file = get_file_by_name(model, arxiv_id) if not existing_file: # We download it tarball = get_tarball_for_model(eng, arxiv_id) if tarball is None: obj.log.error("No tarball found") return add_file_by_name(model, tarball) else: tarball = existing_file.get_syspath() sub_dir, dummy = get_defaults(str(tarball), cfg["CFG_TMPDIR"], "") try: untar(str(tarball), sub_dir) obj.log.info("Extracted tarball to: {0}".format(sub_dir)) except Timeout: eng.log.error("Timeout during tarball extraction on {0}".format(tarball)) xml_files_list = find_matching_files(sub_dir, ["xml"]) obj.log.info("Found xmlfiles: {0}".format(xml_files_list)) for xml_file in xml_files_list: xml_file_fd = open(xml_file, "r") xml_content = xml_file_fd.read() xml_file_fd.close() match = REGEXP_AUTHLIST.findall(xml_content) if match: obj.log.info("Found a match for author extraction") authors_xml = convert(xml_content, stylesheet) authorlist_record = get_json_from_marcxml(authors_xml)[0] record.update(authorlist_record) obj.update_task_results("authors", [{"name": "authors", "results": authorlist_record["authors"]}]) obj.update_task_results( "number_of_authors", [{"name": "number_of_authors", "results": authorlist_record["number_of_authors"]}], ) break model.update()
def test_get_defaults(self): """plotextractor - get defaults""" from invenio.utils.shell import run_shell_command from invenio.utils.plotextractor.cli import get_defaults sdir_should_be = os.path.join(cfg['CFG_TMPDIR'], self.arXiv_id + '_plots') refno_should_be = "15" # Note: For ATLANTIS DEMO site sdir, refno = get_defaults(tarball=self.tarball, sdir=None, refno_url=cfg['CFG_SITE_URL']) if sdir != None: run_shell_command("rm -rf %s" % (sdir,)) self.assertTrue(sdir == sdir_should_be, \ "didn\'t get correct default scratch dir") self.assertTrue(refno == refno_should_be, \ 'didn\'t get correct default reference number')
def test_get_defaults(self): """plotextractor - get defaults""" from invenio.utils.shell import run_shell_command from invenio.utils.plotextractor.cli import get_defaults sdir_should_be = os.path.join(cfg['CFG_TMPDIR'], self.arXiv_id + '_plots') refno_should_be = "15" # Note: For ATLANTIS DEMO site sdir, refno = get_defaults(tarball=self.tarball, sdir=None, refno_url=cfg['CFG_SITE_URL']) if sdir != None: run_shell_command("rm -rf %s" % (sdir, )) self.assertTrue(sdir == sdir_should_be, \ "didn\'t get correct default scratch dir") self.assertTrue(refno == refno_should_be, \ 'didn\'t get correct default reference number')
def author_list(obj, eng): """ Performs the special authorlist extraction step (Mostly INSPIRE/CERN related). :param obj: Bibworkflow Object to process :param eng: BibWorkflowEngine processing the object """ from invenio.legacy.oaiharvest.utils import (translate_fieldvalues_from_latex, find_matching_files) from invenio.legacy.bibrecord import create_records, record_xml_output from invenio.legacy.bibconvert.xslt_engine import convert from invenio.utils.plotextractor.cli import get_defaults identifiers = obj.data["system_number_external"]["value"] bibtask.task_sleep_now_if_required() if "_result" not in obj.extra_data: obj.extra_data["_result"] = {} if "tarball" not in obj.extra_data["_result"]: extract_path = plotextractor_getter.make_single_directory(cfg['CFG_TMPSHAREDDIR'], eng.uuid) tarball, pdf = plotextractor_getter.harvest_single(obj.data["system_number_external"]["value"], extract_path, ["tarball"]) tarball = str(tarball) if tarball is None: raise workflows_error.WorkflowError(str("Error harvesting tarball from id: %s %s" % (identifiers, extract_path)), eng.uuid, id_object=obj.id) obj.extra_data["_result"]["tarball"] = tarball sub_dir, dummy = get_defaults(obj.extra_data["_result"]["tarball"], cfg['CFG_TMPDIR'], "") try: untar(obj.extra_data["_result"]["tarball"], sub_dir) except Timeout: eng.log.error('Timeout during tarball extraction on %s' % (obj.extra_data["_result"]["tarball"])) xml_files_list = find_matching_files(sub_dir, ["xml"]) authors = "" for xml_file in xml_files_list: xml_file_fd = open(xml_file, "r") xml_content = xml_file_fd.read() xml_file_fd.close() match = REGEXP_AUTHLIST.findall(xml_content) if not match == []: authors += match[0] # Generate file to store conversion results if authors is not '': authors = convert(authors, "authorlist2marcxml.xsl") authorlist_record = create_records(authors) if len(authorlist_record) == 1: if authorlist_record[0][0] is None: eng.log.error("Error parsing authorlist record for id: %s" % (identifiers,)) authorlist_record = authorlist_record[0][0] # Convert any LaTeX symbols in authornames translate_fieldvalues_from_latex(authorlist_record, '100', code='a') translate_fieldvalues_from_latex(authorlist_record, '700', code='a') updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' + record_xml_output(authorlist_record) \ + '</collection>' if not None == updated_xml: # We store the path to the directory the tarball contents live # Read and grab MARCXML from plotextractor run new_dict_representation = records_api.create_record(updated_xml, master_format="marc").dumps() obj.data['authors'] = new_dict_representation["authors"] obj.data['number_of_authors'] = new_dict_representation["number_of_authors"] obj.add_task_result("authors", new_dict_representation["authors"]) obj.add_task_result("number_of_authors", new_dict_representation["number_of_authors"])
def _plot_extract(obj, eng): """ Performs the plotextraction step. """ # Download tarball for each harvested/converted record, then run plotextrator. # Update converted xml files with generated xml or add it for upload bibtask.task_sleep_now_if_required() if "_result" not in obj.extra_data: obj.extra_data["_result"] = {} if 'p_extraction-source' not in obj.extra_data["_repository"]["arguments"]: p_extraction_source = plotextractor_types else: p_extraction_source = obj.extra_data["_repository"]["arguments"]['p_extraction-source'] if not isinstance(p_extraction_source, list): p_extraction_source = [p_extraction_source] if 'latex' in p_extraction_source: # Run LaTeX plotextractor if "tarball" not in obj.extra_data["_result"]: # turn oaiharvest_23_1_20110214161632_converted -> oaiharvest_23_1_material # to let harvested material in same folder structure extract_path = plotextractor_getter.make_single_directory(cfg['CFG_TMPSHAREDDIR'], eng.uuid) tarball, pdf = plotextractor_getter.harvest_single(obj.data["system_number_external"]["value"], extract_path, ["tarball"]) tarball = str(tarball) if tarball is None: raise workflows_error.WorkflowError(str("Error harvesting tarball from id: %s %s" % (obj.data["system_number_external"]["value"], extract_path)), eng.uuid, id_object=obj.id) obj.extra_data["_result"]["tarball"] = tarball else: tarball = obj.extra_data["_result"]["tarball"] sub_dir, refno = get_defaults(tarball, cfg['CFG_TMPDIR'], "") tex_files = None image_list = None try: extracted_files_list, image_list, tex_files = untar(tarball, sub_dir) except Timeout: eng.log.error('Timeout during tarball extraction on %s' % (tarball,)) converted_image_list = convert_images(image_list) eng.log.info('converted %d of %d images found for %s' % (len(converted_image_list), len(image_list), os.path.basename(tarball))) extracted_image_data = [] if tex_files == [] or tex_files is None: eng.log.error('%s is not a tarball' % (os.path.split(tarball)[-1],)) run_shell_command('rm -r %s', (sub_dir,)) else: for tex_file in tex_files: # Extract images, captions and labels partly_extracted_image_data = extract_captions(tex_file, sub_dir, converted_image_list) if partly_extracted_image_data: # Add proper filepaths and do various cleaning cleaned_image_data = prepare_image_data(partly_extracted_image_data, tex_file, converted_image_list) # Using prev. extracted info, get contexts for each image found extracted_image_data.extend((extract_context(tex_file, cleaned_image_data))) if extracted_image_data: extracted_image_data = remove_dups(extracted_image_data) create_contextfiles(extracted_image_data) marc_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' marc_xml += create_MARC(extracted_image_data, tarball, None) marc_xml += "\n</collection>" if marc_xml: # We store the path to the directory the tarball contents live # Read and grab MARCXML from plotextractor run new_dict_representation = records_api.create_record(marc_xml, master_format="marc").dumps() try: obj.data['fft'].append(new_dict_representation["fft"]) except KeyError: obj.data['fft'] = [new_dict_representation['fft']] obj.add_task_result("filesfft", new_dict_representation["fft"]) obj.add_task_result("number_picture_converted", len(converted_image_list)) obj.add_task_result("number_of_picture_total", len(image_list))
def _author_list(obj, eng): from invenio.legacy.bibrecord import create_records, record_xml_output from invenio.legacy.bibconvert.xslt_engine import convert from invenio.utils.plotextractor.api import get_tarball_from_arxiv from invenio.utils.plotextractor.cli import get_defaults from invenio.modules.workflows.utils import convert_marcxml_to_bibfield from invenio.utils.plotextractor.converter import untar from invenio.utils.shell import Timeout from ..utils import find_matching_files identifiers = obj.data.get( cfg.get('OAIHARVESTER_RECORD_ARXIV_ID_LOOKUP'), "") if "_result" not in obj.extra_data: obj.extra_data["_result"] = {} if "tarball" not in obj.extra_data["_result"]: extract_path = os.path.join( cfg.get('OAIHARVESTER_STORAGEDIR', cfg.get('CFG_TMPSHAREDDIR')), str(eng.uuid)) tarball = get_tarball_from_arxiv( obj.data.get(cfg.get('OAIHARVESTER_RECORD_ARXIV_ID_LOOKUP')), extract_path) if tarball is None: obj.log.error("No tarball found") return else: tarball = obj.extra_data["_result"]["tarball"] # FIXME tarball = str(tarball) sub_dir, dummy = get_defaults(tarball, cfg['CFG_TMPDIR'], "") try: untar(tarball, sub_dir) obj.log.info("Extracted tarball to: {0}".format(sub_dir)) except Timeout: eng.log.error('Timeout during tarball extraction on %s' % (obj.extra_data["_result"]["tarball"])) xml_files_list = find_matching_files(sub_dir, ["xml"]) obj.log.info("Found xmlfiles: {0}".format(xml_files_list)) authors = "" for xml_file in xml_files_list: xml_file_fd = open(xml_file, "r") xml_content = xml_file_fd.read() xml_file_fd.close() match = REGEXP_AUTHLIST.findall(xml_content) if match: obj.log.info("Found a match for author extraction") authors = convert(xml_content, stylesheet) authorlist_record = create_records(authors) if len(authorlist_record) == 1: if authorlist_record[0][0] is None: eng.log.error( "Error parsing authorlist record for id: %s" % (identifiers, )) authorlist_record = authorlist_record[0][0] author_xml = record_xml_output(authorlist_record) if author_xml: updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' \ + record_xml_output(authorlist_record) + '</collection>' new_dict_representation = convert_marcxml_to_bibfield( updated_xml) obj.data["authors"] = new_dict_representation["authors"] obj.update_task_results( "authors", [{ "name": "authors", "results": new_dict_representation["authors"] }]) obj.update_task_results("number_of_authors", [{ "name": "number_of_authors", "results": new_dict_representation["number_of_authors"] }]) break
def _author_list(obj, eng): from invenio.legacy.bibrecord import create_records, record_xml_output from invenio.legacy.bibconvert.xslt_engine import convert from invenio.utils.plotextractor.api import get_tarball_from_arxiv from invenio.utils.plotextractor.cli import get_defaults from invenio.modules.workflows.utils import convert_marcxml_to_bibfield from invenio.utils.plotextractor.converter import untar from invenio.utils.shell import Timeout from ..utils import find_matching_files identifiers = obj.data.get(cfg.get('OAIHARVESTER_RECORD_ARXIV_ID_LOOKUP'), "") if "_result" not in obj.extra_data: obj.extra_data["_result"] = {} if "tarball" not in obj.extra_data["_result"]: extract_path = os.path.join( cfg.get('OAIHARVESTER_STORAGEDIR', cfg.get('CFG_TMPSHAREDDIR')), str(eng.uuid) ) tarball = get_tarball_from_arxiv( obj.data.get(cfg.get('OAIHARVESTER_RECORD_ARXIV_ID_LOOKUP')), extract_path ) if tarball is None: obj.log.error("No tarball found") return else: tarball = obj.extra_data["_result"]["tarball"] # FIXME tarball = str(tarball) sub_dir, dummy = get_defaults(tarball, cfg['CFG_TMPDIR'], "") try: untar(tarball, sub_dir) obj.log.info("Extracted tarball to: {0}".format(sub_dir)) except Timeout: eng.log.error('Timeout during tarball extraction on %s' % ( obj.extra_data["_result"]["tarball"])) xml_files_list = find_matching_files(sub_dir, ["xml"]) obj.log.info("Found xmlfiles: {0}".format(xml_files_list)) authors = "" for xml_file in xml_files_list: xml_file_fd = open(xml_file, "r") xml_content = xml_file_fd.read() xml_file_fd.close() match = REGEXP_AUTHLIST.findall(xml_content) if match: obj.log.info("Found a match for author extraction") authors = convert(xml_content, stylesheet) authorlist_record = create_records(authors) if len(authorlist_record) == 1: if authorlist_record[0][0] is None: eng.log.error("Error parsing authorlist record for id: %s" % ( identifiers,)) authorlist_record = authorlist_record[0][0] author_xml = record_xml_output(authorlist_record) if author_xml: updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' \ + record_xml_output(authorlist_record) + '</collection>' new_dict_representation = convert_marcxml_to_bibfield(updated_xml) obj.data["authors"] = new_dict_representation["authors"] obj.update_task_results( "authors", [{ "name": "authors", "results": new_dict_representation["authors"] }] ) obj.update_task_results( "number_of_authors", [{ "name": "number_of_authors", "results": new_dict_representation["number_of_authors"] }] ) break
def author_list(obj, eng): """Perform the special authorlist extraction step. :param obj: Bibworkflow Object to process :param eng: BibWorkflowEngine processing the object """ from invenio.legacy.oaiharvest.utils import (translate_fieldvalues_from_latex, find_matching_files) from invenio.legacy.bibrecord import create_records, record_xml_output from invenio.legacy.bibconvert.xslt_engine import convert from invenio.utils.plotextractor.cli import get_defaults from invenio.modules.workflows.utils import convert_marcxml_to_bibfield from invenio.utils.plotextractor.getter import harvest_single from invenio.modules.workflows.errors import WorkflowError from invenio.utils.plotextractor.converter import untar from invenio.utils.shell import Timeout identifiers = obj.data["system_control_number"]["value"] if "_result" not in obj.extra_data: obj.extra_data["_result"] = {} if "tarball" not in obj.extra_data["_result"]: extract_path = os.path.join( cfg['CFG_TMPSHAREDDIR'], str(eng.uuid) ) if not os.path.exists(extract_path): os.makedirs(extract_path) tarball, pdf = harvest_single( obj.data["system_control_number"]["value"], extract_path, ["tarball"]) tarball = str(tarball) if tarball is None: raise WorkflowError(str( "Error harvesting tarball from id: %s %s" % ( identifiers, extract_path)), eng.uuid, id_object=obj.id) obj.extra_data["_result"]["tarball"] = tarball sub_dir, dummy = get_defaults(obj.extra_data["_result"]["tarball"], cfg['CFG_TMPDIR'], "") try: untar(obj.extra_data["_result"]["tarball"], sub_dir) obj.log.info("Extracted tarball to: {0}".format(sub_dir)) except Timeout: eng.log.error('Timeout during tarball extraction on %s' % ( obj.extra_data["_result"]["tarball"])) xml_files_list = find_matching_files(sub_dir, ["xml"]) obj.log.info("Found xmlfiles: {0}".format(xml_files_list)) authors = "" for xml_file in xml_files_list: xml_file_fd = open(xml_file, "r") xml_content = xml_file_fd.read() xml_file_fd.close() match = REGEXP_AUTHLIST.findall(xml_content) if match: obj.log.info("Found a match for author extraction") a_stylesheet = obj.extra_data["repository"]["arguments"].get( "a_stylesheet" ) or "authorlist2marcxml.xsl" authors = convert(xml_content, a_stylesheet) authorlist_record = create_records(authors) if len(authorlist_record) == 1: if authorlist_record[0][0] is None: eng.log.error("Error parsing authorlist record for id: %s" % ( identifiers,)) authorlist_record = authorlist_record[0][0] # Convert any LaTeX symbols in authornames translate_fieldvalues_from_latex(authorlist_record, '100', code='a') translate_fieldvalues_from_latex(authorlist_record, '700', code='a') updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' \ + record_xml_output(authorlist_record) + '</collection>' if not None == updated_xml: # We store the path to the directory the tarball contents live # Read and grab MARCXML from plotextractor run new_dict_representation = convert_marcxml_to_bibfield(updated_xml) obj.data['authors'] = new_dict_representation["authors"] obj.data['number_of_authors'] = new_dict_representation[ "number_of_authors"] obj.add_task_result("authors", new_dict_representation["authors"]) obj.add_task_result("number_of_authors", new_dict_representation["number_of_authors"]) break
def _plot_extract(obj, eng): """Perform the plotextraction step. Download tarball for each harvested/converted record, then run plotextrator. Update converted xml files with generated xml or add it for upload. """ from invenio.utils.plotextractor.output_utils import (create_MARC, create_contextfiles, prepare_image_data, remove_dups) from invenio.utils.plotextractor.cli import (get_defaults, extract_captions, extract_context) from invenio.utils.plotextractor.converter import convert_images from invenio.utils.plotextractor.getter import harvest_single from invenio.utils.plotextractor.converter import untar from invenio.modules.workflows.errors import WorkflowError from invenio.modules.workflows.utils import convert_marcxml_to_bibfield from invenio.utils.shell import run_shell_command, Timeout if "_result" not in obj.extra_data: obj.extra_data["_result"] = {} repository = obj.extra_data.get("repository", {}) arguments = repository.get("arguments", {}) if 'p_extraction-source' not in arguments: p_extraction_source = plotextractor_types else: p_extraction_source = arguments.get('p_extraction-source', "") if not isinstance(p_extraction_source, list): p_extraction_source = [p_extraction_source] if 'latex' in p_extraction_source: # Run LaTeX plotextractor if "tarball" not in obj.extra_data["_result"]: extract_path = os.path.join( cfg['CFG_TMPSHAREDDIR'], str(eng.uuid) ) if not os.path.exists(extract_path): os.makedirs(extract_path) tarball, pdf = harvest_single( obj.data["system_control_number"]["value"], extract_path, ["tarball"]) tarball = str(tarball) if tarball is None: raise WorkflowError( str("Error harvesting tarball from id: %s %s" % (obj.data["system_control_number"]["value"], extract_path)), eng.uuid, id_object=obj.id) obj.extra_data["_result"]["tarball"] = tarball else: tarball = obj.extra_data["_result"]["tarball"] sub_dir, refno = get_defaults(tarball, cfg['CFG_TMPDIR'], "") tex_files = None image_list = None try: extracted_files_list, image_list, tex_files = untar(tarball, sub_dir) except Timeout: eng.log.error( 'Timeout during tarball extraction on %s' % (tarball,)) converted_image_list = convert_images(image_list) eng.log.info('converted %d of %d images found for %s' % ( len(converted_image_list), len(image_list), os.path.basename(tarball))) extracted_image_data = [] if tex_files == [] or tex_files is None: eng.log.error( '%s is not a tarball' % (os.path.split(tarball)[-1],)) run_shell_command('rm -r %s', (sub_dir,)) else: for tex_file in tex_files: # Extract images, captions and labels partly_extracted_image_data = extract_captions(tex_file, sub_dir, converted_image_list) if partly_extracted_image_data: # Add proper filepaths and do various cleaning cleaned_image_data = prepare_image_data( partly_extracted_image_data, tex_file, converted_image_list) # Using prev. extracted info, get contexts for each # image found extracted_image_data.extend( (extract_context(tex_file, cleaned_image_data))) if extracted_image_data: extracted_image_data = remove_dups(extracted_image_data) create_contextfiles(extracted_image_data) marc_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' marc_xml += create_MARC(extracted_image_data, tarball, None) marc_xml += "\n</collection>" if marc_xml: # We store the path to the directory the tarball # contents live # Read and grab MARCXML from plotextractor run new_dict = convert_marcxml_to_bibfield(marc_xml) try: if isinstance(new_dict["fft"], list): for element in new_dict["fft"]: obj.data['fft'].append(element) else: obj.data['fft'].append(new_dict["fft"]) except KeyError: obj.data['fft'] = [new_dict['fft']] obj.add_task_result("filesfft", new_dict["fft"]) obj.add_task_result("number_picture_converted", len(converted_image_list)) obj.add_task_result("number_of_picture_total", len(image_list))