def refextract(obj, eng): """Perform the reference extraction step. :param obj: Bibworkflow Object to process :param eng: BibWorkflowEngine processing the object """ from invenio.legacy.refextract.api import extract_references_from_file_xml from invenio.utils.plotextractor.getter import harvest_single from invenio.modules.workflows.utils import convert_marcxml_to_bibfield if "_result" not in obj.extra_data: obj.extra_data["_result"] = {} pdf = None if "_result" in obj.extra_data and "pdf" in obj.extra_data["_result"]: pdf = obj.extra_data["_result"]["pdf"] if not pdf: extract_path = os.path.join( cfg['CFG_TMPSHAREDDIR'], str(eng.uuid) ) if not os.path.exists(extract_path): os.makedirs(extract_path) tarball, pdf = harvest_single( obj.data["system_control_number"]["value"], extract_path, ["pdf"] ) obj.extra_data["_result"]["pdf"] = pdf if pdf and os.path.isfile(obj.extra_data["_result"]["pdf"]): references_xml = extract_references_from_file_xml( obj.extra_data["_result"]["pdf"]) if references_xml: obj.log.info("Found references: {0}".format(references_xml)) updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n' \ '<collection>\n' + references_xml + \ "\n</collection>" new_dict_representation = convert_marcxml_to_bibfield(updated_xml) try: obj.data['reference'].append( new_dict_representation["reference"]) except KeyError: if 'reference' in new_dict_representation: obj.data['reference'] = [ new_dict_representation['reference']] obj.add_task_result("References", new_dict_representation['reference'], "workflows/results/refextract.html") else: obj.log.info("No references") else: obj.log.error("Not able to download and process the PDF ")
def arxiv_fulltext_download(obj, eng): """Perform the fulltext download step for arXiv records. :param obj: Bibworkflow Object to process :param eng: BibWorkflowEngine processing the object """ from invenio.utils.plotextractor.getter import harvest_single from invenio.modules.workflows.utils import convert_marcxml_to_bibfield if "result" not in obj.extra_data: obj.extra_data["_result"] = {} if "pdf" not in obj.extra_data["_result"]: extract_path = os.path.join( cfg['CFG_TMPSHAREDDIR'], str(eng.uuid) ) if not os.path.exists(extract_path): os.makedirs(extract_path) tarball, pdf = harvest_single( obj.data["system_control_number"]["value"], extract_path, ["pdf"]) arguments = obj.extra_data["repository"]["arguments"] try: if not arguments['t_doctype'] == '': doctype = arguments['t_doctype'] else: doctype = 'arXiv' except KeyError: eng.log.error("WARNING: HASARDOUS BEHAVIOUR EXPECTED, " "You didn't specified t_doctype in argument" " for fulltext_download," "try to recover by using the default one!") doctype = 'arXiv' if pdf: obj.extra_data["_result"]["pdf"] = pdf fulltext_xml = ( " <datafield tag=\"FFT\" ind1=\" \" ind2=\" \">\n" " <subfield code=\"a\">%(url)s</subfield>\n" " <subfield code=\"t\">%(doctype)s</subfield>\n" " </datafield>" ) % {'url': obj.extra_data["_result"]["pdf"], 'doctype': doctype} updated_xml = '<?xml version="1.0"?>\n' \ '<collection>\n<record>\n' + fulltext_xml + \ '</record>\n</collection>' new_dict_representation = convert_marcxml_to_bibfield(updated_xml) try: if isinstance(new_dict_representation["fft"], list): for element in new_dict_representation["fft"]: obj.data['fft'].append(element) else: obj.data['fft'].append(new_dict_representation["fft"]) except (KeyError, TypeError): obj.data['fft'] = [new_dict_representation['fft']] filename = os.path.basename(pdf) fileinfo = { "type": "Fulltext", "filename": filename, "full_path": pdf, } obj.add_task_result(filename, fileinfo, "workflows/results/files.html") else: obj.log.error("No PDF found.") else: eng.log.info("There was already a pdf register for this record," "perhaps a duplicate task in you workflow.")
def author_list(obj, eng): """Perform the special authorlist extraction step. :param obj: Bibworkflow Object to process :param eng: BibWorkflowEngine processing the object """ from invenio.legacy.oaiharvest.utils import (translate_fieldvalues_from_latex, find_matching_files) from invenio.legacy.bibrecord import create_records, record_xml_output from invenio.legacy.bibconvert.xslt_engine import convert from invenio.utils.plotextractor.cli import get_defaults from invenio.modules.workflows.utils import convert_marcxml_to_bibfield from invenio.utils.plotextractor.getter import harvest_single from invenio.modules.workflows.errors import WorkflowError from invenio.utils.plotextractor.converter import untar from invenio.utils.shell import Timeout identifiers = obj.data["system_control_number"]["value"] if "_result" not in obj.extra_data: obj.extra_data["_result"] = {} if "tarball" not in obj.extra_data["_result"]: extract_path = os.path.join( cfg['CFG_TMPSHAREDDIR'], str(eng.uuid) ) if not os.path.exists(extract_path): os.makedirs(extract_path) tarball, pdf = harvest_single( obj.data["system_control_number"]["value"], extract_path, ["tarball"]) tarball = str(tarball) if tarball is None: raise WorkflowError(str( "Error harvesting tarball from id: %s %s" % ( identifiers, extract_path)), eng.uuid, id_object=obj.id) obj.extra_data["_result"]["tarball"] = tarball sub_dir, dummy = get_defaults(obj.extra_data["_result"]["tarball"], cfg['CFG_TMPDIR'], "") try: untar(obj.extra_data["_result"]["tarball"], sub_dir) obj.log.info("Extracted tarball to: {0}".format(sub_dir)) except Timeout: eng.log.error('Timeout during tarball extraction on %s' % ( obj.extra_data["_result"]["tarball"])) xml_files_list = find_matching_files(sub_dir, ["xml"]) obj.log.info("Found xmlfiles: {0}".format(xml_files_list)) authors = "" for xml_file in xml_files_list: xml_file_fd = open(xml_file, "r") xml_content = xml_file_fd.read() xml_file_fd.close() match = REGEXP_AUTHLIST.findall(xml_content) if match: obj.log.info("Found a match for author extraction") a_stylesheet = obj.extra_data["repository"]["arguments"].get( "a_stylesheet" ) or "authorlist2marcxml.xsl" authors = convert(xml_content, a_stylesheet) authorlist_record = create_records(authors) if len(authorlist_record) == 1: if authorlist_record[0][0] is None: eng.log.error("Error parsing authorlist record for id: %s" % ( identifiers,)) authorlist_record = authorlist_record[0][0] # Convert any LaTeX symbols in authornames translate_fieldvalues_from_latex(authorlist_record, '100', code='a') translate_fieldvalues_from_latex(authorlist_record, '700', code='a') updated_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' \ + record_xml_output(authorlist_record) + '</collection>' if not None == updated_xml: # We store the path to the directory the tarball contents live # Read and grab MARCXML from plotextractor run new_dict_representation = convert_marcxml_to_bibfield(updated_xml) obj.data['authors'] = new_dict_representation["authors"] obj.data['number_of_authors'] = new_dict_representation[ "number_of_authors"] obj.add_task_result("authors", new_dict_representation["authors"]) obj.add_task_result("number_of_authors", new_dict_representation["number_of_authors"]) break
def _plot_extract(obj, eng): """Perform the plotextraction step. Download tarball for each harvested/converted record, then run plotextrator. Update converted xml files with generated xml or add it for upload. """ from invenio.utils.plotextractor.output_utils import (create_MARC, create_contextfiles, prepare_image_data, remove_dups) from invenio.utils.plotextractor.cli import (get_defaults, extract_captions, extract_context) from invenio.utils.plotextractor.converter import convert_images from invenio.utils.plotextractor.getter import harvest_single from invenio.utils.plotextractor.converter import untar from invenio.modules.workflows.errors import WorkflowError from invenio.modules.workflows.utils import convert_marcxml_to_bibfield from invenio.utils.shell import run_shell_command, Timeout if "_result" not in obj.extra_data: obj.extra_data["_result"] = {} repository = obj.extra_data.get("repository", {}) arguments = repository.get("arguments", {}) if 'p_extraction-source' not in arguments: p_extraction_source = plotextractor_types else: p_extraction_source = arguments.get('p_extraction-source', "") if not isinstance(p_extraction_source, list): p_extraction_source = [p_extraction_source] if 'latex' in p_extraction_source: # Run LaTeX plotextractor if "tarball" not in obj.extra_data["_result"]: extract_path = os.path.join( cfg['CFG_TMPSHAREDDIR'], str(eng.uuid) ) if not os.path.exists(extract_path): os.makedirs(extract_path) tarball, pdf = harvest_single( obj.data["system_control_number"]["value"], extract_path, ["tarball"]) tarball = str(tarball) if tarball is None: raise WorkflowError( str("Error harvesting tarball from id: %s %s" % (obj.data["system_control_number"]["value"], extract_path)), eng.uuid, id_object=obj.id) obj.extra_data["_result"]["tarball"] = tarball else: tarball = obj.extra_data["_result"]["tarball"] sub_dir, refno = get_defaults(tarball, cfg['CFG_TMPDIR'], "") tex_files = None image_list = None try: extracted_files_list, image_list, tex_files = untar(tarball, sub_dir) except Timeout: eng.log.error( 'Timeout during tarball extraction on %s' % (tarball,)) converted_image_list = convert_images(image_list) eng.log.info('converted %d of %d images found for %s' % ( len(converted_image_list), len(image_list), os.path.basename(tarball))) extracted_image_data = [] if tex_files == [] or tex_files is None: eng.log.error( '%s is not a tarball' % (os.path.split(tarball)[-1],)) run_shell_command('rm -r %s', (sub_dir,)) else: for tex_file in tex_files: # Extract images, captions and labels partly_extracted_image_data = extract_captions(tex_file, sub_dir, converted_image_list) if partly_extracted_image_data: # Add proper filepaths and do various cleaning cleaned_image_data = prepare_image_data( partly_extracted_image_data, tex_file, converted_image_list) # Using prev. extracted info, get contexts for each # image found extracted_image_data.extend( (extract_context(tex_file, cleaned_image_data))) if extracted_image_data: extracted_image_data = remove_dups(extracted_image_data) create_contextfiles(extracted_image_data) marc_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<collection>\n' marc_xml += create_MARC(extracted_image_data, tarball, None) marc_xml += "\n</collection>" if marc_xml: # We store the path to the directory the tarball # contents live # Read and grab MARCXML from plotextractor run new_dict = convert_marcxml_to_bibfield(marc_xml) try: if isinstance(new_dict["fft"], list): for element in new_dict["fft"]: obj.data['fft'].append(element) else: obj.data['fft'].append(new_dict["fft"]) except KeyError: obj.data['fft'] = [new_dict['fft']] obj.add_task_result("filesfft", new_dict["fft"]) obj.add_task_result("number_picture_converted", len(converted_image_list)) obj.add_task_result("number_of_picture_total", len(image_list))