def test_format_record(self): """ bibformat - correct formatting""" #use output format that has no match TEST DISABLED DURING MIGRATION #result = bibformat_engine.format_record(recID=None, of="test2", xml_record=self.xml_text_2) #self.assertEqual(result.replace("\n", ""),"") #use output format that link to unknown template result = bibformat_engine.format_record(recID=None, of="test3", xml_record=self.xml_text_2) self.assertEqual(result.replace("\n", ""), "") #Unknown output format TEST DISABLED DURING MIGRATION #result = bibformat_engine.format_record(recID=None, of="unkno", xml_record=self.xml_text_3) #self.assertEqual(result.replace("\n", ""),"") #Default formatting result = bibformat_engine.format_record(recID=None, ln='fr', of="test3", xml_record=self.xml_text_3) self.assertEqual( result, '''<h1>hi</h1> this is my template\ntest<bfe_non_existing_element must disappear/><test_1 non prefixed element must stay as any normal tag/>tfrgarbage\n<br/>test me!<b>ok</b>a default valueeditor\n<br/>test me!<b>ok</b>a default valueeditor\n<br/>test me!<b>ok</b>a default valueeditor\n''' )
def test_format_record(self): """ bibformat - correct formatting""" bibformat_engine.CFG_BIBFORMAT_OUTPUTS_PATH = CFG_BIBFORMAT_OUTPUTS_PATH bibformat_engine.CFG_BIBFORMAT_ELEMENTS_PATH = CFG_BIBFORMAT_ELEMENTS_PATH bibformat_engine.CFG_BIBFORMAT_ELEMENTS_IMPORT_PATH = CFG_BIBFORMAT_ELEMENTS_IMPORT_PATH bibformat_engine.CFG_BIBFORMAT_TEMPLATES_PATH = CFG_BIBFORMAT_TEMPLATES_PATH # use output format that has no match TEST DISABLED DURING MIGRATION # result = bibformat_engine.format_record(recID=None, of="test2", xml_record=self.xml_text_2) # self.assertEqual(result.replace("\n", ""),"") # use output format that link to unknown template result = bibformat_engine.format_record(recID=None, of="test3", xml_record=self.xml_text_2) self.assertEqual(result.replace("\n", ""), "") # Unknown output format TEST DISABLED DURING MIGRATION # result = bibformat_engine.format_record(recID=None, of="unkno", xml_record=self.xml_text_3) # self.assertEqual(result.replace("\n", ""),"") # Default formatting result = bibformat_engine.format_record(recID=None, ln="fr", of="test3", xml_record=self.xml_text_3) self.assertEqual( result, """<h1>hi</h1> this is my template\ntest<bfe_non_existing_element must disappear/><test_1 non prefixed element must stay as any normal tag/>tfrgarbage\n<br/>test me!<b>ok</b>a default valueeditor\n<br/>test me!<b>ok</b>a default valueeditor\n<br/>test me!<b>ok</b>a default valueeditor\n""", )
def clean(r) : #print r fr = format_record(r, "hlxu", "fr") fr = format_record(r, "hlxu") #print fr fr = fr[0] fr = fr.replace('<br>','\n') fr = fr.replace('<pre>','\n') fr = fr.replace('<br />','\n') fr = fr.replace('</pre>','\n') fr = fr.replace(' ','') fr = fr.replace('%``','``') return fr
def clean(r): #print r fr = format_record(r, "hlxu", "fr") fr = format_record(r, "tlcv") #print fr fr = fr[0] fr = fr.replace('<br>', '\n') fr = fr.replace('<pre>', '\n') fr = fr.replace('<br />', '\n') fr = fr.replace('</pre>', '\n') fr = fr.replace(' ', '') fr = fr.replace('%``', '``') return fr
def afs_sync(modified_records, time_estimator, tot, now): """Sync to AFS.""" write_message("Appending output to %s" % CFG_OUTPUT_PATH) prodsyncname = CFG_OUTPUT_PATH + now.strftime("%Y%m%d%H%M%S") + '.xml.gz' r = gzip.open(prodsyncname, "w") print >> r, '<collection xmlns="http://www.loc.gov/MARC21/slim">' for i, recid in enumerate(modified_records): record = format_record(recid, 'xme', user_info=ADMIN_USER_INFO)[0] if not record: write_message("Error formatting record {0} as 'xme': {1}".format( recid, record )) else: print >> r, record if shall_sleep(recid, i, tot, time_estimator): r.flush() task_sleep_now_if_required() print >> r, '</collection>' r.close() prodsync_tarname = CFG_OUTPUT_PATH + '.tar' write_message("Adding %s to %s" % (prodsyncname, prodsync_tarname)) prodsync_tar = tarfile.open(prodsync_tarname, 'a') prodsync_tar.add(prodsyncname) prodsync_tar.close() os.remove(prodsyncname)
def test_empty_formatting(self): """bibformat - formatting empty record""" result = bibformat_engine.format_record(recID=0, of='hb', verbose=9, xml_record=self.empty_record_xml) self.assertEqual(result, '')
def afs_sync(modified_records, time_estimator, tot, now): """Sync to AFS.""" write_message("Appending output to %s" % CFG_OUTPUT_PATH) prodsyncname = CFG_OUTPUT_PATH + now.strftime("%Y%m%d%H%M%S") + '.xml.gz' r = gzip.open(prodsyncname, "w") print >> r, '<collection xmlns="http://www.loc.gov/MARC21/slim">' for i, recid in enumerate(modified_records): with run_ro_on_slave_db(): record = format_record(recid, 'xme', user_info=ADMIN_USER_INFO)[0] if not record: write_message("Error formatting record {0} as 'xme': {1}".format( recid, record)) else: print >> r, record if shall_sleep(recid, i, tot, time_estimator): r.flush() task_sleep_now_if_required() print >> r, '</collection>' r.close() prodsync_tarname = CFG_OUTPUT_PATH + '.tar' write_message("Adding %s to %s" % (prodsyncname, prodsync_tarname)) prodsync_tar = tarfile.open(prodsync_tarname, 'a') prodsync_tar.add(prodsyncname) prodsync_tar.close() os.remove(prodsyncname)
def test_format_translations_no_2nd_pass_en(self): result, needs_2nd_pass = bibformat_engine.format_record( recID=None, of="test7", xml_record=self.xml_text_2, ln='en') self.assertEqual(result.strip(), 'Title en\n<input type="button" value="Record"/>') self.assertEqual(needs_2nd_pass, False)
def test_format_translations_no_2nd_pass_fr(self): ln = 'fr' result, needs_2nd_pass = bibformat_engine.format_record( recID=None, of="test7", xml_record=self.xml_text_2, ln=ln) _ = gettext_set_language(ln) self.assertEqual( result.strip(), 'Titre fr\n<input type="button" value="%s"/>' % _('Record')) self.assertEqual(needs_2nd_pass, False)
def test_format_2_passes_manually(self): result, needs_2nd_pass = bibformat_engine.format_record( recID=None, of="test6", xml_record=self.xml_text_2) self.assertEqual(result, "<bfe_test_6 />\n") self.assertEqual(needs_2nd_pass, True) out = bibformat_engine.format_record_2nd_pass(recID=None, template=result) self.assertEqual(out, "helloworld\n")
def test_format_translations_no_2nd_pass_fr(self): ln = 'fr' result, needs_2nd_pass = bibformat_engine.format_record( recID=None, of="test7", xml_record=self.xml_text_2, ln=ln) _ = gettext_set_language(ln) self.assertEqual(result.strip(), 'Titre fr\n<input type="button" value="%s"/>' % _('Record')) self.assertEqual(needs_2nd_pass, False)
def test_format_record(self): """ bibformat - correct formatting""" #use output format that has no match TEST DISABLED DURING MIGRATION #result = bibformat_engine.format_record(recID=None, of="test2", xml_record=self.xml_text_2) #self.assertEqual(result.replace("\n", ""),"") #use output format that link to unknown template result, needs_2nd_pass = bibformat_engine.format_record(recID=None, of="test3", xml_record=self.xml_text_2) self.assertEqual(result.replace("\n", ""), "") self.assertEqual(needs_2nd_pass, False) #Unknown output format TEST DISABLED DURING MIGRATION #result = bibformat_engine.format_record(recID=None, of="unkno", xml_record=self.xml_text_3) #self.assertEqual(result.replace("\n", ""),"") #Default formatting result, needs_2nd_pass = bibformat_engine.format_record(recID=None, ln='fr', of="test3", xml_record=self.xml_text_3) self.assertEqual(result, '''<h1>hi</h1> this is my template\ntest<bfe_non_existing_element must disappear/><test_1 non prefixed element must stay as any normal tag/>tfrgarbage\n<br/>test me!<b>ok</b>a default valueeditor\n<br/>test me!<b>ok</b>a default valueeditor\n<br/>test me!<b>ok</b>a default valueeditor\n''') self.assertEqual(needs_2nd_pass, False)
def bst_dump_records(): try: os.makedirs(os.path.join(CFG_WEBDIR, 'dumps')) except OSError: pass html_index = open(os.path.join(CFG_WEBDIR, 'dumps', '.inspire-dump.html'), "w") print >> html_index, "<html><head><title>INSPIRE Dump</title></head><body><ul>" for collection in CFG_EXPORTED_COLLECTIONS: task_update_progress(collection) print >> html_index, """ <li><a href="%(prefix)s/dumps/%(collection)s-records.xml.gz">%(collection)s</a> (<a href="%(prefix)s/dumps/%(collection)s-records.xml.gz.md5">MD5</a>): %(date)s</li>""" % { 'prefix': CFG_SITE_URL, 'collection': collection, 'date': time.ctime() } write_message("Preparing %s-records.xml.gz" % collection) output_path = os.path.join(CFG_WEBDIR, 'dumps', '.%s-records.xml.gz' % collection) output = gzip.open(output_path, "w") print >> output, "<collection>" reclist = get_collection_reclist(collection) tot = len(reclist) time_estimator = get_time_estimator(tot) for i, recid in enumerate(reclist): with run_ro_on_slave_db(): print >> output, format_record(recid, 'xme', user_info={})[0] time_estimation = time_estimator()[1] if (i + 1) % 100 == 0: task_update_progress( "%s %s (%s%%) -> %s" % (collection, recid, (i + 1) * 100 / tot, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time_estimation)))) task_sleep_now_if_required() print >> output, "</collection>" output.close() write_message("Computing checksum") print >> open(output_path + '.md5', "w"), calculate_md5(output_path) os.rename( output_path, os.path.join(CFG_WEBDIR, 'dumps', '%s-records.xml.gz' % collection)) os.rename( output_path + '.md5', os.path.join(CFG_WEBDIR, 'dumps', '%s-records.xml.gz.md5' % collection)) write_message("DONE") print >> html_index, "</ul></body></html>" html_index.close() os.rename(os.path.join(CFG_WEBDIR, 'dumps', '.inspire-dump.html'), os.path.join(CFG_WEBDIR, 'dumps', 'inspire-dump.html'))
def redis_sync(modified_records, time_estimator, tot): """Sync to redis.""" r = redis.StrictRedis.from_url(CFG_REDIS_HOST_LABS) for i, recid in enumerate(modified_records): with run_ro_on_slave_db(): record = format_record(recid, 'xme', user_info=ADMIN_USER_INFO)[0] if not record: write_message("Error formatting record {0} as 'xme': {1}".format( recid, record)) else: r.rpush('legacy_records', zlib.compress(record)) if shall_sleep(recid, i, tot, time_estimator): task_sleep_now_if_required()
def redis_sync(modified_records, time_estimator, tot): """Sync to redis.""" r = redis.StrictRedis.from_url(CFG_REDIS_HOST_LABS) for i, recid in enumerate(modified_records): record = format_record(recid, 'xme', user_info=ADMIN_USER_INFO)[0] if not record: write_message("Error formatting record {0} as 'xme': {1}".format( recid, record )) else: r.rpush('legacy_records', zlib.compress(record)) if shall_sleep(recid, i, tot, time_estimator): task_sleep_now_if_required()
def test_format_translations_with_2nd_pass_en(self): result, needs_2nd_pass = bibformat_engine.format_record( recID=None, of="test8", xml_record=self.xml_text_2, ln='en') self.assertEqual(result.strip(), '<lang>\n <en>Title en</en>\n <fr>Titre fr</fr>\n</lang>\n<bfe_test_6 />\n<input type="button" value="_(Record)_"/>') self.assertEqual(needs_2nd_pass, True) out = bibformat_engine.format_record_2nd_pass(recID=None, template=result, ln='en') self.assertEqual(out, 'Title en\nhelloworld\n<input type="button" value="Record"/>')
def test_format_translations_with_2nd_pass_en(self): result, needs_2nd_pass = bibformat_engine.format_record( recID=None, of="test8", xml_record=self.xml_text_2, ln='en') self.assertEqual( result.strip(), '<lang>\n <en>Title en</en>\n <fr>Titre fr</fr>\n</lang>\n<bfe_test_6 />\n<input type="button" value="_(Record)_"/>' ) self.assertEqual(needs_2nd_pass, True) out = bibformat_engine.format_record_2nd_pass(recID=None, template=result, ln='en') self.assertEqual( out, 'Title en\nhelloworld\n<input type="button" value="Record"/>')
def bst_dump_records(): try: os.makedirs(os.path.join(CFG_WEBDIR, "dumps")) except OSError: pass html_index = open(os.path.join(CFG_WEBDIR, "dumps", ".inspire-dump.html"), "w") print >> html_index, "<html><head><title>INSPIRE Dump</title></head><body><ul>" for collection in CFG_EXPORTED_COLLECTIONS: task_update_progress(collection) print >> html_index, """ <li><a href="%(prefix)s/dumps/%(collection)s-records.xml.gz">%(collection)s</a> (<a href="%(prefix)s/dumps/%(collection)s-records.xml.gz.md5">MD5</a>): %(date)s</li>""" % { "prefix": CFG_SITE_URL, "collection": collection, "date": time.ctime(), } write_message("Preparing %s-records.xml.gz" % collection) output_path = os.path.join(CFG_WEBDIR, "dumps", ".%s-records.xml.gz" % collection) output = gzip.open(output_path, "w") print >> output, "<collection>" reclist = get_collection_reclist(collection) tot = len(reclist) time_estimator = get_time_estimator(tot) for i, recid in enumerate(reclist): print >> output, format_record(recid, "xme", user_info={})[0] time_estimation = time_estimator()[1] if (i + 1) % 100 == 0: task_update_progress( "%s %s (%s%%) -> %s" % ( collection, recid, (i + 1) * 100 / tot, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time_estimation)), ) ) task_sleep_now_if_required() print >> output, "</collection>" output.close() write_message("Computing checksum") print >>open(output_path + ".md5", "w"), calculate_md5(output_path) os.rename(output_path, os.path.join(CFG_WEBDIR, "dumps", "%s-records.xml.gz" % collection)) os.rename(output_path + ".md5", os.path.join(CFG_WEBDIR, "dumps", "%s-records.xml.gz.md5" % collection)) write_message("DONE") print >> html_index, "</ul></body></html>" html_index.close() os.rename( os.path.join(CFG_WEBDIR, "dumps", ".inspire-dump.html"), os.path.join(CFG_WEBDIR, "dumps", "inspire-dump.html") )
def _entry_data_preview(data, recformat='hd'): """ Formats the data using format_record """ if recformat == 'hd' or recformat == 'xm': try: data = format_record(recID=None, of=recformat, xml_record=data) except: print "This is not a XML string" if data is "" or data is None: print 'NAI EINAI ADEIO' data = 'Could not render data' else: pass return data
def extract_references(self, req, form): """Refrences extraction page This page can be used for authors to test their pdfs against our refrences extraction process""" user_info = collect_user_info(req) # Handle the 3 POST parameters if 'pdf' in form and form['pdf'].value: pdf = form['pdf'].value references_xml = extract_from_pdf_string(pdf) elif 'arxiv' in form and form['arxiv'].value: url = make_arxiv_url(arxiv_id=form['arxiv'].value) references_xml = extract_references_from_url_xml(url) elif 'url' in form and form['url'].value: url = form['url'].value references_xml = extract_references_from_url_xml(url) elif 'txt' in form and form['txt'].value: txt = form['txt'].value references_xml = extract_references_from_string_xml(txt) else: references_xml = None # If we have not uploaded anything yet # Display the form that allows us to do so if not references_xml: out = self.extract_references_template() else: out = """ <style type="text/css"> #referenceinp_link { display: none; } </style> """ out += format_record(0, 'hdref', xml_record=references_xml.encode('utf-8'), user_info=user_info) # Render the page (including header, footer) return page(title='References Extractor', body=out, uid=user_info['uid'], req=req)
def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, xml_record=None, user_info=None, on_the_fly=False): """ Format a record in given output format. Return a formatted version of the record in the specified language, search pattern, and with the specified output format. The function will define which format template must be applied. The record to be formatted can be specified with its ID (with 'recID' parameter) or given as XML representation (with 'xml_record' parameter). If 'xml_record' is specified 'recID' is ignored (but should still be given for reference. A dummy recid 0 or -1 could be used). 'user_info' allows to grant access to some functionalities on a page depending on the user's priviledges. The 'user_info' object makes sense only in the case of on-the-fly formatting. 'user_info' is the same object as the one returned by 'webuser.collect_user_info(req)' @param recID: the ID of record to format. @type recID: int @param of: an output format code (or short identifier for the output format) @type of: string @param ln: the language to use to format the record @type ln: string @param verbose: the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, stop if error in format elements 9: errors and warnings, stop if error (debug mode )) @type verbose: int @param search_pattern: list of strings representing the user request in web interface @type search_pattern: list(string) @param xml_record: an xml string represention of the record to format @type xml_record: string or None @param user_info: the information of the user who will view the formatted page (if applicable) @param on_the_fly: if False, try to return an already preformatted version of the record in the database @type on_the_fly: boolean @return: formatted record @rtype: string """ from invenio.search_engine import record_exists if search_pattern is None: search_pattern = [] out = "" if verbose == 9: out += """\n<span class="quicknote"> Formatting record %i with output format %s. </span>""" % (recID, of) ############### FIXME: REMOVE WHEN MIGRATION IS DONE ############### if CFG_BIBFORMAT_USE_OLD_BIBFORMAT and CFG_PATH_PHP: return bibformat_engine.call_old_bibformat(recID, of=of, on_the_fly=on_the_fly) ############################# END ################################## if not on_the_fly and \ (ln == CFG_SITE_LANG or \ of.lower() == 'xm' or \ CFG_BIBFORMAT_USE_OLD_BIBFORMAT or \ (of.lower() in CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS)) and \ record_exists(recID) != -1: # Try to fetch preformatted record. Only possible for records # formatted in CFG_SITE_LANG language (other are never # stored), or of='xm' which does not depend on language. # Exceptions are made for output formats defined in # CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS, which are # always served from the same cache for any language. Also, # do not fetch from DB when record has been deleted: we want # to return an "empty" record in that case res = bibformat_dblayer.get_preformatted_record(recID, of) if res is not None: # record 'recID' is formatted in 'of', so return it if verbose == 9: last_updated = bibformat_dblayer.get_preformatted_record_date( recID, of) out += """\n<br/><span class="quicknote"> Found preformatted output for record %i (cache updated on %s). </span><br/>""" % (recID, last_updated) if of.lower() == 'xm': res = filter_hidden_fields(res, user_info) # try to replace language links in pre-cached res, if applicable: if ln != CFG_SITE_LANG and of.lower( ) in CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS: # The following statements try to quickly replace any # language arguments in URL links. Not an exact # science, but should work most of the time for most # of the formats, with not too many false positives. # We don't have time to parse output much here. res = res.replace('?ln=' + CFG_SITE_LANG, '?ln=' + ln) res = res.replace('&ln=' + CFG_SITE_LANG, '&ln=' + ln) res = res.replace('&ln=' + CFG_SITE_LANG, '&ln=' + ln) out += res return out else: if verbose == 9: out += """\n<br/><span class="quicknote"> No preformatted output found for record %s. </span>""" % recID # Live formatting of records in all other cases if verbose == 9: out += """\n<br/><span class="quicknote"> Formatting record %i on-the-fly. </span>""" % recID try: out += bibformat_engine.format_record(recID=recID, of=of, ln=ln, verbose=verbose, search_pattern=search_pattern, xml_record=xml_record, user_info=user_info) if of.lower() == 'xm': out = filter_hidden_fields(out, user_info) return out except Exception, e: register_exception(prefix="An error occured while formatting record %i in %s" % \ (recID, of), alert_admin=True) #Failsafe execution mode import invenio.template websearch_templates = invenio.template.load('websearch') if verbose == 9: out += """\n<br/><span class="quicknote"> An error occured while formatting record %i. (%s) </span>""" % (recID, str(e)) if of.lower() == 'hd': if verbose == 9: out += """\n<br/><span class="quicknote"> Formatting record %i with websearch_templates.tmpl_print_record_detailed. </span><br/>""" % recID return out + websearch_templates.tmpl_print_record_detailed( ln=ln, recID=recID, ) if verbose == 9: out += """\n<br/><span class="quicknote"> Formatting record %i with websearch_templates.tmpl_print_record_brief. </span><br/>""" % recID return out + websearch_templates.tmpl_print_record_brief( ln=ln, recID=recID, )
def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, xml_record=None, user_info=None, on_the_fly=False): """ Format a record in given output format. Return a formatted version of the record in the specified language, search pattern, and with the specified output format. The function will define which format template must be applied. The record to be formatted can be specified with its ID (with 'recID' parameter) or given as XML representation (with 'xml_record' parameter). If 'xml_record' is specified 'recID' is ignored (but should still be given for reference. A dummy recid 0 or -1 could be used). 'user_info' allows to grant access to some functionalities on a page depending on the user's priviledges. The 'user_info' object makes sense only in the case of on-the-fly formatting. 'user_info' is the same object as the one returned by 'webuser.collect_user_info(req)' @param recID: the ID of record to format. @type recID: int @param of: an output format code (or short identifier for the output format) @type of: string @param ln: the language to use to format the record @type ln: string @param verbose: the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, stop if error in format elements 9: errors and warnings, stop if error (debug mode )) @type verbose: int @param search_pattern: list of strings representing the user request in web interface @type search_pattern: list(string) @param xml_record: an xml string represention of the record to format @type xml_record: string or None @param user_info: the information of the user who will view the formatted page (if applicable) @param on_the_fly: if False, try to return an already preformatted version of the record in the database @type on_the_fly: boolean @return: formatted record @rtype: string """ from invenio.search_engine import record_exists if search_pattern is None: search_pattern = [] out = "" if verbose == 9: out += """\n<span class="quicknote"> Formatting record %i with output format %s. </span>""" % (recID, of) ############### FIXME: REMOVE WHEN MIGRATION IS DONE ############### if CFG_BIBFORMAT_USE_OLD_BIBFORMAT and CFG_PATH_PHP: return bibformat_engine.call_old_bibformat(recID, of=of, on_the_fly=on_the_fly) ############################# END ################################## if not on_the_fly and \ (ln == CFG_SITE_LANG or \ of.lower() == 'xm' or \ CFG_BIBFORMAT_USE_OLD_BIBFORMAT or \ (of.lower() in CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS)) and \ record_exists(recID) != -1: # Try to fetch preformatted record. Only possible for records # formatted in CFG_SITE_LANG language (other are never # stored), or of='xm' which does not depend on language. # Exceptions are made for output formats defined in # CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS, which are # always served from the same cache for any language. Also, # do not fetch from DB when record has been deleted: we want # to return an "empty" record in that case res = bibformat_dblayer.get_preformatted_record(recID, of) if res is not None: # record 'recID' is formatted in 'of', so return it if verbose == 9: last_updated = bibformat_dblayer.get_preformatted_record_date(recID, of) out += """\n<br/><span class="quicknote"> Found preformatted output for record %i (cache updated on %s). </span><br/>""" % (recID, last_updated) if of.lower() == 'xm': res = filter_hidden_fields(res, user_info) # try to replace language links in pre-cached res, if applicable: if ln != CFG_SITE_LANG and of.lower() in CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS: # The following statements try to quickly replace any # language arguments in URL links. Not an exact # science, but should work most of the time for most # of the formats, with not too many false positives. # We don't have time to parse output much here. res = res.replace('?ln=' + CFG_SITE_LANG, '?ln=' + ln) res = res.replace('&ln=' + CFG_SITE_LANG, '&ln=' + ln) res = res.replace('&ln=' + CFG_SITE_LANG, '&ln=' + ln) out += res return out else: if verbose == 9: out += """\n<br/><span class="quicknote"> No preformatted output found for record %s. </span>"""% recID # Live formatting of records in all other cases if verbose == 9: out += """\n<br/><span class="quicknote"> Formatting record %i on-the-fly. </span>""" % recID try: out += bibformat_engine.format_record(recID=recID, of=of, ln=ln, verbose=verbose, search_pattern=search_pattern, xml_record=xml_record, user_info=user_info) if of.lower() == 'xm': out = filter_hidden_fields(out, user_info) return out except Exception, e: register_exception(prefix="An error occured while formatting record %i in %s" % \ (recID, of), alert_admin=True) #Failsafe execution mode import invenio.template websearch_templates = invenio.template.load('websearch') if verbose == 9: out += """\n<br/><span class="quicknote"> An error occured while formatting record %i. (%s) </span>""" % (recID, str(e)) if of.lower() == 'hd': if verbose == 9: out += """\n<br/><span class="quicknote"> Formatting record %i with websearch_templates.tmpl_print_record_detailed. </span><br/>""" % recID return out + websearch_templates.tmpl_print_record_detailed( ln = ln, recID = recID, ) if verbose == 9: out += """\n<br/><span class="quicknote"> Formatting record %i with websearch_templates.tmpl_print_record_brief. </span><br/>""" % recID return out + websearch_templates.tmpl_print_record_brief(ln = ln, recID = recID, )
def extract(self, req, form): """Refrences extraction page This page can be used for authors to test their pdfs against our refrences extraction process""" user_info = collect_user_info(req) plots = None list_image_names = [] list_caption = [] plots_dir = os.path.join(CFG_PREFIX, "var/www/img/plots/") # unique folder name # Handle the 3 POST parameters if 'pdf' in form and form['pdf'].value: pdf = form['pdf'].value references_xml = extract_from_pdf_string(pdf) pdf_string = form['pdf'].file.read() pdf = safe_mkstemp('extract.pdf') f = open(pdf, 'w') f.write(pdf_string) f.close() plots = 'File pdf: ' + str(pdf) + '<br />' (exit_code, output_buffer, stderr_output_buffer) = run_shell_command(CFG_PDFPLOTEXTRACTOR_PATH + ' ' + pdf) plotextracted_pdf_path = pdf + ".extracted/extracted.json" code, figures, extracted = merging_articles(None, plotextracted_pdf_path) id_fulltext = "" marc_path = create_MARCXML(figures, id_fulltext, code, extracted, write_file=True) plots += marc_path + '<br />' f = open (marc_path, 'r') record_xml = f.read() f.close() #plots_dir = "/opt/invenio/var/www/img/plots/" if os.path.exists(plots_dir): shutil.rmtree(plots_dir) os.mkdir(plots_dir) re_list = REGEXP_RECORD.findall(record_xml) for r in re_list: re_subfield = REGEXP_SUBFIELD_A.findall(r) for index, image_path in enumerate(re_subfield): if index == 0: run_shell_command('cp ' + image_path + ' ' + plots_dir) elif 'arxiv' in form and form['arxiv'].value: plots = "" url_pdf = make_arxiv_url(arxiv_id=form['arxiv'].value) references_xml = extract_references_from_url_xml(url_pdf) url_tarball = make_arxiv_tar_url(arxiv_id=form['arxiv'].value) plotextracted_xml_path, plotextracted_pdf_path = extract_plots_from_latex_and_pdf(url_tarball, url_pdf) plots += 'TAR: ' + plotextracted_xml_path + '<br />' plots += 'PDF: ' + plotextracted_pdf_path + '<br />' ''' code, figures, extracted = merging_latex_pdf(plotextracted_xml_path, None, "", ) id_fulltext = "" marc_path = create_MARCXML(figures, id_fulltext, code, extracted, write_file=True) ''' dest_dir = os.path.join(CFG_TMPDIR, 'textmining') try: os.mkdir(dest_dir) except OSError: pass code, message, figures, marc_path = merging_latex_pdf(plotextracted_xml_path, "", "", dest_dir) plots += 'OUTPUT: ' + marc_path + '<br />' f = open (marc_path, 'r') record_xml = f.read() f.close() if os.path.exists(plots_dir): shutil.rmtree(plots_dir) os.mkdir(plots_dir) re_list = REGEXP_RECORD.findall(record_xml) for r in re_list: re_subfield = REGEXP_SUBFIELD_A.findall(r) re_subfield_caption = REGEXP_SUBFIELD_D.findall(r) for index, image_path in enumerate(re_subfield): if index == 0: run_shell_command('cp ' + image_path + ' ' + plots_dir) list_image_names.append(os.path.split(image_path)[1]) list_caption.append(re_subfield_caption[index]) elif 'url' in form and form['url'].value: url = form['url'].value references_xml = extract_references_from_url_xml(url) plots = "ME3" elif 'txt' in form and form['txt'].value: txt = form['txt'].value references_xml = extract_references_from_string_xml(txt) else: references_xml = None # If we have not uploaded anything yet # Display the form that allows us to do so if not references_xml: out = self.extract_references_template() else: out = """ <style type="text/css"> #referenceinp_link { display: none; } /*img.plot { width: 250px; height: 250px; }*/ </style> """ out += format_record(0, 'hdref', xml_record=references_xml.encode('utf-8'), user_info=user_info) if plots: out += "<h2>Plots</h2>" out += plots dirList = os.listdir(plots_dir) for i, fname in enumerate(dirList): out += '<h3>Figure ' + str(i+1) + '</h3> <p><img src="/img/plots/' + fname + '" class="plot"></p>' index = list_image_names.index(fname) out += '<p>' + list_caption[index] + '</p>' # Render the page (including header, footer) return page(title='Document Extractor', body=out, uid=user_info['uid'], req=req)
def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, xml_record=None, user_info=None, on_the_fly=False): """ Formats a record given output format. Returns a formatted version of the record in the specified language, search pattern, and with the specified output format. The function will define which format template must be applied. The record to be formatted can be specified with its ID (with 'recID' parameter) or given as XML representation(with 'xml_record' parameter). If both are specified 'recID' is ignored. 'user_info' allows to grant access to some functionalities on a page depending on the user's priviledges. The 'user_info' object makes sense only in the case of on-the-fly formatting. 'user_info' is the same object as the one returned by 'webuser.collect_user_info(req)' @param recID: the ID of record to format @param of: an output format code (or short identifier for the output format) @param ln: the language to use to format the record @param verbose: the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, stop if error in format elements 9: errors and warnings, stop if error (debug mode )) @param search_pattern: list of strings representing the user request in web interface @param xml_record: an xml string represention of the record to format @param user_info: the information of the user who will view the formatted page (if applicable) @param on_the_fly: if False, try to return an already preformatted version of the record in the database @return: formatted record """ from invenio.search_engine import record_exists if search_pattern is None: search_pattern = [] out = "" if verbose == 9: out += """\n<span class="quicknote"> Formatting record %i with output format %s. </span>""" % (recID, of) ############### FIXME: REMOVE WHEN MIGRATION IS DONE ############### if CFG_BIBFORMAT_USE_OLD_BIBFORMAT and CFG_PATH_PHP: return bibformat_engine.call_old_bibformat(recID, format=of, on_the_fly=on_the_fly) ############################# END ################################## if not on_the_fly and \ (ln == CFG_SITE_LANG or \ of.lower() == 'xm' or \ CFG_BIBFORMAT_USE_OLD_BIBFORMAT or \ (CFG_BIBFORMAT_ENABLE_I18N_BRIEF_FORMAT == False and of.lower() == 'hb')) and \ record_exists(recID) != -1: # Try to fetch preformatted record Only possible for records # formatted in CFG_SITE_LANG language (other are never # stored), or of='xm' which does not depend on language. # Also, when formatting in HB, and when # CFG_BIBFORMAT_ENABLE_I18N_BRIEF_FORMAT is set to False, # ignore other languages and fetch the preformatted output. # Also, do not fetch from DB when record has been deleted: we # want to return an "empty" record in that case res = bibformat_dblayer.get_preformatted_record(recID, of) if res is not None: # record 'recID' is formatted in 'of', so return it if verbose == 9: last_updated = bibformat_dblayer.get_preformatted_record_date(recID, of) out += """\n<br/><span class="quicknote"> Found preformatted output for record %i (cache updated on %s). </span><br/>""" % (recID, last_updated) if of.lower() == 'xm': res = filter_hidden_fields(res, user_info) out += res return out else: if verbose == 9: out += """\n<br/><span class="quicknote"> No preformatted output found for record %s. </span>"""% recID # Live formatting of records in all other cases if verbose == 9: out += """\n<br/><span class="quicknote"> Formatting record %i on-the-fly. </span>""" % recID try: out += bibformat_engine.format_record(recID=recID, of=of, ln=ln, verbose=verbose, search_pattern=search_pattern, xml_record=xml_record, user_info=user_info) if of.lower() == 'xm': out = filter_hidden_fields(out, user_info) return out except Exception, e: register_exception(prefix="An error occured while formatting record %i in %s" % \ (recID, of), alert_admin=True) #Failsafe execution mode import invenio.template websearch_templates = invenio.template.load('websearch') if verbose == 9: out += """\n<br/><span class="quicknote"> An error occured while formatting record %i. (%s) </span>""" % (recID, str(e)) if of.lower() == 'hd': if verbose == 9: out += """\n<br/><span class="quicknote"> Formatting record %i with websearch_templates.tmpl_print_record_detailed. </span><br/>""" % recID return out + websearch_templates.tmpl_print_record_detailed( ln = ln, recID = recID, ) if verbose == 9: out += """\n<br/><span class="quicknote"> Formatting record %i with websearch_templates.tmpl_print_record_brief. </span><br/>""" % recID return out + websearch_templates.tmpl_print_record_brief(ln = ln, recID = recID, )
def format_records(recIDs, of, ln=CFG_SITE_LANG, verbose=0, search_pattern=None, xml_records=None, user_info=None, record_prefix=None, record_separator=None, record_suffix=None, prologue="", epilogue="", req=None, on_the_fly=False): """ Format records given by a list of record IDs or a list of records as xml. Adds a prefix before each record, a suffix after each record, plus a separator between records. Also add optional prologue and epilogue to the complete formatted list. You can either specify a list of record IDs to format, or a list of xml records, but not both (if both are specified recIDs is ignored). 'record_separator' is a function that returns a string as separator between records. The function must take an integer as unique parameter, which is the index in recIDs (or xml_records) of the record that has just been formatted. For example separator(i) must return the separator between recID[i] and recID[i+1]. Alternatively separator can be a single string, which will be used to separate all formatted records. The same applies to 'record_prefix' and 'record_suffix'. 'req' is an optional parameter on which the result of the function are printed lively (prints records after records) if it is given. Note that you should set 'req' content-type by yourself, and send http header before calling this function as it will not do it. This function takes the same parameters as 'format_record' except for: @param recIDs: a list of record IDs @type recIDs: list(int) @param of: an output format code (or short identifier for the output format) @type of: string @param ln: the language to use to format the record @type ln: string @param verbose: the level of verbosity from 0 to 9 (0: silent, 5: errors, 7: errors and warnings, stop if error in format elements 9: errors and warnings, stop if error (debug mode )) @type verbose: int @param search_pattern: list of strings representing the user request in web interface @type search_pattern: list(string) @param user_info: the information of the user who will view the formatted page (if applicable) @param xml_records: a list of xml string representions of the records to format @type xml_records: list(string) @param record_prefix: a string printed before B{each} formatted records (n times) @type record_prefix: string @param record_suffix: a string printed after B{each} formatted records (n times) @type record_suffix: string @param prologue: a string printed at the beginning of the complete formatted records (1x) @type prologue: string @param epilogue: a string printed at the end of the complete formatted output (1x) @type epilogue: string @param record_separator: either a string or a function that returns string to join formatted records @param record_separator: string or function @param req: an optional request object where to print records @param on_the_fly: if False, try to return an already preformatted version of the record in the database @type on_the_fly: boolean @rtype: string """ if req is not None: req.write(prologue) formatted_records = '' #Fill one of the lists with Nones if xml_records is not None: recIDs = map(lambda x: None, xml_records) else: xml_records = map(lambda x: None, recIDs) total_rec = len(recIDs) last_iteration = False for i in range(total_rec): if i == total_rec - 1: last_iteration = True #Print prefix if record_prefix is not None: if isinstance(record_prefix, str): formatted_records += record_prefix if req is not None: req.write(record_prefix) else: string_prefix = record_prefix(i) formatted_records += string_prefix if req is not None: req.write(string_prefix) #Print formatted record formatted_record = format_record(recIDs[i], of, ln, verbose, \ search_pattern, xml_records[i],\ user_info, on_the_fly) formatted_records += formatted_record if req is not None: req.write(formatted_record) #Print suffix if record_suffix is not None: if isinstance(record_suffix, str): formatted_records += record_suffix if req is not None: req.write(record_suffix) else: string_suffix = record_suffix(i) formatted_records += string_suffix if req is not None: req.write(string_suffix) #Print separator if needed if record_separator is not None and not last_iteration: if isinstance(record_separator, str): formatted_records += record_separator if req is not None: req.write(record_separator) else: string_separator = record_separator(i) formatted_records += string_separator if req is not None: req.write(string_separator) if req is not None: req.write(epilogue) return prologue + formatted_records + epilogue