def bst_openaire_altmetric(): """ """ recids = search_pattern(p="0->Z", f="0247_a") a = Altmetric() for recid in recids: try: # Check if we already have an Altmetric id sysno_inst = get_fieldvalues(recid, "035__9") if ['Altmetric'] in sysno_inst: continue doi_val = get_fieldvalues(recid, "0247_a")[0] json_res = a.doi(doi_val) rec = {} record_add_field(rec, "001", controlfield_value=str(recid)) if json_res: record_add_field(rec, '035', subfields=[('a', str(json_res['altmetric_id'])), ('9', 'Altmetric')]) bibupload(rec, opt_mode='correct') except AltmetricHTTPException, e: register_exception(prefix='Altmetric error (status code %s): %s' % (e.status_code, str(e)), alert_admin=False)
def bst_openaire_altmetric(): """ """ recids = search_pattern(p="0->Z", f="0247_a") a = Altmetric() for recid in recids: try: # Check if we already have an Altmetric id sysno_inst = get_fieldvalues(recid, "035__9") if ['Altmetric'] in sysno_inst: continue doi_val = get_fieldvalues(recid, "0247_a")[0] json_res = a.doi(doi_val) rec = {} record_add_field(rec, "001", controlfield_value=str(recid)) if json_res: record_add_field(rec, '035', subfields=[ ('a', str(json_res['altmetric_id'])), ('9', 'Altmetric')] ) bibupload(rec, opt_mode='correct') except AltmetricHTTPException, e: register_exception(prefix='Altmetric error (status code %s): %s' % (e.status_code, str(e)), alert_admin=False)
def openaire_create_icon(docid=None, recid=None, reformat=True): """ Celery task to create an icon for all documents in a given record or for just a specific document. """ if recid: docs = BibRecDocs(recid).list_bibdocs() else: docs = [BibDoc(docid)] # Celery task will fail if BibDoc does not exists (on purpose ;-) for d in docs: logger.debug("Checking document %s" % d) if not d.get_icon(subformat_re=re.compile(ICON_SUBFORMAT)): logger.debug("Document has no icon") for f in d.list_latest_files(): logger.debug("Checking file %s" % f) if not f.is_icon(): logger.debug("File not an icon") file_path = f.get_full_path() icon_path = None try: filename = os.path.splitext(os.path.basename(file_path))[0] logger.info("Creating icon from file %s" % file_path) (icon_dir, icon_name) = create_icon( { "input-file": file_path, "icon-name": "icon-%s" % filename, "multipage-icon": False, "multipage-icon-delay": 0, "icon-scale": ICON_SIZE, "icon-file-format": ICON_FILEFORMAT, "verbosity": 0, } ) icon_path = os.path.join(icon_dir, icon_name) except InvenioWebSubmitIconCreatorError as e: logger.warning("Icon for file %s could not be created: %s" % (file_path, str(e))) register_exception( prefix="Icon for file %s could not be created: %s" % (file_path, str(e)), alert_admin=False ) try: if icon_path and os.path.exists(icon_path): logger.debug("Adding icon %s to document" % icon_path) d.add_icon(icon_path, subformat=ICON_SUBFORMAT) recid_list = ",".join([str(x["recid"]) for x in d.bibrec_links]) if reformat: task_low_level_submission("bibreformat", "openaire", "-i", recid_list) except InvenioBibDocFileError as e: logger.warning( "Icon %s for file %s could not be added to " "document: %s" % (icon_path, f, str(e)) ) register_exception( prefix="Icon %s for file %s could not be added" " to document: %s" % (icon_path, f, str(e)), alert_admin=False, )
def get_bibcatalog_system(): if CFG_BIBCATALOG_SYSTEM == 'RT': try: bc_system = BibCatalogSystemRT() msg = bc_system.check_system() if msg: raise StandardError(msg) except Exception, err: register_exception(alert_admin=True, prefix=err) from invenio.legacy.bibcatalog.system_dummy import BibCatalogSystemDummy ## RT has some troubles... let's fall back on the dummy system bc_system = BibCatalogSystemDummy()
def openaire_altmetric_update(recids, upload=True): """ Retrieve Altmetric information for a record. """ logger.debug("Checking Altmetric for recids %s" % recids) a = Altmetric() records = [] for recid in recids: logger.debug("Checking Altmetric for recid %s" % recid) try: # Check if we already have an Altmetric id sysno_inst = get_fieldvalues(recid, "035__9") if ['Altmetric'] in sysno_inst: continue doi_val = get_fieldvalues(recid, "0247_a")[0] logger.debug("Found DOI %s" % doi_val) json_res = a.doi(doi_val) logger.debug("Altmetric response: %s" % json_res) rec = {} record_add_field(rec, "001", controlfield_value=str(recid)) if json_res: record_add_field(rec, '035', subfields=[('a', str(json_res['altmetric_id'])), ('9', 'Altmetric')]) records.append(rec) except AltmetricHTTPException as e: logger.warning( 'Altmetric error for recid %s with DOI %s (status code %s): %s' % (recid, doi_val, e.status_code, str(e))) register_exception(prefix='Altmetric error (status code %s): %s' % (e.status_code, str(e)), alert_admin=False) except IndexError: logger.debug("No DOI found") pass if upload and records: if len(records) == 1: bibupload(record=records[0], file_prefix="altmetric") else: bibupload(collection=records, file_prefix="altmetric") return records
def openaire_altmetric_update(recids, upload=True): """ Retrieve Altmetric information for a record. """ logger.debug("Checking Altmetric for recids %s" % recids) a = Altmetric() records = [] for recid in recids: logger.debug("Checking Altmetric for recid %s" % recid) try: # Check if we already have an Altmetric id sysno_inst = get_fieldvalues(recid, "035__9") if ["Altmetric"] in sysno_inst: continue doi_val = get_fieldvalues(recid, "0247_a")[0] logger.debug("Found DOI %s" % doi_val) json_res = a.doi(doi_val) logger.debug("Altmetric response: %s" % json_res) rec = {} record_add_field(rec, "001", controlfield_value=str(recid)) if json_res: record_add_field(rec, "035", subfields=[("a", str(json_res["altmetric_id"])), ("9", "Altmetric")]) records.append(rec) except AltmetricHTTPException as e: logger.warning( "Altmetric error for recid %s with DOI %s (status code %s): %s" % (recid, doi_val, e.status_code, str(e)) ) register_exception( prefix="Altmetric error (status code %s): %s" % (e.status_code, str(e)), alert_admin=False ) except IndexError: logger.debug("No DOI found") pass if upload and records: if len(records) == 1: bibupload(record=records[0], file_prefix="altmetric") else: bibupload(collection=records, file_prefix="altmetric") return records
def openaire_altmetric_update(recids, upload=True): """ Retrieve Altmetric information for a record. """ logger.debug("Checking Altmetric for recids %s" % recids) a = Altmetric() records = [] for recid in recids: logger.debug("Checking Altmetric for recid %s" % recid) try: # Check if we already have an Altmetric id sysno_inst = get_fieldvalues(recid, "035__9") if ['Altmetric'] in sysno_inst: continue doi_val = get_fieldvalues(recid, "0247_a")[0] logger.debug("Found DOI %s" % doi_val) json_res = a.doi(doi_val) logger.debug("Altmetric response: %s" % json_res) rec = {} record_add_field(rec, "001", controlfield_value=str(recid)) if json_res: record_add_field(rec, '035', subfields=[ ('a', str(json_res['altmetric_id'])), ('9', 'Altmetric') ]) records.append(rec) except AltmetricHTTPException, e: logger.warning( 'Altmetric error for recid %s with DOI %s (status code %s): %s' % (recid, doi_val, e.status_code, str(e)) ) register_exception( prefix='Altmetric error (status code %s): %s' % ( e.status_code, str(e)), alert_admin=False ) except IndexError: logger.debug("No DOI found") pass
from invenio.ext.email import send_email from invenio.legacy.dbquery import run_sql from invenio.ext.logging.wrappers import register_exception from invenio.legacy.search_engine import search_pattern, get_fieldvalues from invenio.legacy.websubmit.icon_creator import create_icon, \ InvenioWebSubmitIconCreatorError from invenio.modules.pidstore.models import PersistentIdentifier from invenio.modules.communities.models import Community from invenio.ext.template import render_template_to_string try: from altmetric import Altmetric, AltmetricHTTPException except ImportError, e: register_exception( prefix='Altmetric module not installed: %s' % str(e), alert_admin=False ) # Setup logger logger = get_task_logger(__name__) ICON_SIZE = "90" ICON_SUBFORMAT = 'icon-%s' % ICON_SIZE ICON_FILEFORMAT = "png" MAX_RECORDS = 100 DEPOSIT_DATACITE_OF = 'dcite3' def open_temp_file(prefix):
def openaire_create_icon(docid=None, recid=None, reformat=True): """ Celery task to create an icon for all documents in a given record or for just a specific document. """ if recid: docs = BibRecDocs(recid).list_bibdocs() else: docs = [BibDoc(docid)] # Celery task will fail if BibDoc does not exists (on purpose ;-) for d in docs: logger.debug("Checking document %s" % d) if not d.get_icon(subformat_re=re.compile(ICON_SUBFORMAT)): logger.debug("Document has no icon") for f in d.list_latest_files(): logger.debug("Checking file %s" % f) if not f.is_icon(): logger.debug("File not an icon") file_path = f.get_full_path() icon_path = None try: filename = os.path.splitext( os.path.basename(file_path) )[0] logger.info("Creating icon from file %s" % file_path) (icon_dir, icon_name) = create_icon( {'input-file': file_path, 'icon-name': "icon-%s" % filename, 'multipage-icon': False, 'multipage-icon-delay': 0, 'icon-scale': ICON_SIZE, 'icon-file-format': ICON_FILEFORMAT, 'verbosity': 0}) icon_path = os.path.join(icon_dir, icon_name) except InvenioWebSubmitIconCreatorError, e: logger.warning( 'Icon for file %s could not be created: %s' % ( file_path, str(e)) ) register_exception( prefix='Icon for file %s could not be created: %s' % (file_path, str(e)), alert_admin=False ) try: if icon_path and os.path.exists(icon_path): logger.debug( "Adding icon %s to document" % icon_path) d.add_icon(icon_path, subformat=ICON_SUBFORMAT) recid_list = ",".join( [str(x['recid']) for x in d.bibrec_links]) if reformat: task_low_level_submission( 'bibreformat', 'openaire', '-i', recid_list ) except InvenioBibDocFileError, e: logger.warning( 'Icon %s for file %s could not be added to ' 'document: %s' % (icon_path, f, str(e)) ) register_exception( prefix='Icon %s for file %s could not be added' ' to document: %s' % (icon_path, f, str(e)), alert_admin=False )
def load_kbs(cfg, run_sql, in_task=False): for kb, query in cfg.iteritems(): task_sleep_now_if_required(can_stop_too=True) if not kb_exists(kb): add_kb(kb) if in_task: write_message("Updating %s KB..." % kb) try: if not in_task: print "kb:", kb print "kb beginning:", len(get_kb_mappings(kb)) if kb.startswith('json_'): encoder = ComplexEncoder() mapping, description = run_sql(query, with_desc=True) if kb in CFG_ADDITIONAL_ENTRIES: mapping += CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print CFG_ADDITIONAL_ENTRIES[kb] column_counter = {} new_description = [] for column in description[1:]: column = column[0] counter = column_counter[column] = column_counter.get( column, 0) + 1 if counter > 1: new_description.append('%s%d' % (column, counter)) else: new_description.append(column) description = new_description else: mapping = run_sql(query) if kb in CFG_ADDITIONAL_ENTRIES: mapping += CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print "mapping:", len(mapping) if kb == 'projects': mapping += [('000000', 'NO PROJECT')] original_keys = set([key[0] for key in get_kbr_keys(kb)]) if not in_task: print "original_keys before:", len(original_keys) updated = 0 added = 0 for i, row in enumerate(mapping): key, value = row[0], row[1:] if kb.startswith('json_'): value = encoder.encode(dict(zip(description, value))) else: value = value[0] if value: if key in original_keys: original_keys.remove(key) if in_task: task_update_progress("%s - %s%%" % (kb, i * 100 / len(mapping))) if kb_mapping_exists(kb, key): updated += 1 update_kb_mapping(kb, key, key, value) else: added += 1 add_kb_mapping(kb, key, value) if not in_task: print "updated:", updated, "added:", added print "kb after update:", len(get_kb_mappings(kb)) print "original_keys after:", len(original_keys) if in_task: task_update_progress("Cleaning %s" % kb) for key in original_keys: remove_kb_mapping(kb, key) if not in_task: print "kb after remove:", len(get_kb_mappings(kb)) except: register_exception(alert_admin=True, prefix="Error when updating KB %s" % kb) continue
def load_kbs(cfg, run_sql, in_task=False): for kb, query in cfg.iteritems(): task_sleep_now_if_required(can_stop_too=True) if not kb_exists(kb): add_kb(kb) if in_task: write_message("Updating %s KB..." % kb) try: if not in_task: print "kb:", kb print "kb beginning:", len(get_kb_mappings(kb)) if kb.startswith('json_'): encoder = ComplexEncoder() mapping, description = run_sql(query, with_desc=True) if kb in CFG_ADDITIONAL_ENTRIES: mapping += CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print CFG_ADDITIONAL_ENTRIES[kb] column_counter = {} new_description = [] for column in description[1:]: column = column[0] counter = column_counter[ column] = column_counter.get(column, 0) + 1 if counter > 1: new_description.append('%s%d' % (column, counter)) else: new_description.append(column) description = new_description else: mapping = run_sql(query) if kb in CFG_ADDITIONAL_ENTRIES: mapping += CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print "mapping:", len(mapping) if kb == 'projects': mapping += [('000000', 'NO PROJECT')] original_keys = set([key[0] for key in get_kbr_keys(kb)]) if not in_task: print "original_keys before:", len(original_keys) updated = 0 added = 0 for i, row in enumerate(mapping): key, value = row[0], row[1:] if kb.startswith('json_'): value = encoder.encode(dict(zip(description, value))) else: value = value[0] if value: if key in original_keys: original_keys.remove(key) if in_task: task_update_progress( "%s - %s%%" % (kb, i * 100 / len(mapping))) if kb_mapping_exists(kb, key): updated += 1 update_kb_mapping(kb, key, key, value) else: added += 1 add_kb_mapping(kb, key, value) if not in_task: print "updated:", updated, "added:", added print "kb after update:", len(get_kb_mappings(kb)) print "original_keys after:", len(original_keys) if in_task: task_update_progress("Cleaning %s" % kb) for key in original_keys: remove_kb_mapping(kb, key) if not in_task: print "kb after remove:", len(get_kb_mappings(kb)) except: register_exception( alert_admin=True, prefix="Error when updating KB %s" % kb) continue