def bst_openaire_altmetric():
"""
"""
recids = search_pattern(p="0->Z", f="0247_a")
a = Altmetric()
for recid in recids:
try:
# Check if we already have an Altmetric id
sysno_inst = get_fieldvalues(recid, "035__9")
if ['Altmetric'] in sysno_inst:
continue
doi_val = get_fieldvalues(recid, "0247_a")[0]
json_res = a.doi(doi_val)
rec = {}
record_add_field(rec, "001", controlfield_value=str(recid))
if json_res:
record_add_field(rec,
'035',
subfields=[('a',
str(json_res['altmetric_id'])),
('9', 'Altmetric')])
bibupload(rec, opt_mode='correct')
except AltmetricHTTPException, e:
register_exception(prefix='Altmetric error (status code %s): %s' %
(e.status_code, str(e)),
alert_admin=False)
def bst_openaire_altmetric():
"""
"""
recids = search_pattern(p="0->Z", f="0247_a")
a = Altmetric()
for recid in recids:
try:
# Check if we already have an Altmetric id
sysno_inst = get_fieldvalues(recid, "035__9")
if ['Altmetric'] in sysno_inst:
continue
doi_val = get_fieldvalues(recid, "0247_a")[0]
json_res = a.doi(doi_val)
rec = {}
record_add_field(rec, "001", controlfield_value=str(recid))
if json_res:
record_add_field(rec, '035', subfields=[
('a', str(json_res['altmetric_id'])), ('9', 'Altmetric')]
)
bibupload(rec, opt_mode='correct')
except AltmetricHTTPException, e:
register_exception(prefix='Altmetric error (status code %s): %s' %
(e.status_code, str(e)), alert_admin=False)
def openaire_create_icon(docid=None, recid=None, reformat=True):
"""
Celery task to create an icon for all documents in a given record or for
just a specific document.
"""
if recid:
docs = BibRecDocs(recid).list_bibdocs()
else:
docs = [BibDoc(docid)]
# Celery task will fail if BibDoc does not exists (on purpose ;-)
for d in docs:
logger.debug("Checking document %s" % d)
if not d.get_icon(subformat_re=re.compile(ICON_SUBFORMAT)):
logger.debug("Document has no icon")
for f in d.list_latest_files():
logger.debug("Checking file %s" % f)
if not f.is_icon():
logger.debug("File not an icon")
file_path = f.get_full_path()
icon_path = None
try:
filename = os.path.splitext(os.path.basename(file_path))[0]
logger.info("Creating icon from file %s" % file_path)
(icon_dir, icon_name) = create_icon(
{
"input-file": file_path,
"icon-name": "icon-%s" % filename,
"multipage-icon": False,
"multipage-icon-delay": 0,
"icon-scale": ICON_SIZE,
"icon-file-format": ICON_FILEFORMAT,
"verbosity": 0,
}
)
icon_path = os.path.join(icon_dir, icon_name)
except InvenioWebSubmitIconCreatorError as e:
logger.warning("Icon for file %s could not be created: %s" % (file_path, str(e)))
register_exception(
prefix="Icon for file %s could not be created: %s" % (file_path, str(e)), alert_admin=False
)
try:
if icon_path and os.path.exists(icon_path):
logger.debug("Adding icon %s to document" % icon_path)
d.add_icon(icon_path, subformat=ICON_SUBFORMAT)
recid_list = ",".join([str(x["recid"]) for x in d.bibrec_links])
if reformat:
task_low_level_submission("bibreformat", "openaire", "-i", recid_list)
except InvenioBibDocFileError as e:
logger.warning(
"Icon %s for file %s could not be added to " "document: %s" % (icon_path, f, str(e))
)
register_exception(
prefix="Icon %s for file %s could not be added" " to document: %s" % (icon_path, f, str(e)),
alert_admin=False,
)
def get_bibcatalog_system():
if CFG_BIBCATALOG_SYSTEM == 'RT':
try:
bc_system = BibCatalogSystemRT()
msg = bc_system.check_system()
if msg:
raise StandardError(msg)
except Exception, err:
register_exception(alert_admin=True, prefix=err)
from invenio.legacy.bibcatalog.system_dummy import BibCatalogSystemDummy
## RT has some troubles... let's fall back on the dummy system
bc_system = BibCatalogSystemDummy()
def get_bibcatalog_system():
if CFG_BIBCATALOG_SYSTEM == 'RT':
try:
bc_system = BibCatalogSystemRT()
msg = bc_system.check_system()
if msg:
raise StandardError(msg)
except Exception, err:
register_exception(alert_admin=True, prefix=err)
from invenio.legacy.bibcatalog.system_dummy import BibCatalogSystemDummy
## RT has some troubles... let's fall back on the dummy system
bc_system = BibCatalogSystemDummy()
def openaire_altmetric_update(recids, upload=True):
"""
Retrieve Altmetric information for a record.
"""
logger.debug("Checking Altmetric for recids %s" % recids)
a = Altmetric()
records = []
for recid in recids:
logger.debug("Checking Altmetric for recid %s" % recid)
try:
# Check if we already have an Altmetric id
sysno_inst = get_fieldvalues(recid, "035__9")
if ['Altmetric'] in sysno_inst:
continue
doi_val = get_fieldvalues(recid, "0247_a")[0]
logger.debug("Found DOI %s" % doi_val)
json_res = a.doi(doi_val)
logger.debug("Altmetric response: %s" % json_res)
rec = {}
record_add_field(rec, "001", controlfield_value=str(recid))
if json_res:
record_add_field(rec,
'035',
subfields=[('a',
str(json_res['altmetric_id'])),
('9', 'Altmetric')])
records.append(rec)
except AltmetricHTTPException as e:
logger.warning(
'Altmetric error for recid %s with DOI %s (status code %s): %s'
% (recid, doi_val, e.status_code, str(e)))
register_exception(prefix='Altmetric error (status code %s): %s' %
(e.status_code, str(e)),
alert_admin=False)
except IndexError:
logger.debug("No DOI found")
pass
if upload and records:
if len(records) == 1:
bibupload(record=records[0], file_prefix="altmetric")
else:
bibupload(collection=records, file_prefix="altmetric")
return records
def openaire_altmetric_update(recids, upload=True):
"""
Retrieve Altmetric information for a record.
"""
logger.debug("Checking Altmetric for recids %s" % recids)
a = Altmetric()
records = []
for recid in recids:
logger.debug("Checking Altmetric for recid %s" % recid)
try:
# Check if we already have an Altmetric id
sysno_inst = get_fieldvalues(recid, "035__9")
if ["Altmetric"] in sysno_inst:
continue
doi_val = get_fieldvalues(recid, "0247_a")[0]
logger.debug("Found DOI %s" % doi_val)
json_res = a.doi(doi_val)
logger.debug("Altmetric response: %s" % json_res)
rec = {}
record_add_field(rec, "001", controlfield_value=str(recid))
if json_res:
record_add_field(rec, "035", subfields=[("a", str(json_res["altmetric_id"])), ("9", "Altmetric")])
records.append(rec)
except AltmetricHTTPException as e:
logger.warning(
"Altmetric error for recid %s with DOI %s (status code %s): %s"
% (recid, doi_val, e.status_code, str(e))
)
register_exception(
prefix="Altmetric error (status code %s): %s" % (e.status_code, str(e)), alert_admin=False
)
except IndexError:
logger.debug("No DOI found")
pass
if upload and records:
if len(records) == 1:
bibupload(record=records[0], file_prefix="altmetric")
else:
bibupload(collection=records, file_prefix="altmetric")
return records
def openaire_altmetric_update(recids, upload=True):
"""
Retrieve Altmetric information for a record.
"""
logger.debug("Checking Altmetric for recids %s" % recids)
a = Altmetric()
records = []
for recid in recids:
logger.debug("Checking Altmetric for recid %s" % recid)
try:
# Check if we already have an Altmetric id
sysno_inst = get_fieldvalues(recid, "035__9")
if ['Altmetric'] in sysno_inst:
continue
doi_val = get_fieldvalues(recid, "0247_a")[0]
logger.debug("Found DOI %s" % doi_val)
json_res = a.doi(doi_val)
logger.debug("Altmetric response: %s" % json_res)
rec = {}
record_add_field(rec, "001", controlfield_value=str(recid))
if json_res:
record_add_field(rec, '035', subfields=[
('a', str(json_res['altmetric_id'])),
('9', 'Altmetric')
])
records.append(rec)
except AltmetricHTTPException, e:
logger.warning(
'Altmetric error for recid %s with DOI %s (status code %s): %s'
% (recid, doi_val, e.status_code, str(e))
)
register_exception(
prefix='Altmetric error (status code %s): %s' % (
e.status_code, str(e)),
alert_admin=False
)
except IndexError:
logger.debug("No DOI found")
pass
from invenio.ext.email import send_email
from invenio.legacy.dbquery import run_sql
from invenio.ext.logging.wrappers import register_exception
from invenio.legacy.search_engine import search_pattern, get_fieldvalues
from invenio.legacy.websubmit.icon_creator import create_icon, \
InvenioWebSubmitIconCreatorError
from invenio.modules.pidstore.models import PersistentIdentifier
from invenio.modules.communities.models import Community
from invenio.ext.template import render_template_to_string
try:
from altmetric import Altmetric, AltmetricHTTPException
except ImportError, e:
register_exception(
prefix='Altmetric module not installed: %s' % str(e),
alert_admin=False
)
# Setup logger
logger = get_task_logger(__name__)
ICON_SIZE = "90"
ICON_SUBFORMAT = 'icon-%s' % ICON_SIZE
ICON_FILEFORMAT = "png"
MAX_RECORDS = 100
DEPOSIT_DATACITE_OF = 'dcite3'
def open_temp_file(prefix):
def openaire_create_icon(docid=None, recid=None, reformat=True):
"""
Celery task to create an icon for all documents in a given record or for
just a specific document.
"""
if recid:
docs = BibRecDocs(recid).list_bibdocs()
else:
docs = [BibDoc(docid)]
# Celery task will fail if BibDoc does not exists (on purpose ;-)
for d in docs:
logger.debug("Checking document %s" % d)
if not d.get_icon(subformat_re=re.compile(ICON_SUBFORMAT)):
logger.debug("Document has no icon")
for f in d.list_latest_files():
logger.debug("Checking file %s" % f)
if not f.is_icon():
logger.debug("File not an icon")
file_path = f.get_full_path()
icon_path = None
try:
filename = os.path.splitext(
os.path.basename(file_path)
)[0]
logger.info("Creating icon from file %s" % file_path)
(icon_dir, icon_name) = create_icon(
{'input-file': file_path,
'icon-name': "icon-%s" % filename,
'multipage-icon': False,
'multipage-icon-delay': 0,
'icon-scale': ICON_SIZE,
'icon-file-format': ICON_FILEFORMAT,
'verbosity': 0})
icon_path = os.path.join(icon_dir, icon_name)
except InvenioWebSubmitIconCreatorError, e:
logger.warning(
'Icon for file %s could not be created: %s' % (
file_path, str(e))
)
register_exception(
prefix='Icon for file %s could not be created: %s'
% (file_path, str(e)),
alert_admin=False
)
try:
if icon_path and os.path.exists(icon_path):
logger.debug(
"Adding icon %s to document" % icon_path)
d.add_icon(icon_path, subformat=ICON_SUBFORMAT)
recid_list = ",".join(
[str(x['recid']) for x in d.bibrec_links])
if reformat:
task_low_level_submission(
'bibreformat', 'openaire', '-i',
recid_list
)
except InvenioBibDocFileError, e:
logger.warning(
'Icon %s for file %s could not be added to '
'document: %s' % (icon_path, f, str(e))
)
register_exception(
prefix='Icon %s for file %s could not be added'
' to document: %s' % (icon_path, f, str(e)),
alert_admin=False
)
from invenio.ext.email import send_email
from invenio.legacy.dbquery import run_sql
from invenio.ext.logging.wrappers import register_exception
from invenio.legacy.search_engine import search_pattern, get_fieldvalues
from invenio.legacy.websubmit.icon_creator import create_icon, \
InvenioWebSubmitIconCreatorError
from invenio.modules.pidstore.models import PersistentIdentifier
from invenio.modules.communities.models import Community
from invenio.ext.template import render_template_to_string
try:
from altmetric import Altmetric, AltmetricHTTPException
except ImportError, e:
register_exception(
prefix='Altmetric module not installed: %s' % str(e),
alert_admin=False
)
# Setup logger
logger = get_task_logger(__name__)
ICON_SIZE = "90"
ICON_SUBFORMAT = 'icon-%s' % ICON_SIZE
ICON_FILEFORMAT = "png"
MAX_RECORDS = 100
DEPOSIT_DATACITE_OF = 'dcite3'
def open_temp_file(prefix):
def load_kbs(cfg, run_sql, in_task=False):
for kb, query in cfg.iteritems():
task_sleep_now_if_required(can_stop_too=True)
if not kb_exists(kb):
add_kb(kb)
if in_task:
write_message("Updating %s KB..." % kb)
try:
if not in_task:
print "kb:", kb
print "kb beginning:", len(get_kb_mappings(kb))
if kb.startswith('json_'):
encoder = ComplexEncoder()
mapping, description = run_sql(query, with_desc=True)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
column_counter = {}
new_description = []
for column in description[1:]:
column = column[0]
counter = column_counter[column] = column_counter.get(
column, 0) + 1
if counter > 1:
new_description.append('%s%d' % (column, counter))
else:
new_description.append(column)
description = new_description
else:
mapping = run_sql(query)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print "mapping:", len(mapping)
if kb == 'projects':
mapping += [('000000', 'NO PROJECT')]
original_keys = set([key[0] for key in get_kbr_keys(kb)])
if not in_task:
print "original_keys before:", len(original_keys)
updated = 0
added = 0
for i, row in enumerate(mapping):
key, value = row[0], row[1:]
if kb.startswith('json_'):
value = encoder.encode(dict(zip(description, value)))
else:
value = value[0]
if value:
if key in original_keys:
original_keys.remove(key)
if in_task:
task_update_progress("%s - %s%%" %
(kb, i * 100 / len(mapping)))
if kb_mapping_exists(kb, key):
updated += 1
update_kb_mapping(kb, key, key, value)
else:
added += 1
add_kb_mapping(kb, key, value)
if not in_task:
print "updated:", updated, "added:", added
print "kb after update:", len(get_kb_mappings(kb))
print "original_keys after:", len(original_keys)
if in_task:
task_update_progress("Cleaning %s" % kb)
for key in original_keys:
remove_kb_mapping(kb, key)
if not in_task:
print "kb after remove:", len(get_kb_mappings(kb))
except:
register_exception(alert_admin=True,
prefix="Error when updating KB %s" % kb)
continue
def load_kbs(cfg, run_sql, in_task=False):
for kb, query in cfg.iteritems():
task_sleep_now_if_required(can_stop_too=True)
if not kb_exists(kb):
add_kb(kb)
if in_task:
write_message("Updating %s KB..." % kb)
try:
if not in_task:
print "kb:", kb
print "kb beginning:", len(get_kb_mappings(kb))
if kb.startswith('json_'):
encoder = ComplexEncoder()
mapping, description = run_sql(query, with_desc=True)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
column_counter = {}
new_description = []
for column in description[1:]:
column = column[0]
counter = column_counter[
column] = column_counter.get(column, 0) + 1
if counter > 1:
new_description.append('%s%d' % (column, counter))
else:
new_description.append(column)
description = new_description
else:
mapping = run_sql(query)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print "mapping:", len(mapping)
if kb == 'projects':
mapping += [('000000', 'NO PROJECT')]
original_keys = set([key[0] for key in get_kbr_keys(kb)])
if not in_task:
print "original_keys before:", len(original_keys)
updated = 0
added = 0
for i, row in enumerate(mapping):
key, value = row[0], row[1:]
if kb.startswith('json_'):
value = encoder.encode(dict(zip(description, value)))
else:
value = value[0]
if value:
if key in original_keys:
original_keys.remove(key)
if in_task:
task_update_progress(
"%s - %s%%" % (kb, i * 100 / len(mapping)))
if kb_mapping_exists(kb, key):
updated += 1
update_kb_mapping(kb, key, key, value)
else:
added += 1
add_kb_mapping(kb, key, value)
if not in_task:
print "updated:", updated, "added:", added
print "kb after update:", len(get_kb_mappings(kb))
print "original_keys after:", len(original_keys)
if in_task:
task_update_progress("Cleaning %s" % kb)
for key in original_keys:
remove_kb_mapping(kb, key)
if not in_task:
print "kb after remove:", len(get_kb_mappings(kb))
except:
register_exception(
alert_admin=True, prefix="Error when updating KB %s" % kb)
continue
