def test_get_record_using_field_filter(self): """BibField - get record filtering fields""" authors = get_record(12, fields=('authors', )) self.assertEquals(len(authors['authors']), 19) mainauthor_title = get_record(12, fields=('authors[0]', 'title')) self.assertTrue('authors[0].full_name' in mainauthor_title) self.assertTrue('title' in mainauthor_title)
def test_get_record_using_field_filter(self): """bibfield - get record filtering fields""" authors = get_record(12, fields=('authors',)) self.assertEquals(len(authors['authors']), 19) mainauthor_title = get_record(12, fields=('authors[0]', 'title')) self.assertTrue('authors[0].full_name' in mainauthor_title) self.assertTrue('title' in mainauthor_title)
def test_calculated_fields_availability_and_values(self): """BibField - values of calculated fields""" record = get_record(31) self.assertEqual(2, record['number_of_copies']) run_sql("insert into crcITEM(barcode, id_bibrec) VALUES('test',31)") self.assertEqual(3, record.get('number_of_copies', reset_cache=True)) run_sql("delete from crcITEM WHERE barcode='test'") self.assertEqual(2, record.get('number_of_copies', reset_cache=True)) self.assertEqual(0, record['number_of_citations']) record = get_record(81) self.assertEqual(4, record['number_of_citations'])
def test_compare_field_values_with_bibrecord_values(self): """bibfield - same value as in bibrecord""" from invenio.bibrecord import record_get_field_values from invenio.search_engine import get_record as search_engine_get_record record = get_record(1) bibrecord_value = record_get_field_values(search_engine_get_record(1), '245', ' ', ' ', 'a')[0] self.assertEqual(bibrecord_value, record['title.title'])
def test_get_using_formating_function(self): """bibfield - format values using formating function""" def dummy(s): return s.upper() record = get_record(1) self.assertEqual('ALEPH EXPERIMENT: CANDIDATE OF HIGGS BOSON PRODUCTION', record.get('title.title', formatfunction=dummy))
def record_id_process(form, field, submit=False): value = field.data or '' if value == "" or value.isspace(): return def is_number(s): try: float(s) return True except ValueError: return False if is_number(field.data): json_reader = get_record(value) else: field.add_message("Record id must be a number!", state='error') return if json_reader is not None: webdeposit_json = form.uncook_json(json_reader, {}, value) #FIXME: update current json, past self, what do you mean?? :S field.add_message('<a href="/record/"' + value + '>Record</a> was loaded successfully', state='info') form.process(MultiDict(webdeposit_json)) else: field.add_message("Record doesn't exist", state='info')
def tokenize_via_recjson(self, recID): """ Will tokenize with use of bibfield. @param recID: id of the record """ rec = get_record(recID) return [str(rec.get(self.nonmarc_tag) or 0)]
def test_produce_json_for_dublin_core(self): """bibfield - produce json dublin core""" record = get_record(1) date = record.get('version_id').strftime('%Y-%m-%dT%H:%M:%SZ') produced_dc = record.produce_json_for_dc() self.assertTrue({'dc:date': date} in produced_dc)
def test_get_legacy_recstruct(self): """bibfield - legacy functions""" bibfield_recstruct = get_record(8).get_legacy_recstruct() bibrecord = search_engine_get_record(8) self.assertEqual(bibfield_recstruct['100'][0][0], bibrecord['100'][0][0]) self.assertEqual(len(bibfield_recstruct['999']), len(bibrecord['999']))
def openaire_upload_notification(recid): """ Send a notification to all user collections. """ ctx = { 'record': get_record(recid), } ucolls = UserCollection.from_recid(recid, provisional=True) for c in ucolls: try: if c.owner.email: ctx.update({ 'usercollection': c, }) content = render_template_to_string( "usercollection_new_upload_email.html", **ctx) send_email(CFG_SITE_SUPPORT_EMAIL, c.owner.email.encode('utf8'), "[%s] New upload to %s" % (CFG_SITE_NAME, c.title.encode('utf8')), content=content.encode('utf8')) logger.info("Sent email for new record %s to %s." % (recid, c.owner.email.encode('utf8'))) except AttributeError: pass
def test_validate_xml_against_xsd(self): """ Validate generated DataCite XML for all public records """ from invenio.websearch_model import Collection from invenio.bibformat import format_record from invenio.bibfield import get_record etree.clear_error_log() for recid in Collection.query.filter_by(name='zenodo').first().reclist: try: xml = None record = get_record(recid) for identifier in record.get('related_identifiers', []): if identifier['scheme'] != identifier['scheme'].lower(): raise Exception("Record %s has problem with upper-case scheme %s" % (recid, identifier['scheme'])) if record.get('doi', None): xml = StringIO(format_record(recid, 'dcite')) xml_doc = etree.parse(xml) self.schema.assertValid(xml_doc) except Exception, e: print recid if xml: print xml.getvalue() raise e
def test_validate_xml_against_xsd(self): """ Validate generated DataCite XML for all public records """ from invenio.websearch_model import Collection from invenio.bibformat import format_record from invenio.bibfield import get_record etree.clear_error_log() for recid in Collection.query.filter_by(name='zenodo').first().reclist: try: xml = None record = get_record(recid) for identifier in record.get('related_identifiers', []): if identifier['scheme'] != identifier['scheme'].lower(): raise Exception( "Record %s has problem with upper-case scheme %s" % (recid, identifier['scheme'])) if record.get('doi', None): xml = StringIO(format_record(recid, 'dcite')) xml_doc = etree.parse(xml) self.schema.assertValid(xml_doc) except Exception, e: print recid if xml: print xml.getvalue() raise e
def resolve_doi(req, doi, ln=CFG_SITE_LANG, verbose=0): """ Redirect to given DOI, or display error page when DOI cannot be resolved. """ _ = gettext_set_language(ln) # Fetch user ID: try: uid = getUid(req) except Error: register_exception(req=req, alert_admin=True) return page(title=_("Internal Error"), body=create_error_box(req, verbose=verbose, ln=ln), description="%s - Internal Error" % CFG_SITE_NAME, keywords="%s, Internal Error" % CFG_SITE_NAME, language=ln, req=req, navmenuid='search') # Resolve DOI recids = perform_request_search(p='doi:"%s"' % doi, of="id", verbose=verbose) recids = [recid for recid in recids if doi.lower() in \ [doi.lower() for doi in get_record(recid).get('doi', '') if doi]] # Answer if len(recids) == 1: # Found unique matching record return redirect_to_url(req, CFG_SITE_URL + '/' + CFG_SITE_RECORD + '/' + str(recids[0])) elif len(recids) == 0: # No corresponding record found page_body = '<p>' + (_("Sorry, DOI %s could not be resolved.") % \ ('<strong>' + str(doi) + '</strong>')) + '</p>' if req.header_only: raise apache.SERVER_RETURN, apache.HTTP_NOT_FOUND return page(title=_('DOI "%s" Not Found') % cgi.escape(doi), body=page_body, description=(CFG_SITE_NAME + ' - ' + _("Not found") + ': ' + cgi.escape(str(doi))), keywords="%s" % CFG_SITE_NAME, uid=uid, language=ln, req=req, navmenuid='search') else: # Found multiple matching records try: raise Exception('DOI "%s" matched multiple records (%s) -- Please check' % (doi, ', '.join([str(recid) for recid in recids]))) except Exception, e: register_exception(req=req, alert_admin=True) page_body = websearch_templates.tmpl_multiple_dois_found_page(doi, recids, ln) return page(title=_('Found multiple records matching DOI %s') % cgi.escape(doi), body=page_body, description=(CFG_SITE_NAME + ' - ' + _("Found multiple records matching DOI") + ': ' + cgi.escape(str(doi))), keywords="%s" % CFG_SITE_NAME, uid=uid, language=ln, req=req, navmenuid='search')
def test_get_legacy_recstruct(self): """BibField - legacy functions""" from invenio.search_engine import get_record as search_engine_get_record from invenio.bibrecord import record_get_field_value bibfield_recstruct = get_record(8).legacy_create_recstruct() bibrecord = search_engine_get_record(8) self.assertEqual(record_get_field_value(bibfield_recstruct, '100', code='a'), record_get_field_value(bibrecord, '100', code='a')) self.assertEqual(len(bibfield_recstruct['999']), len(bibrecord['999']))
def test_bibdoc_integration(self): """BibField - bibdoc integration""" rec = get_record(7) self.assertTrue('files' in rec) self.assertEquals(len(rec['files']), 2) image = rec['files'][1] self.assertEquals(image['eformat'], '.jpeg') self.assertEquals(image['name'], '9806033') bibdoc = rec['bibdocs'].list_latest_files()[1] self.assertEquals(image['name'], bibdoc.name)
def test_get_legacy_recstruct(self): """BibField - legacy functions""" from invenio.search_engine import get_record as search_engine_get_record from invenio.bibrecord import record_get_field_value bibfield_recstruct = get_record(8).legacy_create_recstruct() bibrecord = search_engine_get_record(8) self.assertEqual( record_get_field_value(bibfield_recstruct, '100', code='a'), record_get_field_value(bibrecord, '100', code='a')) self.assertEqual(len(bibfield_recstruct['999']), len(bibrecord['999']))
def _collect_recjson(self, recIDs, termslist): """ Collects terms from recjson with use of bibfield. Used together with recjson tokenizer. """ tokenizing_function = self.tokenizing_function for recID in recIDs: record = get_record(recID) if record: new_words = tokenizing_function(record) if not recID in termslist: termslist[recID] = [] termslist[recID] = list_union(new_words, termslist[recID]) return termslist
def test_normal_fields_availability_and_values(self): """bibfield - access to normal fields""" record = get_record(12) self.assertTrue(record.get('asdas') is None) self.assertEqual('12', record['recid']) self.assertTrue('recid' in record.get_persistent_identifiers()) self.assertEqual(record['recid'], record.get('recid')) self.assertEqual('Physics at the front-end of a neutrino factory : a quantitative appraisal', record['title.title']) self.assertEqual('Physics at the front-end of a neutrino factory : a quantitative appraisal', record['title']['title']) self.assertEqual(None, record['title.subtitle']) self.assertEqual('Physics at the front-end of a neutrino factory : a quantitative appraisal', record.get('title.title')) self.assertEqual('Mangano', record['authors[0].last_name']) self.assertEqual('M L', record['authors[0].first_name']) self.assertEqual(19, len(record['authors'])) self.assertEqual(19, len(record['authors.last_name']))
def tokenize_via_recjson(self, recID): """ Tokenizes for journal info. Uses bibfield. """ phrases = [] rec = get_record(recID) recjson_field = rec.get(self.nonmarc_tag) get_values_recursively(recjson_field, phrases) final = [] append = final.append for phrase in phrases: info = phrase.split("-", 1) append(info[0]) return final
def test_normal_fields_availability_and_values(self): """bibfield - access to normal fields""" record = get_record(12) self.assertTrue(record.get('asdas') is None) self.assertEqual(12, record['recid']) self.assertTrue('recid' in record.get_persistent_identifiers()) self.assertEqual(record['recid'], record.get('recid')) self.assertEqual('Physics at the front-end of a neutrino factory : a quantitative appraisal', record['title.title']) self.assertEqual('Physics at the front-end of a neutrino factory : a quantitative appraisal', record['title']['title']) self.assertFalse('title.subtitle' in record) self.assertEqual('Physics at the front-end of a neutrino factory : a quantitative appraisal', record.get('title.title')) self.assertEqual('Mangano', record['authors[0].last_name']) self.assertEqual('M L', record['authors[0].first_name']) self.assertEqual(19, len(record['authors'])) self.assertEqual(19, len(record['authors.last_name']))
def record_id_validate(field, form=None): value = field.data is_number = number_validate(field)['error'] == 0 \ and (value != "" or not value.isspace()) if is_number: json_reader = get_record(value) else: return dict(error=1, error_message="Record id must be a number!") if json_reader is not None: webdeposit_json = form.uncook_json(json_reader, {}, value) #FIXME: update current json return dict(info=1, info_message='<a href="/record/"' + value + '>Record</a> loaded successfully', fields=webdeposit_json) else: return dict(info=1, info_message="Record doesn't exist")
def openaire_upload_notification(recid): """ Send a notification to all user collections. """ ctx = { 'record': get_record(recid), } ucolls = UserCollection.from_recid(recid, provisional=True) for c in ucolls: try: if c.owner.email: ctx.update({ 'usercollection': c, }) content = render_template_to_string("usercollection_new_upload_email.html", **ctx) send_email(CFG_SITE_SUPPORT_EMAIL, c.owner.email.encode('utf8'), "[%s] New upload to %s" % (CFG_SITE_NAME, c.title.encode('utf8')), content=content.encode('utf8')) logger.info("Sent email for new record %s to %s." % (recid, c.owner.email.encode('utf8'))) except AttributeError: pass
def _collect_string(self, recIDs, termslist): """ Collects terms from specific tags or fields. Used together with string tokenizer. """ tags = self.tags for recID in recIDs: rec = get_record(recID) new_words = [] extend = new_words.extend for tag in tags: tokenizing_function = self.special_tags.get(tag, self.tokenizing_function) phrases = [] recjson_field = rec.get(tag) get_values_recursively(recjson_field, phrases) for phrase in phrases: extend(tokenizing_function(phrase)) if recID not in termslist and new_words: termslist[recID] = [] if new_words: termslist[recID] = list_union(new_words, termslist[recID]) return termslist
def _collect_string(self, recIDs, termslist): """ Collects terms from specific tags or fields. Used together with string tokenizer. """ tags = self.tags for recID in recIDs: rec = get_record(recID) new_words = [] extend = new_words.extend for tag in tags: tokenizing_function = self.special_tags.get( tag, self.tokenizing_function) phrases = [] recjson_field = rec.get(tag) get_values_recursively(recjson_field, phrases) for phrase in phrases: extend(tokenizing_function(phrase)) if recID not in termslist and new_words: termslist[recID] = [] if new_words: termslist[recID] = list_union(new_words, termslist[recID]) return termslist
def test_produce_json_for_marc(self): """BibField - produce json marc""" record = get_record(1) produced_marc = record.produce('json_for_marc') self.assertTrue({'001': 1} in produced_marc)
def decorated(recid, *args, **kwargs): # ensure recid to be integer recid = int(recid) g.collection = collection = Collection.query.filter( Collection.name == guess_primary_collection_of_a_record(recid)).\ one() (auth_code, auth_msg) = check_user_can_view_record(current_user, recid) # only superadmins can use verbose parameter for obtaining debug information if not current_user.is_super_admin and 'verbose' in kwargs: kwargs['verbose'] = 0 if auth_code and current_user.is_guest: cookie = mail_cookie_create_authorize_action( VIEWRESTRCOLL, {'collection': guess_primary_collection_of_a_record(recid)}) url_args = {'action': cookie, 'ln': g.ln, 'referer': request.url} flash(_("Authorization failure"), 'error') return redirect(url_for('webaccount.login', **url_args)) elif auth_code: flash(auth_msg, 'error') abort(apache.HTTP_UNAUTHORIZED) from invenio.bibfield import get_record from invenio.search_engine import record_exists, get_merged_recid # check if the current record has been deleted # and has been merged, case in which the deleted record # will be redirect to the new one record_status = record_exists(recid) merged_recid = get_merged_recid(recid) if record_status == -1 and merged_recid: return redirect(url_for('record.metadata', recid=merged_recid)) elif record_status == -1: abort(apache.HTTP_GONE) # The record is gone! g.bibrec = Bibrec.query.get(recid) record = get_record(recid) title = record.get('title.title', '') b = [(_('Home'), '')] + collection.breadcrumbs()[1:] b += [(title, 'record.metadata', dict(recid=recid))] current_app.config['breadcrumbs_map'][request.endpoint] = b g.record_tab_keys = [] tabs = [] counts = get_detailed_page_tabs_counts(recid) for k, v in get_detailed_page_tabs(collection.id, recid, g.ln).iteritems(): t = {} b = 'record' if k == '': k = 'metadata' if k == 'comments' or k == 'reviews': b = 'webcomment' if k == 'linkbacks': b = 'weblinkback' k = 'index' t['key'] = b + '.' + k t['count'] = counts.get(k.capitalize(), -1) t.update(v) tabs.append(t) if v['visible']: g.record_tab_keys.append(b + '.' + k) if CFG_WEBLINKBACK_TRACKBACK_ENABLED: @register_template_context_processor def trackback_context(): from invenio.weblinkback_templates import get_trackback_auto_discovery_tag return dict(headerLinkbackTrackbackLink= get_trackback_auto_discovery_tag(recid)) def _format_record(recid, of='hd', user_info=current_user, *args, **kwargs): return print_record(recid, format=of, user_info=user_info, *args, **kwargs) @register_template_context_processor def record_context(): return dict(recid=recid, record=record, tabs=tabs, title=title, get_mini_reviews=lambda *args, **kwargs: get_mini_reviews(*args, **kwargs).decode('utf8'), collection=collection, format_record=_format_record) return f(recid, *args, **kwargs)
def test_derived_fields_availability_and_values(self): """bibfield - values of derived fields""" record = get_record(12) self.assertEqual(19, record['number_of_authors'])
def _update_recjson_format(recid, *args, **kwargs): """Update RECJSON cache. :param int recid: record id to process """ dummy = get_record(recid, reset_cache=True)
def tokenize(self, recID): """Uses get_field_count from bibindex_engine_utils for finding a number of references of a publication and pass it in the list""" from invenio.search_engine import get_record rec = get_record(recID) return [str(len(record_get_field_instances(rec, '999', 'C', '5')))]
def test_record_creation(self): import os from wtforms import TextAreaField from datetime import datetime from invenio.search_engine import record_exists from invenio.cache import cache from invenio.config import CFG_PREFIX from invenio.webuser_flask import login_user from invenio.bibworkflow_model import Workflow from invenio.bibworkflow_config import CFG_WORKFLOW_STATUS from invenio.bibsched_model import SchTASK from invenio.webdeposit_utils import get_form, create_workflow, \ set_form_status, CFG_DRAFT_STATUS from invenio.webdeposit_load_deposition_types import \ deposition_metadata from invenio.webdeposit_workflow_utils import \ create_record_from_marc from invenio.bibfield import get_record login_user(1) for deposition_type in deposition_metadata.keys(): deposition = create_workflow(deposition_type, 1) assert deposition is not None # Check if deposition creates a record create_rec = create_record_from_marc() function_exists = False for workflow_function in deposition.workflow: if create_rec.func_code == workflow_function.func_code: function_exists = True if not function_exists: # if a record is not created, #continue with the next deposition continue uuid = deposition.get_uuid() cache.delete_many("1:current_deposition_type", "1:current_uuid") cache.add("1:current_deposition_type", deposition_type) cache.add("1:current_uuid", uuid) # Run the workflow deposition.run() # Create form's json based on the field name form = get_form(1, uuid=uuid) webdeposit_json = {} # Fill the json with dummy data for field in form: if isinstance(field, TextAreaField): # If the field is associated with a marc field if field.has_recjson_key() or field.has_cook_function(): webdeposit_json[field.name] = "test " + field.name draft = dict( form_type=form.__class__.__name__, form_values=webdeposit_json, step=0, # dummy step status=CFG_DRAFT_STATUS['finished'], timestamp=str(datetime.now())) # Add a draft for the first step Workflow.set_extra_data(user_id=1, uuid=uuid, key='drafts', value={0: draft}) workflow_status = CFG_WORKFLOW_STATUS.RUNNING while workflow_status != CFG_WORKFLOW_STATUS.COMPLETED: # Continue workflow deposition.run() set_form_status(1, uuid, CFG_DRAFT_STATUS['finished']) workflow_status = deposition.get_status() # Workflow is finished. Test if record is created recid = deposition.get_data('recid') assert recid is not None # Test that record id exists assert record_exists(recid) == 1 # Test that the task exists task_id = deposition.get_data('task_id') assert task_id is not None bibtask = SchTASK.query.filter(SchTASK.id == task_id).first() assert bibtask is not None # Run bibupload, bibindex, webcoll manually cmd = "%s/bin/bibupload %s" % (CFG_PREFIX, task_id) assert not os.system(cmd) rec = get_record(recid) marc = rec.legacy_export_as_marc() for field in form: if isinstance(field, TextAreaField): # If the field is associated with a marc field if field.has_recjson_key() or field.has_cook_function(): assert "test " + field.name in marc
def decorated(recid, *args, **kwargs): # ensure recid to be integer recid = int(recid) g.collection = collection = Collection.query.filter( Collection.name == guess_primary_collection_of_a_record(recid)).\ one() (auth_code, auth_msg) = check_user_can_view_record(current_user, recid) # only superadmins can use verbose parameter for obtaining debug information if not current_user.is_super_admin and 'verbose' in kwargs: kwargs['verbose'] = 0 if auth_code and current_user.is_guest: cookie = mail_cookie_create_authorize_action(VIEWRESTRCOLL, { 'collection': guess_primary_collection_of_a_record(recid)}) url_args = {'action': cookie, 'ln': g.ln, 'referer': request.url} flash(_("Authorization failure"), 'error') return redirect(url_for('webaccount.login', **url_args)) elif auth_code: flash(auth_msg, 'error') abort(apache.HTTP_UNAUTHORIZED) from invenio.bibfield import get_record from invenio.search_engine import record_exists, get_merged_recid # check if the current record has been deleted # and has been merged, case in which the deleted record # will be redirect to the new one record_status = record_exists(recid) merged_recid = get_merged_recid(recid) if record_status == -1 and merged_recid: return redirect(url_for('record.metadata', recid=merged_recid)) elif record_status == -1: abort(apache.HTTP_GONE) # The record is gone! g.bibrec = Bibrec.query.get(recid) record = get_record(recid) title = record.get('title.title', '') b = [(_('Home'), '')] + collection.breadcrumbs()[1:] b += [(title, 'record.metadata', dict(recid=recid))] current_app.config['breadcrumbs_map'][request.endpoint] = b g.record_tab_keys = [] tabs = [] counts = get_detailed_page_tabs_counts(recid) for k, v in get_detailed_page_tabs(collection.id, recid, g.ln).iteritems(): t = {} b = 'record' if k == '': k = 'metadata' if k == 'comments' or k == 'reviews': b = 'webcomment' if k == 'linkbacks': b = 'weblinkback' k = 'index' t['key'] = b + '.' + k t['count'] = counts.get(k.capitalize(), -1) t.update(v) tabs.append(t) if v['visible']: g.record_tab_keys.append(b + '.' + k) if CFG_WEBLINKBACK_TRACKBACK_ENABLED: @register_template_context_processor def trackback_context(): from invenio.weblinkback_templates import get_trackback_auto_discovery_tag return dict(headerLinkbackTrackbackLink=get_trackback_auto_discovery_tag(recid)) def _format_record(recid, of='hd', user_info=current_user, *args, **kwargs): return print_record(recid, format=of, user_info=user_info, *args, **kwargs) @register_template_context_processor def record_context(): files = [f for f in BibRecDocs(recid, human_readable=True).list_latest_files(list_hidden=False) \ if not f.is_icon() and f.is_restricted(current_user)[0] == 0] return dict(recid=recid, record=record, tabs=tabs, title=title, get_mini_reviews=lambda *args, **kwargs: get_mini_reviews(*args, **kwargs).decode('utf8'), collection=collection, format_record=_format_record, files=files ) return f(recid, *args, **kwargs)
def test_compare_field_values_with_bibrecord_values(self): """bibfield - same value as in bibrecord""" record = get_record(1) bibrecord_value = record_get_field_values(search_engine_get_record(1), '245', ' ', ' ', 'a')[0] self.assertEqual(bibrecord_value, record['title.title'])
def test_produce_json_for_marc(self): """bibfield - produce json marc""" record = get_record(1) produced_marc = record.produce_json_for_marc() self.assertTrue({'001': 1} in produced_marc)
def test_produce_json_for_marc(self): """bibfield - produce json marc""" record = get_record(1) produced_marc = record.produce_json_for_marc() self.assertTrue({'001': '1'} in produced_marc)
def test_compare_field_values_with_bibrecord_values(self): """BibField - same value as in bibrecord""" record = get_record(1) bibrecord_value = record_get_field_values(search_engine_get_record(1), '245', ' ', ' ', 'a')[0] self.assertEqual(bibrecord_value, record['title.title'])