def test_recids_restricted_collection(self):
"""Test that recids API works"""
from invenio_ext.sqlalchemy import db
from invenio_search.api import Query
from invenio_records.api import Record
# FIXME All of this needs rewrite on Invenio 3.
self.assertEqual(len(Query("").search(collection='CDF Internal Notes').recids), 0)
# Adds special user info to allow test user to search in restricted collections
self.assertEqual(len(Query("").search(
collection='CDF Internal Notes',
user_info={'precached_permitted_restricted_collections': ['CDF Internal Notes']}
).recids), 1)
test_recid = 1396160
original_record = dict(Record.get_record(test_recid))
try:
rec = Record.get_record(test_recid)
rec['collections'].append({'primary': 'CDF-INTERNAL-NOTE'})
rec.commit()
# Wait for indexing
time.sleep(10)
self.assertEqual(len(Query("").search(
collection='CDF Internal Notes',
user_info={'precached_permitted_restricted_collections': ['CDF Internal Notes']}
).recids), 2)
finally:
rec = Record.get_record(test_recid)
rec.model.json = original_record
db.session.commit()
def test_resource_type(app, db, minimal_record, val, passing):
"""Test resource type."""
minimal_record['resource_type'] = val
if passing:
Record.create(minimal_record)
else:
pytest.raises(ValidationError, Record.create, minimal_record)
def test_jsonalchemy_toint_usage(self):
"""Test the usage of ``to_int`` function in real life example.
The ``test_toint`` model contains a field which contains an integer
subfield. Whenever the record is obtained from ``MARCXML``, the
string in mentioned subfield has to be converted to an integer.
However, JSONAlchemy fills every absent subfield with a ``None`` value.
If the record is not provided with the integer subfield and the
built-in ``int`` function is used, the code will crash.
The ``to_int`` function used inside definition of ``test_toint`` field
prevents it. Here the unprovided subfield is ``999__a``.
"""
xml = '<collection><record><datafield tag="999" ind1="" ind2= "">' \
'<subfield code="b">Value</subfield></datafield></record>' \
'</collection>'
from invenio_records.api import Record
simple_record = Record.create(xml, master_format='marc',
model="test_toint",
namespace='testsuite')
self.assertEqual(len(simple_record.__dict__['_dict']['__meta_metadata__']['__errors__']), 0)
# Check if it works when the value is provided.
xml = '<collection><record><datafield tag="999" ind1="" ind2= "">' \
'<subfield code="a">9999</subfield>' \
'<subfield code="b">Value</subfield></datafield></record>' \
'</collection>'
simple_record = Record.create(xml, master_format='marc',
model="test_toint",
namespace='testsuite')
self.assertEqual(simple_record['with_integers'][0]['some_int'], 9999)
def test_admin(app):
"""Test flask-admin interace."""
admin = Admin(app, name="Test")
assert 'model' in record_adminview
assert 'modelview' in record_adminview
# Register both models in admin
model = record_adminview.pop('model')
view = record_adminview.pop('modelview')
admin.add_view(view(model, db.session, **record_adminview))
# Check if generated admin menu contains the correct items
menu_items = {str(item.name): item for item in admin.menu()}
assert 'Records' in menu_items
assert menu_items['Records'].is_category()
submenu_items = {
str(item.name): item for item in menu_items['Records'].get_children()}
assert 'Record Metadata' in submenu_items
assert isinstance(submenu_items['Record Metadata'], menu.MenuView)
# Create a test record.
with app.app_context():
rec_uuid = str(uuid.uuid4())
Record.create({'title': 'test'}, id_=rec_uuid)
db.session.commit()
with app.test_request_context():
index_view_url = url_for('recordmetadata.index_view')
delete_view_url = url_for('recordmetadata.delete_view')
detail_view_url = url_for(
'recordmetadata.details_view', id=rec_uuid)
with app.test_client() as client:
# List index view and check record is there.
res = client.get(index_view_url)
assert res.status_code == 200
# Fake a problem with SQLAlchemy.
with patch('invenio_records.admin.Record') as db_mock:
db_mock.side_effect = SQLAlchemyError()
res = client.post(
delete_view_url, data={'id': rec_uuid}, follow_redirects=True)
assert res.status_code == 200
# Delete it.
res = client.post(
delete_view_url, data={'id': rec_uuid}, follow_redirects=True)
assert res.status_code == 200
# View the delete record
res = client.get(detail_view_url)
assert res.status_code == 200
assert '<pre>null</pre>' in res.get_data(as_text=True)
# Delete it again
res = client.post(
delete_view_url, data={'id': rec_uuid}, follow_redirects=True)
assert res.status_code == 200
def test_appoint_profile_from_claimed_signature(small_app):
"""Check the module for the case where claimed signature takes
everything.
"""
from inspirehep.modules.disambiguation.tasks import disambiguation_clustering, update_authors_recid
old_record_id = str(PersistentIdentifier.get("literature", 11883).object_uuid)
old_record = get_es_record_by_uuid(old_record_id)
old_author_uuid = old_record["authors"][0]["uuid"]
# Add phonetic block to the record.
old_record["authors"][0]["signature_block"] = "HAGp"
old_record["authors"][0]["recid"] = "2"
es.index(index="records-hep", doc_type="hep", id=old_record_id, body=old_record)
es.indices.refresh("records-hep")
record_id = str(PersistentIdentifier.get("literature", 1358492).object_uuid)
record = get_es_record_by_uuid(record_id)
author_uuid = record["authors"][0]["uuid"]
# Add phonetic block to the record.
record["authors"][0]["signature_block"] = "HAGp"
record["authors"][0]["recid"] = "314159265"
record["authors"][0]["curated_relation"] = True
es.index(index="records-hep", doc_type="hep", id=record_id, body=record)
es.indices.refresh("records-hep")
with patch("celery.current_app.send_task", return_value=_BeardObject(({"2": [old_author_uuid, author_uuid]}, {}))):
with patch(
"inspirehep.modules.disambiguation.tasks.update_authors_recid.delay", side_effect=update_authors_recid
):
disambiguation_clustering("HAGp")
assert Record.get_record(old_record_id)["authors"][0]["recid"] == "314159265"
assert Record.get_record(record_id)["authors"][0]["recid"] == "314159265"
def test_view_documents(app_assets_db):
"""Test the view invenio_previewer.document"""
with app_assets_db.test_request_context():
with app_assets_db.test_client() as client:
tmpdirname = tempfile.mktemp()
if not os.path.exists(tmpdirname):
os.makedirs(tmpdirname)
content = ''.join(random.choice(string.ascii_letters)
for i in range(16536))
filename = 'file.txt'
filename_path = os.path.join(tmpdirname, filename)
with open(filename_path, 'w') as file:
file.write(content)
rec_uuid = uuid.uuid4()
with db.session.begin_nested():
Record.create({
"title": "TestDefault",
"files": [{"uri": filename_path}]
}, id_=rec_uuid)
url = url_for('invenio_previewer.document',
recid=rec_uuid, filename=filename)
response = client.get(url)
assert content == response.data.decode('ascii')
shutil.rmtree(tmpdirname)
def setup_record_fixture(app):
"""Setup a record fixture."""
records = []
def _create_pid(record):
pid = PersistentIdentifier.create(
'recid', record['recid'], pid_provider='recid')
pid.assign('rec', record['recid'])
pid.register()
with before_record_insert.connected_to(_create_pid):
with app.app_context():
records.append(Record.create(
{'title': 'Test record 1', 'recid': 1},
identifier_key='recid'
))
records.append(Record.create(
{'title': 'Test record 2', 'recid': 2},
identifier_key='recid'
))
pid = PersistentIdentifier.create('recid', 3, pid_provider='recid')
db.session.add(pid)
db.session.commit()
pid = PersistentIdentifier.get('recid', 2, pid_provider='recid')
pid.delete()
db.session.commit()
return records
def test_grant_schema_ep_resolving(app, db):
"""Test schema validation using entry-point registered schemas."""
json_valid = {
'$schema': (
'http://inveniosoftware.org/schemas/grants/grant-v1.0.0.json'),
'internal_id': '10.13039/001::0001',
'identifiers': {
'oai_id': 'oai_id00001',
'eurepo': '/eurepo/id00001',
},
'code': '0001',
'title': 'Grant Foobar',
'acronym': 'GF',
'startdate': 'startdate',
'enddate': 'startdate',
'funder': {'$ref': 'http://dx.doi.org/10.13039/001'},
}
# Should not raise validation errors
R.create(json_valid)
# Should raise validation error because of the field 'acronyms'
json_invalid = dict(json_valid)
json_invalid['identifiers'] = 'not_an_object'
with pytest.raises(ValidationError) as exc_info:
R.create(json_invalid)
assert exc_info.value.instance == 'not_an_object'
def test_resource_type(app, db, minimal_record):
"""Test recid property."""
# String instead of number
minimal_record['resource_type'] = 'publication'
pytest.raises(ValidationError, Record.create, minimal_record)
minimal_record['resource_type'] = {'type': 'publication', 'subtype': 'x'}
Record.create(minimal_record)
def oaiserver(number):
"""Initialize OAI-PMH server."""
from invenio_db import db
from invenio_oaiserver.models import OAISet
from invenio_records.api import Record
# create a OAI Set
with db.session.begin_nested():
for i in range(number):
db.session.add(OAISet(
spec='test{0}'.format(i),
name='Test{0}'.format(i),
description='test desc {0}'.format(i),
search_pattern='title:Test{0}'.format(i),
))
# create a record
schema = {
'type': 'object',
'properties': {
'title': {'type': 'string'},
'field': {'type': 'boolean'},
},
'required': ['title'],
}
with db.session.begin_nested():
for i in range(number):
record_id = uuid.uuid4()
data = {'title': 'Test{0}'.format(i), '$schema': schema}
recid_minter(record_id, data)
oaiid_minter(record_id, data)
Record.create(data, id_=record_id)
db.session.commit()
def grant_records(db, funder_record):
"""Create grant records."""
grants = [
Record.create(dict(
internal_id='10.13039/501100000780::282896',
funder={'$ref': 'https://dx.doi.org/10.13039/501100000780'},
identifiers=dict(
eurepo='info:eu-repo/grantAgreement/EC/FP7/282896',
),
code='282896',
title='Open Access Research Infrastructure in Europe',
acronym='OpenAIREplus',
program='FP7',
)),
Record.create(dict(
internal_id='10.13039/501100000780::027819',
funder={'$ref': 'https://dx.doi.org/10.13039/501100000780'},
identifiers=dict(
eurepo='info:eu-repo/grantAgreement/EC/FP6/027819',
),
code='027819',
title='Integrating cognition, emotion and autonomy',
acronym='ICEA',
program='FP6',
)),
]
for g in grants:
PersistentIdentifier.create(
pid_type='grant', pid_value=g['internal_id'], object_type='rec',
object_uuid=g.id, status='R')
db.session.commit()
return grants
def formatter(bwo, **kwargs):
"""Nicely format the record."""
from pprint import pformat
from invenio_records.api import Record
try:
data = bwo.data
except AttributeError:
data = bwo.get_data()
if not data:
return ''
formatter = kwargs.get("formatter", None)
of = kwargs.get("of", None)
if formatter:
# A separate formatter is supplied
return formatter(data)
if isinstance(data, collections.Mapping):
# Dicts are cool on its own, but maybe its SmartJson (record)
try:
data = Record(data.dumps()).legacy_export_as_marc()
except (TypeError, KeyError):
pass
if isinstance(data, string_types):
# We can try formatter!
# If already XML, format_record does not like it.
if of and of != 'xm':
try:
from invenio_formatter import format_record
formatted_data = format_record(
recID=None,
of=of,
xml_record=data
)
except TypeError:
# Wrong kind of type
pass
else:
# So, XML then
from xml.dom.minidom import parseString
try:
unpretty_data = parseString(data)
formatted_data = unpretty_data.toprettyxml()
except TypeError:
# Probably not proper XML string then
return "Data cannot be parsed: %s" % (data,)
except Exception:
# Just return raw string
pass
if not formatted_data:
formatted_data = data
if isinstance(formatted_data, dict):
formatted_data = pformat(formatted_data)
return formatted_data
def test_listmetadataformats_record(app):
"""Test ListMetadataFormats for a record."""
schema = {
'type': 'object',
'properties': {
'title': {'type': 'string'},
'field': {'type': 'boolean'},
},
'required': ['title'],
}
with app.test_request_context():
with db.session.begin_nested():
record_id = uuid.uuid4()
data = {'title': 'Test0', '$schema': schema}
recid_minter(record_id, data)
pid = oaiid_minter(record_id, data)
Record.create(data, id_=record_id)
pid_value = pid.pid_value
db.session.commit()
_listmetadataformats(
app=app,
query='/oai2d?verb=ListMetadataFormats&identifier={0}'.format(
pid_value))
def records():
"""Load test data fixture."""
import uuid
from invenio_records.api import Record
from invenio_pidstore.models import PersistentIdentifier, PIDStatus
create_test_user()
indexer = RecordIndexer()
# Record 1 - Live record
with db.session.begin_nested():
rec_uuid = uuid.uuid4()
pid1 = PersistentIdentifier.create(
'recid', '1', object_type='rec', object_uuid=rec_uuid,
status=PIDStatus.REGISTERED)
Record.create({
'title': 'Registered',
'description': 'This is an awesome description',
'control_number': '1',
'access_right': 'restricted',
'access_conditions': 'fuu',
'owners': [1, 2],
'recid': 1
}, id_=rec_uuid)
indexer.index_by_id(pid1.object_uuid)
db.session.commit()
sleep(3)
def references(self):
"""Reference export for single record in datatables format.
:returns: list
List of lists where every item represents a datatables row.
A row consists of [reference_number, reference, num_citations]
"""
out = []
references = self.record.get('references')
if references:
refs_to_get_from_es = [
ref['recid'] for ref in references if ref.get('recid')
]
query = IQ(' OR '.join('recid:' + str(ref)
for ref in refs_to_get_from_es))
records_from_es = current_search_client.search(
index='records-hep',
doc_type='hep',
body={"query": query.to_dict()},
size=9999,
_source=[
'control_number',
'citation_count',
'titles',
'earliest_date',
'authors',
'collaboration',
'corporate_author',
'publication_info'
]
)['hits']['hits']
refs_from_es = {
str(ref['_source']['control_number']): ref['_source'] for ref in records_from_es
}
for reference in references:
row = []
recid = reference.get('recid')
ref_record = refs_from_es.get(str(recid)) if recid else None
if recid and ref_record:
ref_record = Record(ref_record)
if ref_record:
row.append(render_template_to_string(
"inspirehep_theme/references.html",
record=ref_record,
reference=reference
))
row.append(ref_record.get('citation_count', ''))
out.append(row)
else:
row.append(render_template_to_string(
"inspirehep_theme/references.html",
reference=reference))
row.append('')
out.append(row)
return out
def index_holdingpen_record(sender, **kwargs):
"""Index a Holding Pen record."""
from invenio.ext.es import es
from invenio_records.api import Record
from invenio_records.signals import before_record_insert
from .registry import workflows
from .models import ObjectVersion
if not sender.workflow:
# No workflow registered to object yet. Skip indexing
return
workflow = workflows.get(sender.workflow.name)
if not workflow:
current_app.logger.error(
"No workflow found for sender: {0}".format(sender.id)
)
return
if not hasattr(sender, 'data'):
sender.data = sender.get_data()
if not hasattr(sender, 'extra_data'):
sender.extra_data = sender.get_extra_data()
record = Record({})
record["version"] = ObjectVersion.name_from_version(sender.version)
record["type"] = sender.data_type
record["status"] = sender.status
record["created"] = sender.created.isoformat()
record["modified"] = sender.modified.isoformat()
record["uri"] = sender.uri
record["id_workflow"] = sender.id_workflow
record["id_user"] = sender.id_user
record["id_parent"] = sender.id_parent
record["workflow"] = sender.workflow.name
try:
record.update(workflow.get_record(sender))
except Exception as err:
current_app.logger.exception(err)
try:
record.update(workflow.get_sort_data(sender))
except Exception as err:
current_app.logger.exception(err)
from invenio.modules.jsonalchemy.registry import functions
list(functions('recordext'))
toposort_send(before_record_insert, record)
es.index(
index=current_app.config.get("WORKFLOWS_HOLDING_PEN_INDEX"),
doc_type='record',
body=dict(record),
id=sender.id
)
def test_collection_tree_matcher(app):
"""Test database backend."""
# a
# (None)
# +------------------+--------------------+
# | |
# b e
# (None) (title:Test2 OR title:Test3)
# +------+-----+ +------------+------------+
# | | | | |
# c d f g h
# (title:Test0) (title:Test1) (title:Test2) (None) (None)
# | |
# i j
# (title:Test3) (title:Test4))
with app.test_request_context():
a = Collection(name="a")
b = Collection(name="b", parent=a)
e = Collection(
name="e", dbquery="title:Test2 OR title:Test3", parent=a)
c = Collection(name="c", dbquery="title:Test0", parent=b)
d = Collection(name="d", dbquery="title:Test1", parent=b)
f = Collection(name="f", dbquery="title:Test2", parent=e)
g = Collection(name="g", parent=e)
h = Collection(name="h", parent=e)
i = Collection(name="i", dbquery="title:Test3", parent=g)
j = Collection(name="j", dbquery="title:Test4", parent=h)
with db.session.begin_nested():
for coll in [a, b, c, d, e, f, g, h, i, j]:
db.session.add(coll)
db.session.commit()
# start tests
schema = {
'type': 'object',
'properties': {
'title': {'type': 'string'},
'field': {'type': 'boolean'},
'hello': {'type': 'array'},
},
'required': ['title'],
}
record0 = Record.create({'title': 'Test0', '$schema': schema})
record1 = Record.create({'title': 'Test1', '$schema': schema})
record2 = Record.create({'title': 'Test2', '$schema': schema})
record3 = Record.create({'title': 'Test3', '$schema': schema})
record4 = Record.create({'title': 'Test4', '$schema': schema})
assert set(record0['_collections']) == set(['a', 'c', 'b'])
assert set(record1['_collections']) == set(['a', 'b', 'd'])
assert set(record2['_collections']) == set(['a', 'e', 'f'])
assert set(record3['_collections']) == set(['a', 'e', 'g', 'i'])
assert set(record4['_collections']) == set(['h', 'j'])
def test_funder_ep_resolving(app, db):
"""Test funder resolving through entry point-registered JSON resolver."""
json1 = {"internal_id": "10.13039/001", "parent": "", "name": "Foo"}
json2 = {"internal_id": "10.13039/002", "parent": {"$ref": "http://dx.doi.org/10.13039/001"}, "name": "Bar"}
r1 = R.create(json1)
PID.create("frdoi", json1["internal_id"], object_type="rec", object_uuid=r1.id, status=PIDStatus.REGISTERED)
r2 = R.create(json2)
PID.create("frdoi", json2["internal_id"], object_type="rec", object_uuid=r2.id, status=PIDStatus.REGISTERED)
assert r2.replace_refs()["parent"] == json1
def test_identifier_schemes(app, db, minimal_record):
"""Test supported identifier schemes."""
supported_schemes = [s for s, _ in idutils.PID_SCHEMES]
minimal_record['related_identifiers'] = [
{'scheme': scheme, 'relation': 'references', 'identifier': 'foobar'}
for scheme in supported_schemes
]
# JSONSchema validation should allow all supported schemes
Record.create(minimal_record)
def test_file_download_ui(base_app, objects, db):
"""Test get buckets."""
app = base_app
app.config.update(dict(
RECORDS_UI_DEFAULT_PERMISSION_FACTORY=None, # No permission checking
RECORDS_UI_ENDPOINTS=dict(
recid=dict(
pid_type='recid',
route='/records/<pid_value>',
),
recid_files=dict(
pid_type='recid',
route='/records/<pid_value>/files/<filename>',
view_imp='invenio_files_rest.views.file_download_ui',
),
)
))
InvenioRecords(app)
InvenioPIDStore(app)
InvenioRecordsUI(app)
obj1 = objects[0]
with app.app_context():
# Record 1 - Live record
rec_uuid = uuid.uuid4()
PersistentIdentifier.create(
'recid', '1', object_type='rec', object_uuid=rec_uuid,
status=PIDStatus.REGISTERED)
Record.create({
'title': 'Registered',
'recid': 1,
'files': [
{'filename': obj1.key, 'bucket': str(obj1.bucket_id),
'checksum': 'invalid'},
]
}, id_=rec_uuid)
db.session.commit()
main_url = url_for('invenio_records_ui.recid', pid_value='1')
file_url = url_for(
'invenio_records_ui.recid_files', pid_value='1', filename=obj1.key)
no_file_url = url_for(
'invenio_records_ui.recid_files', pid_value='1', filename='')
invalid_file_url = url_for(
'invenio_records_ui.recid_files', pid_value='1', filename='no')
with app.test_client() as client:
res = client.get(main_url)
assert res.status_code == 200
res = client.get(file_url)
assert res.status_code == 200
res = client.get(no_file_url)
assert res.status_code == 404
res = client.get(invalid_file_url)
assert res.status_code == 404
def test_contributors(app, db, minimal_record):
"""Test contributors."""
minimal_record['contributors'] = [
{'name': 'test', 'affiliation': 'test', 'type': 'ContactPerson'}
]
Record.create(minimal_record)
minimal_record['contributors'] = [
{'name': 'test', 'affiliation': 'test', 'type': 'Invalid'}
]
pytest.raises(ValidationError, Record.create, minimal_record)
def records():
"""Load test data fixture."""
with db.session.begin_nested():
for idx in range(20):
# create the record
Record.create({
'title': 'LHC experiment {}'.format(idx),
'description': 'Data from experiment {}.'.format(idx),
'type': 'data',
})
db.session.commit()
def test_model_init(app):
"""Test basic model initialization and actions."""
with app.app_context():
# Init the User and the Community
user1 = create_test_user()
comm1 = Community(id='comm1', id_user=user1.id)
db.session.add(comm1)
db.session.commit()
communities_key = app.config["COMMUNITIES_RECORD_KEY"]
# Create a record and accept it into the community by creating an
# InclusionRequest and then calling the accept action
rec1 = Record.create({'title': 'Foobar'})
InclusionRequest.create(community=comm1, record=rec1)
assert InclusionRequest.query.count() == 1
comm1.accept_record(rec1)
assert 'comm1' in rec1[communities_key]
assert InclusionRequest.query.count() == 0
# Likewise, reject a record from the community
rec2 = Record.create({'title': 'Bazbar'})
InclusionRequest.create(community=comm1, record=rec2)
assert InclusionRequest.query.count() == 1
comm1.reject_record(rec2)
assert communities_key not in rec2 # dict key should not be created
assert InclusionRequest.query.count() == 0
# Add record to another community
comm2 = Community(id='comm2', id_user=user1.id)
db.session.add(comm2)
db.session.commit()
InclusionRequest.create(community=comm2, record=rec1)
comm2.accept_record(rec1)
assert communities_key in rec1
assert len(rec1[communities_key]) == 2
assert comm1.id in rec1[communities_key]
assert comm2.id in rec1[communities_key]
# Accept/reject a record to/from a community without inclusion request
rec3 = Record.create({'title': 'Spam'})
pytest.raises(InclusionRequestMissingError, comm1.accept_record, rec3)
pytest.raises(InclusionRequestMissingError, comm1.reject_record, rec3)
# Create two inclusion requests
comm3 = Community(id='comm3', id_user=user1.id)
db.session.add(comm3)
db.session.commit()
InclusionRequest.create(community=comm3, record=rec1)
pytest.raises(InclusionRequestExistsError, InclusionRequest.create,
community=comm3, record=rec1)
# Try to accept a record to a community twice (should raise)
# (comm1 is already in rec1)
pytest.raises(InclusionRequestObsoleteError, InclusionRequest.create,
community=comm1, record=rec1)
def test_contributors(app, db, minimal_record):
"""Test recid property."""
# String instead of number
minimal_record['contributors'] = [
{'name': 'test', 'affiliation': 'test', 'type': 'ContactPerson'}
]
Record.create(minimal_record)
minimal_record['contributors'] = [
{'name': 'test', 'affiliation': 'test', 'type': 'Invalid'}
]
pytest.raises(ValidationError, Record.create, minimal_record)
def create_record(data, force=False, dry_run=False):
record = marc_create_record(data)
recid = None
if '001' in record:
recid = int(record['001'][0])
if not dry_run and recid:
prod_record = InspireProdRecords(recid=recid)
prod_record.marcxml = data
try:
if _collection_in_record(record, 'institution'):
json = strip_empty_values(institutions.do(record))
elif _collection_in_record(record, 'experiment'):
json = strip_empty_values(experiments.do(record))
elif _collection_in_record(record, 'journals'):
json = strip_empty_values(journals.do(record))
elif _collection_in_record(record, 'hepnames'):
json = strip_empty_values(hepnames.do(record))
elif _collection_in_record(record, 'job') or \
_collection_in_record(record, 'jobhidden'):
json = strip_empty_values(jobs.do(record))
elif _collection_in_record(record, 'conferences'):
json = strip_empty_values(conferences.do(record))
else:
json = strip_empty_values(hep.do(record))
if dry_run:
return recid, json
if force and any(key in json for key in ('control_number', 'recid')):
try:
control_number = json['control_number']
except KeyError:
control_number = json['recid']
control_number = int(control_number)
# Searches if record already exists.
record = Record.get_record(control_number)
if record is None:
# Adds the record to the db session.
rec = RecordModel(id=control_number)
db.session.merge(rec)
record = Record.create(json)
else:
record = Record(json, model=record.model)
record.commit()
if recid:
prod_record.successful = True
db.session.merge(prod_record)
logger.info("Elaborated record {}".format(control_number))
return control_number, dict(record)
except Exception:
if recid:
prod_record.successful = False
db.session.merge(prod_record)
logger.exception("Error in elaborating record ID {}".format(recid))
raise
def test_remote_openaire_loader(app, db):
"""Test the remote OAI-PMH OpenAIRE loader."""
loader = RemoteOAIRELoader()
pytest.raises(OAIRELoadingError, list, loader.iter_grants())
recuuid = uuid.uuid4()
PersistentIdentifier.create(
'frdoi', '10.13039/501100000925',
object_type='rec', object_uuid=recuuid, status='R')
Record.create({'acronyms': ['EC']}, id_=recuuid)
records = list(loader.iter_grants())
assert len(records) == 5
def test_update_authors_recid_method(small_app):
"""Test the method responsible for updating author's recid."""
from inspirehep.modules.disambiguation.tasks import update_authors_recid
pid = PersistentIdentifier.get("literature", 4328)
publication_id = str(pid.object_uuid)
signature = Record.get_record(publication_id)["authors"][0]["uuid"]
profile_recid = "314159265"
update_authors_recid(publication_id, signature, profile_recid)
assert Record.get_record(publication_id)["authors"][0]["recid"] == profile_recid
def get_title(bwo):
"""Return title of object."""
try:
deposit_object = Deposition(bwo)
except InvalidDepositionType:
return "This submission is disabled: {0}.".format(bwo.workflow.name)
sip = deposit_object.get_latest_sip()
if sip:
# Get the SmartJSON object
record = Record(sip.metadata)
return record.get("titles.title", ["No title"])[0]
else:
return "User submission in progress"
def test_view_preview_default_extension(app_assets_db):
"""Test view by default."""
with app_assets_db.test_request_context():
with app_assets_db.test_client() as client:
rec_uuid = uuid.uuid4()
with db.session.begin_nested():
Record.create({
"title": "TestDefault",
"files": [{"uri": "/tmp/TestDefault.def"}]
}, id_=rec_uuid)
url = url_for('invenio_previewer.preview', recid=rec_uuid)
response = client.get(url)
assert 'we are unfortunately not' in response.data.decode('utf-8')
def records():
"""Load test data fixture."""
import uuid
from invenio_records.api import Record
from invenio_pidstore.models import PersistentIdentifier, PIDStatus
# Record 1 - Live record
with db.session.begin_nested():
pid1 = PersistentIdentifier.create(
'recid', '1', object_type='rec', object_uuid=rec1_uuid,
status=PIDStatus.REGISTERED)
Record.create({
'title': 'Registered ',
'authors': [
{'name': 'Ellis Jonathan'},
{'name': 'Higgs Peter'},
],
'keywords': ['CERN', 'higgs'],
}, id_=rec1_uuid)
# Record 2 - Deleted PID with record
rec2_uuid = uuid.uuid4()
pid = PersistentIdentifier.create(
'recid', '2', object_type='rec', object_uuid=rec2_uuid,
status=PIDStatus.REGISTERED)
pid.delete()
Record.create({'title': 'Live '}, id_=rec2_uuid)
# Record 3 - Deleted PID without a record
PersistentIdentifier.create(
'recid', '3', status=PIDStatus.DELETED)
# Record 4 - Registered PID without a record
PersistentIdentifier.create(
'recid', '4', status=PIDStatus.REGISTERED)
# Record 5 - Redirected PID
pid = PersistentIdentifier.create(
'recid', '5', status=PIDStatus.REGISTERED)
pid.redirect(pid1)
# Record 6 - Redirected non existing endpoint
doi = PersistentIdentifier.create(
'doi', '10.1234/foo', status=PIDStatus.REGISTERED)
pid = PersistentIdentifier.create(
'recid', '6', status=PIDStatus.REGISTERED)
pid.redirect(doi)
# Record 7 - Unregistered PID
PersistentIdentifier.create(
'recid', '7', status=PIDStatus.RESERVED)
def login_and_test(user_id):
login_user(User.query.get(user_id))
# Create record
user = User.query.get(user_id)
id = uuid.uuid4()
record = Record.create(access, id_=id)
factory = RecordPermission(record, action)
if user.has_role('admin'):
# super user can do EVERYTHING
assert factory.can()
elif user.has_role('librarian') and action != 'delete':
# librarian should be able to update, create, and read everything
assert factory.can()
else:
assert factory.can() if is_allowed else not factory.can()
def test_get_author_from_authors_a_list_with_two_elements():
authors_a_list_with_two_elements = Record({
'authors': [{
'full_name': 'Englert, F.'
}, {
'full_name': 'Brout, R.'
}]
})
cv_latex_html_text = Cv_latex_html_text(authors_a_list_with_two_elements,
'cv_latex_html', ',')
expected = ['F. Englert', 'R. Brout']
result = cv_latex_html_text._get_author()
assert expected == result
def test_get_publi_info_from_publication_info_with_journal_issue_latex_us():
journal_issue = Record({
'publication_info': [
{
'journal_title': 'Class.Quant.Grav.',
'journal_issue': '10'
}
]
})
latex_us = Latex(journal_issue, 'latex_us')
expected = ['Class.\\ Quant.\\ Grav.\\ , no. 10']
result = latex_us._get_publi_info()
assert expected == result
def test_operation_log_jsonresolver(item_lib_martigny):
"""Test operation logs json resolver."""
oplg = OperationLog.get_record_by_pid('1')
rec = Record.create({
'operation_log': {
'$ref': 'https://ils.rero.ch/api/operation_logs/1'
}
})
assert rec.replace_refs().get('operation_log') == \
{'pid': '1', 'type': 'oplg'}
# deleted record
oplg.delete()
with pytest.raises(JsonRefError):
rec.replace_refs().dumps()
# non existing record
rec = Record.create({
'operation_logs': {
'$ref': 'https://ils.rero.ch/api/operation_logs/n_e'
}
})
with pytest.raises(JsonRefError):
rec.replace_refs().dumps()
def test_get_publi_info_from_publication_info_with_journal_volume():
journal_volume = Record({
'publication_info': [
{
'journal_title': 'eConf',
'journal_volume': 'C050318'
}
]
})
latex = Latex(journal_volume, 'latex_eu')
expected = ['eConf C {\\bf 050318}']
result = latex._get_publi_info()
assert expected == result
def test_publication_info_from_pubinfo_freetext():
with_pubinfo_freetext = Record({
'publication_info': [
{'pubinfo_freetext': 'Phys. Rev. 127 (1962) 965-970'}
]
})
expected = {
'pub_info': [
'Phys. Rev. 127 (1962) 965-970'
]
}
result = publication_info(with_pubinfo_freetext)
assert expected == result
def test_get_publi_info_from_publication_info_with_journal_issue_latex_eu():
journal_issue = Record({
'publication_info': [
{
'journal_title': 'Int.J.Mod.Phys.',
'journal_issue': '29'
}
]
})
latex_eu = Latex(journal_issue, 'latex_eu')
expected = ['Int.\\ J.\\ Mod.\\ Phys.\\ 29, ']
result = latex_eu._get_publi_info()
assert expected == result
def test_get_publi_info_from_publication_info_with_year_not_a_list():
year_not_a_list = Record({
'publication_info': [
{
'journal_title': 'Phys.Lett.',
'year': '1999'
}
]
})
latex = Latex(year_not_a_list, 'latex_eu')
expected = ['Phys.\\ Lett.\\ (1999)']
result = latex._get_publi_info()
assert expected == result
def create_record(data):
"""Create a record.
:param dict data: The record data.
"""
with db.session.begin_nested():
# create uuid
rec_uuid = uuid.uuid4()
# create PID
current_pidstore.minters['recid'](rec_uuid, data)
# create record
created_record = Record.create(data, id_=rec_uuid)
# index the record
RecordIndexer().index(created_record)
db.session.commit()
def test_get_subtitle_returns_the_non_arxiv_subtitle():
double_subtitle = Record({
"titles": [{
"subtitle":
"Importance of a consistent choice of alpha(s) in the matching of AlpGen and Pythia"
}, {
"source": "arXiv",
"subtitle": "Monte Carlo tuning in the presence of Matching"
}],
})
expected = 'Importance of a consistent choice of alpha(s) in the matching of AlpGen and Pythia'
result = get_subtitle(double_subtitle)
assert expected == result
def test_get_abstract_with_multiple_sources_returns_the_non_arxiv_abstract():
double_abstract = Record({
"abstracts": [{
"source": "arXiv",
"value": "arXiv abstract"
}, {
"source": "other",
"value": "abstract"
}]
})
expected = 'abstract'
result = get_abstract(double_abstract)
assert expected == result
def test_get_publi_info_from_publication_info_with_year_a_list_of_two_elements(
):
year_a_list_of_two_elements = Record({
'publication_info': [{
'journal_title': 'Phys.Rev.Lett.',
'year': ['1999', '2000']
}]
})
cv_latex_html_text = Cv_latex_html_text(year_a_list_of_two_elements,
'cv_latex_html_text', ',')
expected = ['Submitted to:Phys.Rev.Lett. (2000)']
result = cv_latex_html_text._get_publi_info()
assert expected == result
def test_ill_requests_jsonresolver(ill_request_martigny):
"""Ill request resolver tests."""
rec = Record.create({
'ill_request': {
'$ref': 'https://ils.rero.ch/api/ill_requests/illr1'
}
})
assert rec.replace_refs().get('ill_request') == {
'type': 'illr',
'pid': 'illr1'
}
# deleted record
ill_request_martigny.delete()
with pytest.raises(JsonRefError):
rec.replace_refs().dumps()
# non existing record
rec = Record.create(
{'ill_request': {
'$ref': 'https://ils.rero.ch/api/ill_requests/n_e'
}})
with pytest.raises(JsonRefError):
rec.replace_refs().dumps()
def test_get_publi_info_from_publication_info_with_page_artid_a_list_of_one_element():
page_artid_a_list_of_one_element = Record({
'publication_info': [
{
'journal_title': 'Eur.Phys.J.',
'page_artid': ['2466']
}
]
})
latex = Latex(page_artid_a_list_of_one_element, 'latex_eu')
expected = ['Eur.\\ Phys.\\ J.\\ 2466']
result = latex._get_publi_info()
assert expected == result
def test_create_record(app, db):
"""Test create record task."""
r_id = Record.create({'title': 'existing', 'another_key': 'test2'}).id
db.session.commit()
# New record
assert RecordMetadata.query.count() == 1
create_record({'title': 'test'})
assert RecordMetadata.query.count() == 2
# Existing record id - no overwrite
create_record({'title': 'new val'}, id_=str(r_id))
assert RecordMetadata.query.count() == 2
assert Record.get_record(r_id)['title'] == 'existing'
# Clean session for SQLite.
db.session.expunge_all()
# Existing record id - with overwrite
create_record({'title': 'new val'}, id_=str(r_id), force=True)
assert RecordMetadata.query.count() == 2
r = Record.get_record(r_id)
assert r['title'] == 'new val'
assert 'another_key' not in r
def test_get_publi_info_from_publication_info_with_page_artid_a_list_of_two_elements(
):
page_artid_a_list_of_two_elements = Record({
'publication_info': [{
'journal_title': 'Phys.Rev.Lett.',
'page_artid': ['321-323', '1-188']
}]
})
cv_latex_html_text = Cv_latex_html_text(page_artid_a_list_of_two_elements,
'cv_latex_html_text', ',')
expected = ['Phys.Rev.Lett. 1-188']
result = cv_latex_html_text._get_publi_info()
assert expected == result
def test_get_doi_removes_duplicates():
with_duplicates = Record(
{'dois': [{
'value': 'foo'
}, {
'value': 'bar'
}, {
'value': 'foo'
}]})
expected = 'foo, bar'
result = Export(with_duplicates)._get_doi()
assert expected == result
def test_get_publi_info_from_publication_info_with_year_a_list_of_one_element():
year_a_list_of_one_element = Record({
'publication_info': [
{
'journal_title': 'JHEP',
'year': ['1999']
}
]
})
latex = Latex(year_a_list_of_one_element, 'latex_eu')
expected = ['JHEP (1999)']
result = latex._get_publi_info()
assert expected == result
def create_record(cls, dump):
"""Create a new record from dump."""
# Reserve record identifier, create record and recid pid in one
# operation.
timestamp, data = dump.pop_first_revision()
record = Record.create(data)
record.model.created = timestamp.replace(tzinfo=None)
RecordIdentifier.insert(dump.recid)
PersistentIdentifier.create(pid_type='recid',
pid_value=str(dump.recid),
object_type='rec',
object_uuid=str(record.id),
status=PIDStatus.REGISTERED)
db.session.commit()
return cls.update_record(dump, record=record)
def test_get_publi_info_from_publication_info_with_journal_volume_with_letter():
journal_volume_with_letter = Record({
'publication_info': [
{
'journal_title': 'Eur.Phys.J.',
'journal_volume': 'C73'
}
]
})
latex = Latex(journal_volume_with_letter, 'latex_eu')
expected = ['Eur.\\ Phys.\\ J.\\ C {\\bf 73}']
result = latex._get_publi_info()
assert expected == result
def test_get_publi_info_from_publication_info_with_year_a_list_of_two_elements():
year_a_list_of_two_elements = Record({
'publication_info': [
{
'journal_title': 'Phys.Rev.Lett.',
'year': ['1999', '2000']
}
]
})
latex = Latex(year_a_list_of_two_elements, 'latex_eu')
expected = ['Phys.\\ Rev.\\ Lett.\\ (2000)']
result = latex._get_publi_info()
assert expected == result
def test_get_publi_info_from_publication_info_with_page_artid_an_empty_list():
page_artid_an_empty_list = Record({
'publication_info': [
{
'journal_title': 'Phys.Lett.',
'page_artid': []
}
]
})
latex = Latex(page_artid_an_empty_list, 'latex_eu')
expected = ['Phys.\\ Lett.\\ ']
result = latex._get_publi_info()
assert expected == result
def funder_record(db):
"""Create a funder record."""
funder = Record.create(
dict(
doi='10.13039/501100000780',
name='European Commission',
acronyms=['EC'],
))
PersistentIdentifier.create(pid_type='frdoi',
pid_value=funder['doi'],
object_type='rec',
object_uuid=funder.id,
status='R')
db.session.commit()
return funder
def test_get_publi_info_from_publication_info_with_page_artid_not_a_list():
page_artid_not_a_list = Record({
'publication_info': [
{
'journal_title': 'JHEP',
'page_artid': '190'
}
]
})
latex = Latex(page_artid_not_a_list, 'latex_eu')
expected = ['JHEP 190']
result = latex._get_publi_info()
assert expected == result
def test_part_of(app, minimal_record, recid_pid):
"""Test journal record."""
minimal_record['part_of'] = {
'title': 'Conference proceedings title',
'pages': '10-20',
}
minimal_record['imprint'] = {
'publisher': 'The Good Publisher',
'place': 'Somewhere',
}
obj = csl_v1.transform_record(recid_pid, Record(minimal_record))
assert obj['container_title'] == 'Conference proceedings title'
assert obj['page'] == '10-20'
assert obj['publisher'] == 'The Good Publisher'
assert obj['publisher_place'] == 'Somewhere'
def test_get_publi_info_from_publication_info_with_page_artid_a_list_of_one_element(
):
page_artid_a_list_of_one_element = Record({
'publication_info': [{
'journal_title': 'Eur.Phys.J.',
'page_artid': ['2466']
}]
})
cv_latex_html_text = Cv_latex_html_text(page_artid_a_list_of_one_element,
'cv_latex_html_text', ',')
expected = ['Eur.Phys.J. 2466']
result = cv_latex_html_text._get_publi_info()
assert expected == result
def test_get_publi_info_from_publication_info_with_journal_volume_with_letter(
):
journal_volume_with_letter = Record({
'publication_info': [{
'journal_title': 'Eur.Phys.J.',
'journal_volume': 'C73'
}]
})
cv_latex_html_text = Cv_latex_html_text(journal_volume_with_letter,
'cv_latex_html_text', ',')
expected = ['Eur.Phys.J. C73']
result = cv_latex_html_text._get_publi_info()
assert expected == result
def test_get_publi_info_from_publication_info_with_year_an_empty_list():
year_an_empty_list = Record({
'publication_info': [
{
'journal_title': 'Phys.Rev.',
'year': []
}
]
})
latex = Latex(year_an_empty_list, 'latex_eu')
expected = ['Phys.\\ Rev.\\ ']
result = latex._get_publi_info()
assert expected == result
def test_format_output_row_more_than_eight_authors_collaboration_in_collaboration():
collaboration_in_collaboration = Record({
'collaboration': [
{'value': 'The ATLAS Collaboration'}
]
})
latex = Latex(collaboration_in_collaboration, 'latex_eu')
expected = u' G.~Aad {{\it et al.}} [The ATLAS Collaboration],\n'
result = latex._format_output_row('author', [
'G.~Aad', 'B.~Abbott', 'J.~Abdallah', 'O.~Abdinov', 'B.~Abeloos',
'R.~Aben', 'M.~Abolins', 'O.~AbouZeid', 'H.~Abramowicz'
])
assert expected == result
def get(self, record_id, **kwargs):
"""Get a Record.
:Parameters:
- `record_id`: id of the record to retrieve.
:Returns: The requested record.
"""
try:
record = Record.get_record(record_id)
except NoResultFound:
abort(404)
self.check_etag(str(record.model.version_id))
return record
