def test_appoint_profile_from_claimed_signature(small_app):
"""Check the module for the case where claimed signature takes
everything.
"""
from inspirehep.modules.disambiguation.tasks import disambiguation_clustering, update_authors_recid
old_record_id = str(PersistentIdentifier.get("literature", 11883).object_uuid)
old_record = get_es_record_by_uuid(old_record_id)
old_author_uuid = old_record["authors"][0]["uuid"]
# Add phonetic block to the record.
old_record["authors"][0]["signature_block"] = "HAGp"
old_record["authors"][0]["recid"] = "2"
es.index(index="records-hep", doc_type="hep", id=old_record_id, body=old_record)
es.indices.refresh("records-hep")
record_id = str(PersistentIdentifier.get("literature", 1358492).object_uuid)
record = get_es_record_by_uuid(record_id)
author_uuid = record["authors"][0]["uuid"]
# Add phonetic block to the record.
record["authors"][0]["signature_block"] = "HAGp"
record["authors"][0]["recid"] = "314159265"
record["authors"][0]["curated_relation"] = True
es.index(index="records-hep", doc_type="hep", id=record_id, body=record)
es.indices.refresh("records-hep")
with patch("celery.current_app.send_task", return_value=_BeardObject(({"2": [old_author_uuid, author_uuid]}, {}))):
with patch(
"inspirehep.modules.disambiguation.tasks.update_authors_recid.delay", side_effect=update_authors_recid
):
disambiguation_clustering("HAGp")
assert Record.get_record(old_record_id)["authors"][0]["recid"] == "314159265"
assert Record.get_record(record_id)["authors"][0]["recid"] == "314159265"
def test_search_pattern_change(app, without_oaiset_signals, schema):
"""Test search pattern change."""
record0 = create_record(app, {
'_oai': {'sets': ['a']}, 'title_statement': {'title': 'Test0'},
'$schema': schema
})
rec_uuid = record0.id
oaiset = OAISet(spec="a", search_pattern="title_statement.title:Test0")
db.session.add(oaiset)
db.session.commit()
run_after_insert_oai_set()
sleep(2)
record = Record.get_record(rec_uuid)
assert record['_oai']['sets'] == ['a']
# change search pattern: record0 will not inside it anymore
oaiset = OAISet.query.first()
oaiset.search_pattern = 'title_statement.title:Test1'
db.session.merge(oaiset)
db.session.commit()
after_update_oai_set(None, None, oaiset)
sleep(2)
record = Record.get_record(rec_uuid)
record.commit()
assert record['_oai']['sets'] == []
def delete_records(index_tree_id):
"""Bulk delete records."""
record_indexer = RecordIndexer()
hits = get_tree_items(index_tree_id)
for hit in hits:
recid = hit.get('_id')
record = Record.get_record(recid)
if record is not None and record['path'] is not None:
paths = record['path']
if len(paths) > 0:
# Remove the element which matches the index_tree_id
removed_path = None
for path in paths:
if path.endswith(str(index_tree_id)):
removed_path = path
paths.remove(path)
break
# Do update the path on record
record.update({'path': paths})
record.commit()
db.session.commit()
# Indexing
indexer = WekoIndexer()
indexer.update_path(record, update_revision=False)
if len(paths) == 0 and removed_path is not None:
from weko_deposit.api import WekoDeposit
WekoDeposit.delete_by_index_tree_id(removed_path)
Record.get_record(recid).delete() # flag as deleted
db.session.commit() # terminate the transaction
def test_recids_restricted_collection(self):
"""Test that recids API works"""
from invenio_ext.sqlalchemy import db
from invenio_search.api import Query
from invenio_records.api import Record
# FIXME All of this needs rewrite on Invenio 3.
self.assertEqual(len(Query("").search(collection='CDF Internal Notes').recids), 0)
# Adds special user info to allow test user to search in restricted collections
self.assertEqual(len(Query("").search(
collection='CDF Internal Notes',
user_info={'precached_permitted_restricted_collections': ['CDF Internal Notes']}
).recids), 1)
test_recid = 1396160
original_record = dict(Record.get_record(test_recid))
try:
rec = Record.get_record(test_recid)
rec['collections'].append({'primary': 'CDF-INTERNAL-NOTE'})
rec.commit()
# Wait for indexing
time.sleep(10)
self.assertEqual(len(Query("").search(
collection='CDF Internal Notes',
user_info={'precached_permitted_restricted_collections': ['CDF Internal Notes']}
).recids), 2)
finally:
rec = Record.get_record(test_recid)
rec.model.json = original_record
db.session.commit()
def test_update_authors_recid_method(small_app):
"""Test the method responsible for updating author's recid."""
from inspirehep.modules.disambiguation.tasks import update_authors_recid
pid = PersistentIdentifier.get("literature", 4328)
publication_id = str(pid.object_uuid)
signature = Record.get_record(publication_id)["authors"][0]["uuid"]
profile_recid = "314159265"
update_authors_recid(publication_id, signature, profile_recid)
assert Record.get_record(publication_id)["authors"][0]["recid"] == profile_recid
def test_record_unindex(app, test_users, test_records, script_info,
login_user):
"""Check that deleting a record also removes it from the search index."""
creator = test_users['deposits_creator']
with app.app_context():
Record.get_record(test_records[0].record_id).delete()
# execute scheduled tasks synchronously
process_bulk_queue.delay()
# flush the indices so that indexed records are searchable
current_search_client.indices.flush('*')
# deleted record should not be searchable
subtest_record_search(app, creator, test_records[1:], test_records,
login_user)
def test_record_unindex(app, test_users, test_records, script_info,
login_user):
"""Check that deleting a record also removes it from the search index."""
creator = test_users['deposits_creator']
with app.app_context():
Record.get_record(test_records[0].record_id).delete()
# execute scheduled tasks synchronously
process_bulk_queue.delay()
# flush the indices so that indexed records are searchable
current_search_client.indices.flush('*')
# deleted record should not be searchable
subtest_record_search(app, creator, test_records[1:], test_records,
login_user)
def test_update_authors_recid_method(small_app):
"""Test the method responsible for updating author's recid."""
from inspirehep.modules.disambiguation.tasks import update_authors_recid
pid = PersistentIdentifier.get("literature", 4328)
publication_id = str(pid.object_uuid)
signature = Record.get_record(publication_id)['authors'][0]['uuid']
profile_recid = "314159265"
update_authors_recid(publication_id, signature, profile_recid)
assert Record.get_record(publication_id)['authors'][0]['recid'] == \
profile_recid
def put(self, record_id, **kwargs):
"""Replace a Record metadata.
The body should be a json object, which will fully replace the current
record metadata.
:Parameters:
- `record_id` (int): id of the record to retrieve.
:Returns: Modified record.
"""
if request.content_type != 'application/json':
abort(415)
# TODO: accept non json content (MARC21...)
data = request.get_json()
if data is None:
abort(400)
try:
record = Record.get_record(record_id)
except NoResultFound:
abort(404)
self.check_etag(str(record.model.version_id))
record.clear()
record.update(data)
record.commit()
db.session.commit()
return self.make_response(record)
def patch(self, record_id, **kwargs):
"""Modify a Record.
The data should be a json-patch, which will be applied to the record.
:Parameters:
- `record_id` (int): id of the record to retrieve.
:Returns: Modified record.
"""
if request.content_type != 'application/json-patch+json':
abort(415)
# TODO: accept 'application/json' mediatype and use the object
# to replace the specified attributes
data = request.get_json(force=True)
if data is None:
abort(400)
try:
record = Record.get_record(record_id)
except NoResultFound:
abort(404)
self.check_etag(str(record.model.version_id))
try:
record = record.patch(data)
except (JsonPatchException, JsonPointerException):
abort(400)
record.commit()
db.session.commit()
return record
def recordstest(recid=None, with_traceback=False, with_dump=False):
"""Test records data migration."""
if recid:
uuids = [get_uuid_from_pid_value(recid)]
else:
uuids = get_record_uuids()
for uid in uuids:
record = Record.get_record(uid)
try:
if with_dump:
click.secho('# Before:', fg='green')
click.echo(json.dumps(record.dumps(), indent=2,
sort_keys=True))
record = transform_record(record)
record.pop('provisional_communities', None)
record.validate()
if with_dump:
click.secho('# After:', fg='green')
click.echo(json.dumps(record.dumps(), indent=2,
sort_keys=True))
# click.secho(
# 'Success: {0}'.format(record.get('recid', uid)), fg='green')
except Exception:
click.secho('Failure {0}'.format(record.get('recid', uid)),
fg='red')
if with_traceback:
traceback.print_exc()
def files_permission_factory(obj, action=None):
"""Permission for files are always based on the type of bucket.
1. Community bucket: Read access for everyone
2. Record bucket: Read access only with open and restricted access.
3. Deposit bucket: Read/update with restricted access.
4. Any other bucket is restricted to admins only.
"""
# Extract bucket id
bucket_id = None
if isinstance(obj, Bucket):
bucket_id = str(obj.id)
elif isinstance(obj, ObjectVersion):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, MultipartObject):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, FileObject):
bucket_id = str(obj.bucket_id)
# Retrieve record
if bucket_id is not None:
# Community bucket
if str(bucket_id) == current_app.config['COMMUNITIES_BUCKET_UUID']:
return CommunityBucketPermission(action)
# Record or deposit bucket
rb = RecordsBuckets.query.filter_by(bucket_id=bucket_id).one_or_none()
if rb is not None:
record = Record.get_record(rb.record_id)
if is_record(record):
return RecordFilesPermission.create(record, action)
elif is_deposit(record):
return DepositFilesPermission.create(record, action)
return DynamicPermission(ActionNeed('admin-access'))
def test_record_index_after_update(app, test_users, test_records, script_info,
login_user):
"""Check that updating a record also reindex it."""
creator = test_users['deposits_creator']
with app.app_context():
rec = Record.get_record(test_records[0].record_id)
pid = test_records[0].pid
rec.update({'title': 'my modified title'})
# execute scheduled tasks synchronously
process_bulk_queue.delay()
# flush the indices so that indexed records are searchable
current_search_client.indices.flush('*')
search_url = url_for('b2share_records_rest.b2rec_list')
headers = [('Content-Type', 'application/json'),
('Accept', 'application/json')]
with app.test_client() as client:
record_search_res = client.get(
search_url,
data='',
headers=headers)
assert record_search_res.status_code == 200
record_search_data = json.loads(
record_search_res.get_data(as_text=True))
assert record_search_data['hits']['total'] == len(test_records)
found_rec = [rec for rec in record_search_data['hits']['hits']
if rec['id'] == pid][0]
assert rec['title'] == 'my modified title'
def test_reharvest_fundref(app, db, es):
"""Test harvest_openaire_projects."""
with app.app_context():
harvest_fundref(source='tests/testdata/fundref_test.rdf')
assert PersistentIdentifier.query.count() == 6
assert RecordMetadata.query.count() == 5
recid = PersistentIdentifier.query.first().object_uuid
record = Record.get_record(recid)
record['title'] = 'Foobar'
record.commit()
db.session.commit()
harvest_fundref(source='tests/testdata/fundref_test.rdf')
assert PersistentIdentifier.query.count() == 6
assert RecordMetadata.query.count() == 5
record = Record.get_record(recid)
assert record['remote_modified'] != 'Foobar'
def can(self):
"""Cross-check user's CERN groups with the record's '_access' field."""
# Get record
rec = Record.get_record(record.id)
# Check access
return _has_access(rec)
def register(uuid):
"""Register record with given uuid in DataCite."""
record = Record.get_record(uuid)
experiment = record.get('experiment', None)
doi = record['doi']
try:
provider = DataCiteProviderWrapper.get(pid_value=doi,
pid_type='doi')
except PIDDoesNotExistError:
provider = DataCiteProviderWrapper.create(pid_value=doi,
experiment=experiment)
# serialize record to schema40
doc = DataCiteSerializer().dump(record).data
schema40.validate(doc)
doc = schema40.tostring(doc)
landing_page = '{}/{}'.format(
current_app.config.get('PIDSTORE_LANDING_BASE_URL'),
doi)
provider.register(url=landing_page,
doc=doc)
db.session.commit()
click.echo('Record registered with DOI {}'.format(doi))
def test_record_delete_legacy(dc_mock, app, db, users, deposit, deposit_file):
"""Delete the non-versioned record."""
deposit = publish_and_expunge(db, deposit)
recid, record = deposit.fetch_published()
# 'Simulate' a non-versioned record by removing 'conceptdoi' key
del deposit['conceptdoi']
del record['conceptdoi']
deposit.commit()
record.commit()
db.session.commit()
# Stash a copy of record metadata for later
rec = deepcopy(record)
record_uuid = str(record.id)
assert dc_mock().metadata_delete.call_count == 0
# users[0] is not an Admin but it doesn't matter in this case.
delete_record(record.id, 'spam', users[0]['id'])
# Make sure all PIDs are deleted
# TODO: oai PID is left registered
# assert PID.get('oai', rec['_oai']['id']) == PIDStatus.DELETED
assert PID.get('doi', rec['doi']).status == PIDStatus.DELETED
assert PID.get('recid', rec['recid']).status == PIDStatus.DELETED
assert PID.get('recid', rec['conceptrecid']).status == PIDStatus.DELETED
assert PID.get('depid', rec['_deposit']['id']).status == PIDStatus.DELETED
assert dc_mock().metadata_delete.call_count == 1
dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.2')
record = Record.get_record(record_uuid)
assert record['removed_by'] == users[0]['id']
assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def recordstest(recid=None, with_traceback=False, with_dump=False):
"""Test records data migration."""
if recid:
uuids = [get_uuid_from_pid_value(recid)]
else:
uuids = get_record_uuids()
for uid in uuids:
record = Record.get_record(uid)
try:
if with_dump:
click.secho('# Before:', fg='green')
click.echo(
json.dumps(record.dumps(), indent=2, sort_keys=True))
record = transform_record(record)
record.pop('provisional_communities', None)
record.validate()
if with_dump:
click.secho('# After:', fg='green')
click.echo(
json.dumps(record.dumps(), indent=2, sort_keys=True))
# click.secho(
# 'Success: {0}'.format(record.get('recid', uid)), fg='green')
except Exception:
click.secho(
'Failure {0}'.format(record.get('recid', uid)), fg='red')
if with_traceback:
traceback.print_exc()
def format_request_email_templ(increq, template, **ctx):
"""Format the email message element for inclusion request notification.
Formats the message according to the provided template file, using
some default fields from 'increq' object as default context.
Arbitrary context can be provided as keywords ('ctx'), and those will
not be overwritten by the fields from 'increq' object.
:param increq: Inclusion request object for which the request is made.
:type increq: `invenio_communities.models.InclusionRequest`
:param template: relative path to jinja template.
:type template: str
:param ctx: Optional extra context parameters passed to formatter.
:type ctx: dict.
:returns: Formatted message.
:rtype: str
"""
# Add minimal information to the contex (without overwriting).
curate_link = '{site_url}/communities/{id}/curate/'.format(
site_url=current_app.config['THEME_SITEURL'],
id=increq.community.id),
min_ctx = dict(
record=Record.get_record(increq.record.id),
requester=increq.user,
community=increq.community,
curate_link=curate_link,
)
for k, v in min_ctx.items():
if k not in ctx:
ctx[k] = v
msg_element = render_template_to_string(template, **ctx)
return msg_element
def load_oaiid(uuid):
"""Mint OAI ID information for the record.
:type uuid: str
"""
rec = Record.get_record(uuid)
recid = str(rec['recid'])
pid_value = current_app.config['OAISERVER_ID_PREFIX'] + recid
try:
pid = PersistentIdentifier.query.filter_by(pid_value=pid_value).one()
if str(pid.get_assigned_object()) == uuid:
rec.setdefault('_oai', {})
rec['_oai']['id'] = pid.pid_value
rec.commit()
db.session.commit()
logger.info('Matching OAI PID ({pid}) for {id}'.format(
pid=pid, id=uuid))
else:
logger.exception(
'OAI PID ({pid}) for record {id} ({recid}) is '
'pointing to a different object ({id2})'.format(
pid=pid, id=uuid, id2=str(pid.get_assigned_object()),
recid=recid))
except NoResultFound:
oaiid_minter(rec.id, rec)
rec.commit()
db.session.commit()
except MultipleResultsFound:
logger.exception(
'Multiple OAI PIDs found for record {id} '
'({recid})'.format(id=uuid, recid=recid))
def ill_request(record_id=None):
"""Interface to request an inter library loan for the user.
Without a record_id, an empty form will be presented.
"""
try:
get_user(current_user)
except AttributeError:
# Anonymous User
return render_template('invenio_theme/401.html')
if record_id:
_uuid = PersistentIdentifier.get('recid', record_id).object_uuid
rec = Record.get_record(_uuid)
else:
rec = {}
_prepare_record(rec, rec_fields)
_prepare_record_authors(rec)
start_date = datetime.date.today().isoformat()
end_date = datetime.date.today() + datetime.timedelta(weeks=4)
return render_template('circulation_ill_request.html',
action='request', record_id=record_id,
start_date=start_date, end_date=end_date, **rec)
def files_permission_factory(obj, action=None):
"""Permission for files are always based on the type of bucket.
1. Community bucket: Read access for everyone
2. Record bucket: Read access only with open and restricted access.
3. Deposit bucket: Read/update with restricted access.
4. Any other bucket is restricted to admins only.
"""
# Extract bucket id
bucket_id = None
if isinstance(obj, Bucket):
bucket_id = str(obj.id)
elif isinstance(obj, ObjectVersion):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, MultipartObject):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, FileObject):
bucket_id = str(obj.bucket_id)
# Retrieve record
if bucket_id is not None:
# Community bucket
if str(bucket_id) == current_app.config['COMMUNITIES_BUCKET_UUID']:
return CommunityBucketPermission(action)
# Record or deposit bucket
rb = RecordsBuckets.query.filter_by(bucket_id=bucket_id).one_or_none()
if rb is not None:
record = Record.get_record(rb.record_id)
if is_record(record):
return RecordFilesPermission.create(record, action)
elif is_deposit(record):
return DepositFilesPermission.create(record, action)
return DynamicPermission(ActionNeed('admin-access'))
def test_record_delete(mocker, app, db, users, deposit, deposit_file):
"""Delete the record with a single version."""
dc_mock = mocker.patch(
'invenio_pidstore.providers.datacite.DataCiteMDSClient')
deposit = publish_and_expunge(db, deposit)
recid, record = deposit.fetch_published()
# Stash a copy of record metadata for later
rec = deepcopy(record)
record_uuid = str(record.id)
assert dc_mock().metadata_delete.call_count == 0
# users[0] is not an Admin but it doesn't matter in this case.
delete_record(record.id, 'spam', users[0]['id'])
# Make sure all PIDs are deleted
# TODO: oai PID is left registered
# assert PID.get('oai', rec['_oai']['id']) == PIDStatus.DELETED
assert PID.get('doi', rec['doi']).status == PIDStatus.DELETED
assert PID.get('doi', rec['conceptdoi']).status == PIDStatus.DELETED
assert PID.get('recid', rec['recid']).status == PIDStatus.DELETED
assert PID.get('recid', rec['conceptrecid']).status == PIDStatus.DELETED
assert PID.get('depid', rec['_deposit']['id']).status == PIDStatus.DELETED
assert dc_mock().metadata_delete.call_count == 2
dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.1')
dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.2')
record = Record.get_record(record_uuid)
assert record['removed_by'] == users[0]['id']
assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def ill_register(record_id=None):
"""Interface to register an inter library loan for the administrator.
Without a record_id, an empty form will be presented.
"""
if record_id:
_uuid = PersistentIdentifier.get("recid", record_id).object_uuid
rec = Record.get_record(_uuid)
else:
rec = {}
_prepare_record(rec, rec_fields)
_prepare_record_authors(rec)
start_date = datetime.date.today().isoformat()
end_date = datetime.date.today() + datetime.timedelta(weeks=4)
return render_template(
"circulation_ill_register.html",
action="register",
record_id=record_id,
start_date=start_date,
end_date=end_date,
**rec
)
def index_by_id(self, record_uuid, **kwargs):
"""Index a record by record identifier.
:param record_uuid: Record identifier.
:param kwargs: Passed to :meth:`RecordIndexer.index`.
"""
return self.index(Record.get_record(record_uuid), **kwargs)
def _index_action(self, payload):
"""Bulk index action.
:param payload: Decoded message body.
:returns: Dictionary defining an Elasticsearch bulk 'index' action.
"""
record = Record.get_record(payload['id'])
index, doc_type = self.record_to_index(record)
arguments = {}
body = self._prepare_record(record, index, doc_type, arguments)
index, doc_type = self._prepare_index(index, doc_type)
action = {
'_op_type': 'index',
'_index': index,
'_type': doc_type,
'_id': str(record.id),
'_version': record.revision_id,
'_version_type': self._version_type,
'_source': body
}
action.update(arguments)
return action
def ill_request(record_id=None):
"""Interface to request an inter library loan for the user.
Without a record_id, an empty form will be presented.
"""
try:
get_user(current_user)
except AttributeError:
# Anonymous User
return render_template("invenio_theme/401.html")
if record_id:
_uuid = PersistentIdentifier.get("recid", record_id).object_uuid
rec = Record.get_record(_uuid)
else:
rec = {}
_prepare_record(rec, rec_fields)
_prepare_record_authors(rec)
start_date = datetime.date.today().isoformat()
end_date = datetime.date.today() + datetime.timedelta(weeks=4)
return render_template(
"circulation_ill_request.html",
action="request",
record_id=record_id,
start_date=start_date,
end_date=end_date,
**rec
)
def files_permission_factory(obj, action=None):
"""Permission for files are always based on the type of record.
Record bucket: Read access only with open access.
Deposit bucket: Read/update with restricted access.
"""
# Extract bucket id
bucket_id = None
if isinstance(obj, Bucket):
bucket_id = str(obj.id)
elif isinstance(obj, ObjectVersion):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, MultipartObject):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, FileObject):
bucket_id = str(obj.bucket_id)
# Retrieve record
if bucket_id is not None:
# Record or deposit bucket
rb = RecordsBuckets.query.filter_by(bucket_id=bucket_id).one_or_none()
if rb is not None:
record = Record.get_record(rb.record_id)
if is_publication(record.model):
return PublicationFilesPermission(record, action)
elif is_deposit(record.model):
return DepositFilesPermission(record, action)
return DynamicPermission(superuser_access)
def tearDown(self):
from invenio_ext.sqlalchemy import db
from invenio_records.api import Record
rec = Record.get_record(self.test_recid)
rec.model.json = self.original_record
db.session.commit()
def test_reharvest_fundref(app, db, es):
"""Test harvest_openaire_projects."""
with app.app_context():
harvest_fundref(source='tests/testdata/fundref_test.rdf')
assert PersistentIdentifier.query.count() == 6
assert RecordMetadata.query.count() == 5
recid = PersistentIdentifier.query.first().object_uuid
record = Record.get_record(recid)
record['title'] = 'Foobar'
record.commit()
db.session.commit()
harvest_fundref(source='tests/testdata/fundref_test.rdf')
assert PersistentIdentifier.query.count() == 6
assert RecordMetadata.query.count() == 5
record = Record.get_record(recid)
assert record['remote_modified'] != 'Foobar'
def format_request_email_templ(increq, template, **ctx):
"""Format the email message element for inclusion request notification.
Formats the message according to the provided template file, using
some default fields from 'increq' object as default context.
Arbitrary context can be provided as keywords ('ctx'), and those will
not be overwritten by the fields from 'increq' object.
:param increq: Inclusion request object for which the request is made.
:type increq: `invenio_communities.models.InclusionRequest`
:param template: relative path to jinja template.
:type template: str
:param ctx: Optional extra context parameters passed to formatter.
:type ctx: dict.
:returns: Formatted message.
:rtype: str
"""
# Add minimal information to the contex (without overwriting).
curate_link = '{site_url}/communities/{id}/curate/'.format(
site_url=current_app.config['THEME_SITEURL'],
id=increq.community.id)
min_ctx = dict(
record=Record.get_record(increq.record.id),
requester=increq.user,
community=increq.community,
curate_link=curate_link,
)
for k, v in min_ctx.items():
if k not in ctx:
ctx[k] = v
msg_element = render_template_to_string(template, **ctx)
return msg_element
def delete_by_id(self, record_uuid, **kwargs):
"""Delete record from index by record identifier.
:param record_uuid: Record identifier.
:param kwargs: Passed to :meth:`RecordIndexer.delete`.
"""
self.delete(Record.get_record(record_uuid), **kwargs)
def reconstruct_sipfiles_t(recid=None, pid=None):
"""Reconstruct SIPFiles from record metadata."""
if not pid:
pid = PersistentIdentifier.get('recid', recid)
recsip = RecordSIP.query.filter_by(pid_id=pid.id).order_by(
RecordSIP.created).first()
if recsip is None:
raise Exception("RecordSIP does not exist.")
sip = recsip.sip
record = Record.get_record(recsip.pid.object_uuid)
first_json_rec = \
next((rec for rec in record.revisions if '_files' in rec), None)
if first_json_rec is None:
raise Exception("Files information not found in SIPMetadata nor"
" in Record revision")
files_j = first_json_rec['_files']
ovs = [
ObjectVersion.query.filter_by(version_id=fj['version_id']).first()
for fj in files_j
]
for ov in ovs:
q = SIPFile.query.filter_by(sip_id=sip.id,
filepath=ov.key,
file_id=ov.file_id)
if not q.count():
obj = SIPFile(sip_id=sip.id,
filepath=ov.key,
file_id=ov.file_id,
created=sip.created)
db.session.add(obj)
db.session.commit()
def load_oaiid(uuid):
"""Mint OAI ID information for the record.
:type uuid: str
"""
rec = Record.get_record(uuid)
recid = str(rec['recid'])
pid_value = current_app.config['OAISERVER_ID_PREFIX'] + recid
try:
pid = PersistentIdentifier.query.filter_by(pid_value=pid_value).one()
if str(pid.get_assigned_object()) == uuid:
rec.setdefault('_oai', {})
rec['_oai']['id'] = pid.pid_value
rec.commit()
db.session.commit()
logger.info('Matching OAI PID ({pid}) for {id}'.format(pid=pid,
id=uuid))
else:
logger.exception('OAI PID ({pid}) for record {id} ({recid}) is '
'pointing to a different object ({id2})'.format(
pid=pid,
id=uuid,
id2=str(pid.get_assigned_object()),
recid=recid))
except NoResultFound:
oaiid_minter(rec.id, rec)
rec.commit()
db.session.commit()
except MultipleResultsFound:
logger.exception('Multiple OAI PIDs found for record {id} '
'({recid})'.format(id=uuid, recid=recid))
def update(recid):
"""Update metadata for record with given recid in DataCite."""
uuid = PersistentIdentifier.get('recid', recid).object_uuid
record = Record.get_record(uuid)
doi = record['doi']
try:
provider = DataCiteProviderWrapper.get(pid_value=doi,
pid_type='doi')
except PIDDoesNotExistError:
raise ClickException('Record with DOI {} not registered in DataCite.'
.format(doi))
# serialize record to schema40
doc = DataCiteSerializer().dump(record).data
schema40.validate(doc)
doc = schema40.tostring(doc)
landing_page = os.path.join(
current_app.config.get('PIDSTORE_LANDING_BASE_URL'),
recid)
provider.update(url=landing_page,
doc=doc)
db.session.commit()
click.echo('Record with DOI {} updated in DataCite'.format(doi))
def register(recid):
"""Register record with given recid in DataCite."""
uuid = PersistentIdentifier.get('recid', recid).object_uuid
record = Record.get_record(uuid)
experiment = record.get('experiment', None)
doi = record['doi']
try:
provider = DataCiteProviderWrapper.get(pid_value=doi,
pid_type='doi')
except PIDDoesNotExistError:
provider = DataCiteProviderWrapper.create(pid_value=doi,
experiment=experiment)
# serialize record to schema40
doc = DataCiteSerializer().dump(record).data
schema40.validate(doc)
doc = schema40.tostring(doc)
landing_page = os.path.join(
current_app.config.get('PIDSTORE_LANDING_BASE_URL'),
recid)
provider.register(url=landing_page,
doc=doc)
db.session.commit()
click.echo('Record registered with DOI {}'.format(doi))
def files_permission_factory(obj, action=None):
"""Permission for files are always based on the type of record.
Record bucket: Read access only with open access.
Deposit bucket: Read/update with restricted access.
"""
# Extract bucket id
bucket_id = None
if isinstance(obj, Bucket):
bucket_id = str(obj.id)
elif isinstance(obj, ObjectVersion):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, MultipartObject):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, FileObject):
bucket_id = str(obj.bucket_id)
# Retrieve record
if bucket_id is not None:
# Record or deposit bucket
rb = RecordsBuckets.query.filter_by(bucket_id=bucket_id).one_or_none()
if rb is not None:
record = Record.get_record(rb.record_id)
if is_publication(record.model):
return PublicationFilesPermission(record, action)
elif is_deposit(record.model):
return DepositFilesPermission(record, action)
return DynamicPermission(superuser_access)
def test_record_index_after_update(app, test_users, test_records, script_info,
login_user):
"""Check that updating a record also reindex it."""
creator = test_users['deposits_creator']
with app.app_context():
rec = Record.get_record(test_records[0].record_id)
pid = test_records[0].pid
rec.update({'title': 'my modified title'})
# execute scheduled tasks synchronously
process_bulk_queue.delay()
# flush the indices so that indexed records are searchable
current_search_client.indices.flush('*')
search_url = url_for('b2share_records_rest.b2rec_list')
headers = [('Content-Type', 'application/json'),
('Accept', 'application/json')]
with app.test_client() as client:
record_search_res = client.get(
search_url,
data='',
headers=headers)
assert record_search_res.status_code == 200
record_search_data = json.loads(
record_search_res.get_data(as_text=True))
assert record_search_data['hits']['total'] == len(test_records)
found_rec = [rec for rec in record_search_data['hits']['hits']
if rec['id'] == pid][0]
assert rec['title'] == 'my modified title'
def add_oaiset_spec(record_uuid, spec):
"""Add the OAI spec to the record and commit."""
rec = Record.get_record(record_uuid)
rec['_oai']['sets'] = sorted(rec['_oai'].get('sets', []) + [spec, ])
rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow())
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([str(rec.id), ])
def test_reharvest_fundref(app, db):
"""Test harvest_openaire_projects."""
with app.app_context():
harvest_fundref(path='tests/testdata/fundref_test.rdf')
assert PersistentIdentifier.query.count() == 6
assert RecordMetadata.query.count() == 5
recid = PersistentIdentifier.query.first().object_uuid
test_date = "2002-01-01T16:00:00.000000"
record = Record.get_record(recid)
record['remote_modified'] = test_date
record.commit()
db.session.commit()
harvest_fundref(path='tests/testdata/fundref_test.rdf')
assert PersistentIdentifier.query.count() == 6
assert RecordMetadata.query.count() == 5
record = Record.get_record(recid)
assert record['remote_modified'] != test_date
def test_reharvest_fundref(app):
"""Test harvest_openaire_projects."""
with app.app_context():
harvest_fundref(path='tests/testdata/fundref_test.rdf')
assert PersistentIdentifier.query.count() == 5
assert RecordMetadata.query.count() == 5
recid = PersistentIdentifier.query.first().object_uuid
test_date = "2002-01-01T16:00:00.000000"
record = Record.get_record(recid)
record['remote_modified'] = test_date
record.commit()
db.session.commit()
harvest_fundref(path='tests/testdata/fundref_test.rdf')
assert PersistentIdentifier.query.count() == 5
assert RecordMetadata.query.count() == 5
record = Record.get_record(recid)
assert record['remote_modified'] != test_date
def test_appoint_profile_from_claimed_signature(small_app):
"""Check the module for the case where claimed signature takes
everything.
"""
from inspirehep.modules.disambiguation.tasks import (
disambiguation_clustering, update_authors_recid)
old_record_id = str(
PersistentIdentifier.get("literature", 11883).object_uuid)
old_record = get_es_record_by_uuid(old_record_id)
old_author_uuid = old_record['authors'][0]['uuid']
# Add phonetic block to the record.
old_record['authors'][0]['signature_block'] = "HAGp"
old_record['authors'][0]['recid'] = "2"
es.index(index='records-hep',
doc_type='hep',
id=old_record_id,
body=old_record)
es.indices.refresh('records-hep')
record_id = str(
PersistentIdentifier.get("literature", 1358492).object_uuid)
record = get_es_record_by_uuid(record_id)
author_uuid = record['authors'][0]['uuid']
# Add phonetic block to the record.
record['authors'][0]['signature_block'] = "HAGp"
record['authors'][0]['recid'] = "314159265"
record['authors'][0]['curated_relation'] = True
es.index(index='records-hep', doc_type='hep', id=record_id, body=record)
es.indices.refresh('records-hep')
with patch("celery.current_app.send_task",
return_value=_BeardObject(({
"2": [old_author_uuid, author_uuid]
}, {}))):
with patch(
"inspirehep.modules.disambiguation.tasks.update_authors_recid.delay",
side_effect=update_authors_recid):
disambiguation_clustering("HAGp")
assert Record.get_record(old_record_id)['authors'][0]['recid'] == \
"314159265"
assert Record.get_record(record_id)['authors'][0]['recid'] == \
"314159265"
def test_record_delete_v2(mocker, app, db, users, deposit, deposit_file):
"""Delete a record (only last version) with multiple versions."""
dc_mock = mocker.patch(
'invenio_pidstore.providers.datacite.DataCiteMDSClient')
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1.newversion()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
# Stash a copy of v1 for later
rec1 = deepcopy(record_v1)
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
# Stash a copy of v2 for later
rec2 = deepcopy(record_v2)
rec2_id = str(record_v2.id)
assert dc_mock().metadata_delete.call_count == 0
# Remove the first version
delete_record(rec2_id, 'spam', users[0]['id'])
# Make sure all PIDs are deleted
assert PID.get('doi', rec2['doi']).status == PIDStatus.DELETED
assert PID.get('recid', rec2['recid']).status == PIDStatus.DELETED
assert PID.get('depid', rec2['_deposit']['id']).status == PIDStatus.DELETED
# Concept DOI should be left registered
assert PID.get('doi', rec2['conceptdoi']).status == PIDStatus.REGISTERED
# Make sure conceptrecid is redirecting to v1
crecid = PID.get('recid', rec2['conceptrecid'])
assert crecid.status == PIDStatus.REDIRECTED
assert crecid.get_redirect() == PID.get('recid', rec1['recid'])
# Make sure the v1 PIDs are kept intact
assert PID.get('oai', rec1['_oai']['id']).status == PIDStatus.REGISTERED
assert PID.get('doi', rec1['doi']).status == PIDStatus.REGISTERED
assert PID.get('recid', rec1['recid']).status == PIDStatus.REGISTERED
assert PID.get('depid', rec1['_deposit']['id']).status == \
PIDStatus.REGISTERED
# Only the v1 DOI should be deleted
assert dc_mock().doi_post.call_count == 2
assert dc_mock().doi_post.has_any_call('10.5072/zenodo.2')
assert dc_mock().doi_post.has_any_call('10.5072/zenodo.1')
assert dc_mock().metadata_delete.call_count == 1
dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.3')
record = Record.get_record(rec2_id)
assert record['removed_by'] == users[0]['id']
assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def get_dep_uuids(rec_uuids):
"""Get corresponding deposit UUIDs from record's UUIDs."""
return [
str(
PersistentIdentifier.get(
'depid',
Record.get_record(id_)['_deposit']['id']).object_uuid)
for id_ in rec_uuids
]
def update_records_sets(record_ids):
"""Update records sets.
:param record_ids: List of record UUID.
"""
for record_id in record_ids:
record = Record.get_record(record_id)
record.commit()
db.session.commit()
def test_solve_claim_conflicts(small_app):
"""Check the module for the case where at least two claimed
signatures are assigned to the same cluster.
"""
from inspirehep.modules.disambiguation.tasks import disambiguation_clustering, update_authors_recid
# Claimed signature #1.
glashow_record_id_claimed = str(PersistentIdentifier.get("literature", 4328).object_uuid)
glashow_record_claimed = get_es_record_by_uuid(glashow_record_id_claimed)
glashow_record_uuid_claimed = glashow_record_claimed["authors"][0]["uuid"]
# Add phonetic block to the record.
glashow_record_claimed["authors"][0]["signature_block"] = "HAGp"
glashow_record_claimed["authors"][0]["curated_relation"] = True
glashow_record_claimed["authors"][0]["recid"] = "3"
es.index(index="records-hep", doc_type="hep", id=glashow_record_id_claimed, body=glashow_record_claimed)
es.indices.refresh("records-hep")
# Claimed signature #2.
higgs_record_id_claimed = str(PersistentIdentifier.get("literature", 1358492).object_uuid)
higgs_record_claimed = get_es_record_by_uuid(higgs_record_id_claimed)
higgs_record_uuid_claimed = higgs_record_claimed["authors"][0]["uuid"]
# Add phonetic block to the record.
higgs_record_claimed["authors"][0]["signature_block"] = "HAGp"
higgs_record_claimed["authors"][0]["curated_relation"] = True
higgs_record_claimed["authors"][0]["recid"] = "4"
es.index(index="records-hep", doc_type="hep", id=higgs_record_id_claimed, body=higgs_record_claimed)
es.indices.refresh("records-hep")
# Not claimed signature.
higgs_record_id_not_claimed = str(PersistentIdentifier.get("literature", 11883).object_uuid)
higgs_record_not_claimed = get_es_record_by_uuid(higgs_record_id_not_claimed)
higgs_record_uuid_not_claimed = higgs_record_not_claimed["authors"][0]["uuid"]
# Add phonetic block to the record.
higgs_record_not_claimed["authors"][0]["signature_block"] = "HAGp"
es.index(index="records-hep", doc_type="hep", id=higgs_record_id_not_claimed, body=higgs_record_not_claimed)
es.indices.refresh("records-hep")
with patch(
"celery.current_app.send_task",
return_value=_BeardObject(
({"3": [glashow_record_uuid_claimed, higgs_record_uuid_claimed, higgs_record_uuid_not_claimed]}, {})
),
):
with patch(
"inspirehep.modules.disambiguation.logic._solve_claims_conflict",
return_value=_ConflictObject({higgs_record_uuid_claimed: [higgs_record_uuid_not_claimed]}),
):
with patch(
"inspirehep.modules.disambiguation.tasks.update_authors_recid.delay", side_effect=update_authors_recid
):
disambiguation_clustering("HAGp")
assert Record.get_record(higgs_record_id_not_claimed)["authors"][0]["recid"] == "4"
def migrate_deposit(record_uuid):
"""Migrate a record.
:param record_uuid: UUID of the Deposit record.
:type record_uuid: str
"""
# Get the deposit
deposit = transform_deposit(Record.get_record(record_uuid))
deposit.commit()
db.session.commit()
def migrate_deposit(record_uuid):
"""Migrate a record.
:param record_uuid: UUID of the Deposit record.
:type record_uuid: str
"""
# Get the deposit
deposit = transform_deposit(Record.get_record(record_uuid))
deposit.commit()
db.session.commit()
def test_append_updated_record_to_queue_same_data(small_app):
"""Check if for the same record, the receiver will skip the publication."""
pid = PersistentIdentifier.get("literature", 11883)
publication_id = str(pid.object_uuid)
record = Record.get_record(publication_id)
append_updated_record_to_queue(None, record, record, "records-hep", "hep")
assert str(record.id) != \
DisambiguationRecord.query.order_by(desc("id")).first().record_id
def test_append_updated_record_to_queue_same_data(small_app):
"""Check if for the same record, the receiver will skip the publication."""
pid = PersistentIdentifier.get("literature", 11883)
publication_id = str(pid.object_uuid)
record = Record.get_record(publication_id)
append_updated_record_to_queue(None, record, record, "records-hep", "hep")
assert str(record.id) != \
DisambiguationRecord.query.order_by(desc("id")).first().record_id
def test_update_record(self):
"""Test updating record."""
from invenio_ext.sqlalchemy import db
from invenio_records.api import Record
rec = Record.get_record(self.test_recid)
assert rec["languages"] == ["Croatian"]
# 1. Need to make a new clean dict and update it
new_dict = dict(rec)
new_dict["languages"] = ["Norwegian"]
# 2. Create new instance with updated dict, but same model (recid etc.)
rec = Record(new_dict, model=rec.model)
rec.commit()
db.session.commit()
rec = Record.get_record(self.test_recid)
assert rec["languages"] == ["Norwegian"]
def request(community_id, record_id, accept):
"""Request a record acceptance to a community."""
c = Community.get(community_id)
assert c is not None
record = Record.get_record(record_id)
if accept:
c.add_record(record)
db.session.commit()
else:
InclusionRequest.create(community=c, record=record)
db.session.commit()
def acquisition_register(record_id=None):
if record_id:
_uuid = PersistentIdentifier.get('recid', record_id).object_uuid
rec = Record.get_record(_uuid)
else:
rec = {}
_prepare_record(rec, rec_fields)
_prepare_record_authors(rec)
return render_template('circulation_acquisition_request.html',
type='acquisition', action='register', **rec)
def remove_oaiset_spec(record_uuid, spec):
"""Remove the OAI spec from the record and commit."""
rec = Record.get_record(record_uuid)
rec['_oai']['sets'] = sorted([s for s in rec['_oai'].get('sets', [])
if s != spec])
rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow())
if not rec['_oai']['sets']:
del rec['_oai']['sets']
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([str(rec.id), ])
def get_record_title(id, type='recid'):
"""Fetches record title by id."""
from invenio_records.api import Record
from invenio_pidstore.models import PersistentIdentifier
from invenio_pidstore.errors import PIDDoesNotExistError
try:
pid = PersistentIdentifier.get(type, id)
except PIDDoesNotExistError:
return None
record = Record.get_record(pid.object_uuid)
return record.get('title', '')
def create_record(data, force=False, dry_run=False):
record = marc_create_record(data)
recid = None
if '001' in record:
recid = int(record['001'][0])
if not dry_run and recid:
prod_record = InspireProdRecords(recid=recid)
prod_record.marcxml = data
try:
if _collection_in_record(record, 'institution'):
json = strip_empty_values(institutions.do(record))
elif _collection_in_record(record, 'experiment'):
json = strip_empty_values(experiments.do(record))
elif _collection_in_record(record, 'journals'):
json = strip_empty_values(journals.do(record))
elif _collection_in_record(record, 'hepnames'):
json = strip_empty_values(hepnames.do(record))
elif _collection_in_record(record, 'job') or \
_collection_in_record(record, 'jobhidden'):
json = strip_empty_values(jobs.do(record))
elif _collection_in_record(record, 'conferences'):
json = strip_empty_values(conferences.do(record))
else:
json = strip_empty_values(hep.do(record))
if dry_run:
return recid, json
if force and any(key in json for key in ('control_number', 'recid')):
try:
control_number = json['control_number']
except KeyError:
control_number = json['recid']
control_number = int(control_number)
# Searches if record already exists.
record = Record.get_record(control_number)
if record is None:
# Adds the record to the db session.
rec = RecordModel(id=control_number)
db.session.merge(rec)
record = Record.create(json)
else:
record = Record(json, model=record.model)
record.commit()
if recid:
prod_record.successful = True
db.session.merge(prod_record)
logger.info("Elaborated record {}".format(control_number))
return control_number, dict(record)
except Exception:
if recid:
prod_record.successful = False
db.session.merge(prod_record)
logger.exception("Error in elaborating record ID {}".format(recid))
raise
def repair_record_metadata(uuid):
"""Repair the record's metadata using a reference revision."""
rec = Record.get_record(uuid)
good_revision = recent_non_corrupted_revision(rec)
if '_internal' in good_revision:
rec['_internal'] = good_revision['_internal']
files_diff = list(diff(rec['_files'], good_revision['_files']))
if files_diff_safe(files_diff):
rec['_files'] = good_revision['_files']
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([str(rec.id), ])
