def test_valid_delete(app):
"""Test VALID record delete request (DELETE .../records/<record_id>)."""
with app.app_context():
# create the record using the internal API
pid, record = create_record(test_data)
with app.test_client() as client:
headers = [('Accept', 'application/json')]
res = client.delete(url_for('invenio_records_rest.recid_item',
pid_value=pid.pid_value),
headers=headers)
assert res.status_code == 204
# check database state
with pytest.raises(NoResultFound):
Record.get_record(record.id)
assert pid.is_deleted()
# check that DELETE with non JSON accepted format will work
# as it returns nothing
pid, record = create_record(test_data)
headers = [('Accept', 'video/mp4')]
res = client.delete(url_for('invenio_records_rest.recid_item',
pid_value=pid.pid_value),
headers=headers)
assert res.status_code == 204
def test_valid_put(app):
"""Test VALID record patch request (PATCH .../records/<record_id>)."""
with app.app_context():
InvenioRecordsREST(app)
# create the record using the internal API
internal_record = Record.create(test_data)
with app.test_client() as client:
headers = [('Content-Type', 'application/json'),
('Accept', 'application/json')]
res = client.put(url_for(blueprint.name + '.' +
RecordResource.view_name,
record_id=internal_record.model.id),
data=json.dumps(test_data_patched),
headers=headers)
assert res.status_code == 200
# check that the returned record matches the given data
response_data = json.loads(res.get_data(as_text=True))
assert response_data['data'] == test_data_patched
# check that an internal record returned id and that it contains
# the same data
assert 'id' in response_data.keys()
internal_record = Record.get_record(response_data['id'])
assert internal_record == response_data['data']
# check that the returned self link returns the same data
subtest_self_link(response_data, res.headers, client)
def test_validate_with_format(app, db):
"""Test that validation can accept custom format rules."""
with app.app_context():
checker = FormatChecker()
checker.checks("foo")(lambda el: el.startswith("foo"))
data = {"bar": "foo", "$schema": {"properties": {"bar": {"format": "foo"}}}}
# test record creation with valid data
assert data == Record.create(data)
record = Record.create(data, format_checker=checker)
# test direct call to validate with valid data
assert record.validate(format_checker=checker) is None
# test commit with valid data
record.commit(format_checker=checker)
record["bar"] = "bar"
# test direct call to validate with invalid data
with pytest.raises(ValidationError) as excinfo:
record.validate(format_checker=checker)
assert "'bar' is not a 'foo'" in str(excinfo.value)
# test commit with invalid data
with pytest.raises(ValidationError) as excinfo:
record.commit(format_checker=checker)
assert "'bar' is not a 'foo'" in str(excinfo.value)
data["bar"] = "bar"
# test record creation with invalid data
with pytest.raises(ValidationError) as excinfo:
record = Record.create(data, format_checker=checker)
assert "'bar' is not a 'foo'" in str(excinfo.value)
def test_valid_delete(app):
"""Test VALID record delete request (DELETE .../records/<record_id>)."""
with app.app_context():
# create the record using the internal API
pid, record = create_record(test_data)
with app.test_client() as client:
headers = [('Accept', 'application/json')]
res = client.delete(url_for('invenio_records_rest.recid_item',
pid_value=pid.pid_value),
headers=headers)
assert res.status_code == 204
# check database state
with pytest.raises(NoResultFound):
Record.get_record(record.id)
assert pid.is_deleted()
# check that DELETE with non JSON accepted format will work
# as it returns nothing
pid, record = create_record(test_data)
headers = [('Accept', 'video/mp4')]
res = client.delete(url_for('invenio_records_rest.recid_item',
pid_value=pid.pid_value),
headers=headers)
assert res.status_code == 204
def test_valid_put(app):
"""Test VALID record patch request (PATCH .../records/<record_id>)."""
with app.app_context():
InvenioRecordsREST(app)
# create the record using the internal API
internal_record = Record.create(test_data)
with app.test_client() as client:
headers = [('Content-Type', 'application/json'),
('Accept', 'application/json')]
res = client.put(url_for(blueprint.name + '.' +
RecordResource.view_name,
record_id=internal_record.model.id),
data=json.dumps(test_data_patched),
headers=headers)
assert res.status_code == 200
# check that the returned record matches the given data
response_data = json.loads(res.get_data(as_text=True))
assert response_data['data'] == test_data_patched
# check that an internal record returned id and that it contains
# the same data
assert 'id' in response_data.keys()
internal_record = Record.get_record(response_data['id'])
assert internal_record == response_data['data']
# check that the returned self link returns the same data
subtest_self_link(response_data, res.headers, client)
def test_record_update_mutable(app, db):
"""Test updating mutables in a record."""
recid = uuid.UUID("262d2748-ba41-456f-a844-4d043a419a6f")
# Create a new record with two mutables, a list and a dict
rec = Record.create({"title": "Title", "list": ["foo"], "dict": {"moo": "boo"}}, id_=recid)
# Make sure mutables are there before and after commit
assert rec == {"title": "Title", "list": ["foo"], "dict": {"moo": "boo"}}
db.session.commit()
db.session.expunge_all()
rec = Record.get_record(recid)
assert rec == {"title": "Title", "list": ["foo"], "dict": {"moo": "boo"}}
# Set the mutables under key
rec["list"] = ["bar"]
rec["dict"] = {"eggs": "bacon"}
rec.commit()
# Make sure it commits to DB
assert rec == {"title": "Title", "list": ["bar"], "dict": {"eggs": "bacon"}}
db.session.commit()
db.session.expunge_all()
rec = Record.get_record(recid)
assert rec == {"title": "Title", "list": ["bar"], "dict": {"eggs": "bacon"}}
# Update the mutables under key
rec["list"].append("spam")
rec["dict"]["ham"] = "chicken"
rec.commit()
# Make sure it commits to DB
assert rec == {"title": "Title", "list": ["bar", "spam"], "dict": {"eggs": "bacon", "ham": "chicken"}}
db.session.commit()
db.session.expunge_all()
rec = Record.get_record(recid)
assert rec == {"title": "Title", "list": ["bar", "spam"], "dict": {"eggs": "bacon", "ham": "chicken"}}
def test_validate_partial(app, db):
"""Test partial validation."""
schema = {"properties": {"a": {"type": "string"}, "b": {"type": "string"}}, "required": ["b"]}
data = {"a": "hello", "$schema": schema}
with app.app_context():
# Test validation on create()
# normal validation should fail because 'b' is required
with pytest.raises(ValidationError) as exc_info:
Record.create(data)
assert "'b' is a required property" == exc_info.value.message
# validate with a less restrictive validator
record = Record.create(data, validator=PartialDraft4Validator)
# set wrong data types should fails in any case
data_incorrect = copy.deepcopy(data)
data_incorrect["a"] = 1
with pytest.raises(ValidationError) as exc_info:
Record.create(data_incorrect, validator=PartialDraft4Validator)
assert "1 is not of type 'string'" == exc_info.value.message
# Test validation on commit()
# validation not passing with normal validator
with pytest.raises(ValidationError) as exc_info:
record.commit()
assert "'b' is a required property" == exc_info.value.message
# validation passing with less restrictive validator
assert data == record.commit(validator=PartialDraft4Validator)
# set wrong data types should fails in any case
record["a"] = 1
with pytest.raises(ValidationError) as exc_info:
record.commit(validator=PartialDraft4Validator)
def test_serialize():
"""Test JSON serialize."""
data = json.loads(
JSONLDSerializer(CONTEXT, schema_class=_TestSchema).serialize(
PersistentIdentifier(pid_type='recid', pid_value='2'),
Record({'title': 'mytitle', 'recid': '2'})
)
)
assert data == {
'@id': 'http://localhost/record/2',
'http://purl.org/dc/terms/title': [{'@value': 'mytitle'}]
}
data = json.loads(JSONLDSerializer(
CONTEXT, schema_class=_TestSchema, expanded=False).serialize(
PersistentIdentifier(pid_type='recid', pid_value='2'),
Record({'title': 'mytitle', 'recid': '2'})
)
)
assert data == {
'@context': {
'@base': 'http://localhost/record/',
'dct': 'http://purl.org/dc/terms/',
'recid': '@id',
'title': 'dct:title'
},
'recid': '2',
'title': 'mytitle'
}
def record_upsert(json):
"""Insert or update a record."""
control_number = json.get('control_number', json.get('recid'))
if control_number:
control_number = int(control_number)
pid_type = InspireRecordIdProvider.schema_to_pid_type(json['$schema'])
try:
pid = PersistentIdentifier.get(pid_type, control_number)
record = Record.get_record(pid.object_uuid)
record.update(json)
record.commit()
except PIDDoesNotExistError:
record = Record.create(json, id_=None)
# Create persistent identifier.
inspire_recid_minter(str(record.id), json)
if json.get('deleted'):
new_recid = get_recid_from_ref(json.get('new_record'))
if new_recid:
merged_record = get_db_record(pid_type, new_recid)
merge_pidstores_of_two_merged_records(merged_record.id, record.id)
else:
soft_delete_pidstore_for_record(record.id)
return record
def test_rest_delete_record(app, db, es, es_acl_prepare, test_users):
with app.test_client() as client:
with db.session.begin_nested():
acl = DefaultACL(name='default',
schemas=[RECORD_SCHEMA],
priority=0,
originator=test_users.u1,
operation='update')
actor = UserActor(name='u1',
users=[test_users.u1],
acl=acl,
originator=test_users.u1)
db.session.add(acl)
db.session.add(actor)
pid, record = create_record({'keywords': ['blah']},
clz=SchemaEnforcingRecord)
RecordIndexer().index(record)
current_search_client.indices.refresh()
current_search_client.indices.flush()
login(client, test_users.u2)
response = client.delete(record_url(pid))
assert response.status_code == 403
login(client, test_users.u1)
response = client.delete(record_url(pid))
assert response.status_code == 204
with pytest.raises(NoResultFound):
Record.get_record(pid.object_uuid)
def test_metadata_extraction_video(app, db, cds_depid, bucket, video):
"""Test metadata extraction video mp4."""
recid = PersistentIdentifier.get('depid', cds_depid).object_uuid
# simulate a no fully filled record
record = Record.get_record(recid)
if 'title' in record:
del record['title']
validator = 'cds.modules.records.validators.PartialDraft4Validator'
with pytest.raises(ValidationError):
record.commit()
record.commit(validator=import_string(validator))
# Extract metadata
obj = ObjectVersion.create(bucket=bucket, key='video.mp4')
obj_id = str(obj.version_id)
dep_id = str(cds_depid)
task_s = ExtractMetadataTask().s(uri=video,
version_id=obj_id,
deposit_id=dep_id)
task_s.delay()
# Check that deposit's metadata got updated
record = Record.get_record(recid)
assert 'extracted_metadata' in record['_cds']
assert record['_cds']['extracted_metadata']
# Check that ObjectVersionTags were added
tags = ObjectVersion.query.first().get_tags()
assert tags['duration'] == '60.095000'
assert tags['bit_rate'] == '612177'
assert tags['avg_frame_rate'] == '288000/12019'
assert tags['codec_name'] == 'h264'
assert tags['codec_long_name'] == 'H.264 / AVC / MPEG-4 AVC / ' \
'MPEG-4 part 10'
assert tags['width'] == '640'
assert tags['height'] == '360'
assert tags['nb_frames'] == '1440'
assert tags['display_aspect_ratio'] == '16:9'
assert tags['color_range'] == 'tv'
# Undo
ExtractMetadataTask().clean(deposit_id=dep_id, version_id=obj_id)
# Check that deposit's metadata got reverted
record = Record.get_record(recid)
assert 'extracted_metadata' not in record['_cds']
# Check that ObjectVersionTags are still there (attached to the old obj)
tags = ObjectVersion.query.first().get_tags()
assert len(tags) == 11
# Simulate failed task, no extracted_metadata
ExtractMetadataTask().clean(deposit_id=dep_id, version_id=obj_id)
record = Record.get_record(recid)
assert 'extracted_metadata' not in record['_cds']
# check again tags
tags = ObjectVersion.query.first().get_tags()
assert len(tags) == 11
def acl_changed_reindex(acl_id):
"""
ACL has been changed so reindex all the documents in the given index.
:param acl_id: id of ACL instance
"""
logger.info('Reindexing started for ACL=%s', acl_id)
timestamp = datetime.datetime.now().astimezone().isoformat()
acl = ACL.query.filter_by(id=acl_id).one_or_none()
if not acl:
# deleted in the meanwhile, so just return
return # pragma no cover
# make sure all indices are flushed so that no resource is obsolete in index
for schema in acl.schemas:
current_search_client.indices.flush(index=schema_to_index(schema)[0])
indexer = RecordIndexer()
updated_count = 0
removed_count = 0
for id in acl.get_matching_resources():
try:
rec = Record.get_record(id)
except: # pragma no cover
# record removed in the meanwhile by another thread/process,
# indexer should have been called to remove it from ES
# won't test this so pragma no cover
continue
try:
indexer.index(rec)
updated_count += 1
except Exception as e: # pragma no cover
logger.exception('Error indexing ACL for resource %s: %s', id, e)
# reindex the resources those were indexed by this acl but no longer should be
for id in acl.used_in_records(older_than_timestamp=timestamp):
try:
rec = Record.get_record(id)
except NoResultFound: # pragma no cover
continue
except: # pragma no cover
logger.exception('Unexpected exception in record reindexing')
continue
try:
removed_count += 1
indexer.index(rec)
except: # pragma no cover
logger.exception('Error indexing ACL for obsolete resource %s', id)
logger.info(
'Reindexing finished for ACL=%s, acl applied to %s records, acl removed from %s records',
acl_id, updated_count, removed_count)
def records(db):
"""Create sample records with references."""
return [
Record.create({'$ref': 'ref1'}),
Record.create({'$ref': 'ref2'}),
Record.create({'reflist': [
{'$ref': 'ref3'},
{'$ref': 'ref4'}
]})
]
def test_file_permissions(app, db, test_object, # fixtures
user, access_right, expected):
"""Test file permissions."""
# Create test users
admin = User(email='*****@*****.**', password='******')
owner = User(email='*****@*****.**', password='******')
auth = User(email='*****@*****.**', password='******')
db.session.add_all([admin, owner, auth])
db.session.add(
ActionUsers.allow(ActionNeed('admin-access'), user=admin)
)
# Create test record
rec_uuid = uuid.uuid4()
PersistentIdentifier.create(
'recid',
'1',
object_type='rec',
object_uuid=rec_uuid,
status=PIDStatus.REGISTERED
)
Record.create({
'recid': 1,
'owners': [2],
'access_right': access_right,
'_files': [
{
'key': test_object.key,
'bucket': str(test_object.bucket_id),
'checksum': 'invalid'
},
]
}, id_=rec_uuid)
db.session.add(
RecordsBuckets(record_id=rec_uuid, bucket_id=test_object.bucket_id)
)
file_url = url_for(
'invenio_records_ui.recid_files',
pid_value='1',
filename=test_object.key
)
db.session.commit()
with app.test_client() as client:
if user:
# Login as user
with client.session_transaction() as sess:
sess['user_id'] = User.query.filter_by(
email='{}@zenodo.org'.format(user)).one().id
sess['_fresh'] = True
res = client.get(file_url)
assert res.status_code == expected
def test_undelete_with_get(testapp, db):
"""Test undelete a record."""
record = Record.create({'title': 'test'})
db.session.commit()
record.delete()
db.session.commit()
record = Record.get_record(record.id, with_deleted=True)
record.undelete()
record.commit()
db.session.commit()
assert record == {}
def test_record_deletion(test_record):
recid = test_record.get('control_number')
assert recid
test_record.delete()
db.session.commit()
pid = PersistentIdentifier.get('recid', recid)
assert pid.status == PIDStatus.DELETED
with raises(NoResultFound):
Record.get_record(pid.object_uuid)
def files():
"""Load files."""
data_path = os.path.join(os.path.dirname(__file__), 'data')
# Create location
loc = Location(name='local', uri=data_path, default=True)
db.session.commit()
# Bucket
bucket = Bucket.create(loc)
# Example files from the data folder
example_files = (
'markdown.md',
'csvfile.csv',
'zipfile.zip',
'jsonfile.json',
'xmlfile.xml',
'notebook.ipynb',
'jpgfile.jpg',
'pngfile.png',
)
# Create single file records
for f in example_files:
with open(os.path.join(data_path, f), 'rb') as fp:
create_object(bucket, f, fp)
# Create a multi-file record
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
data = {
'pid_value': provider.pid.pid_value,
'files': []
}
# Template to create different files
template_file = {
'uri': '/files/{0}/{1}',
'key': '',
'bucket': str(bucket.id),
'local': True
}
for filename in example_files:
file_data = template_file.copy()
file_data['uri'] = file_data['uri'].format(str(bucket.id), filename)
file_data['key'] = filename
data['files'].append(file_data)
Record.create(data, id_=rec_uuid)
db.session.commit()
def test_change_record_community(app, test_records):
"""Test updating the community field fails."""
with app.app_context():
record = Record.get_record(test_records[0].record_id)
del record['community']
with pytest.raises(AlteredRecordError):
record.commit()
with app.app_context():
record = Record.get_record(test_records[0].record_id)
record['community'] = str(uuid.uuid4())
with pytest.raises(AlteredRecordError):
record.commit()
def files():
"""Load files."""
data_path = os.path.join(os.path.dirname(__file__), 'data')
# Create location
loc = Location(name='local', uri=data_path, default=True)
db.session.commit()
# Bucket
bucket = Bucket.create(loc)
# Example files from the data folder
example_files = (
'markdown.md',
'csvfile.csv',
'zipfile.zip',
'jsonfile.json',
'xmlfile.xml',
'notebook.ipynb',
'jpgfile.jpg',
'pngfile.png',
)
# Create single file records
for f in example_files:
with open(os.path.join(data_path, f), 'rb') as fp:
create_object(bucket, f, fp)
# Create a multi-file record
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
data = {'pid_value': provider.pid.pid_value, 'files': []}
# Template to create different files
template_file = {
'uri': '/files/{0}/{1}',
'key': '',
'bucket': str(bucket.id),
'local': True
}
for filename in example_files:
file_data = template_file.copy()
file_data['uri'] = file_data['uri'].format(str(bucket.id), filename)
file_data['key'] = filename
data['files'].append(file_data)
Record.create(data, id_=rec_uuid)
db.session.commit()
def test_db():
"""Test database backend."""
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get(
'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'
)
FlaskCLI(app)
InvenioDB(app)
InvenioRecords(app)
with app.app_context():
create_database(db.engine.url)
db.create_all()
assert len(db.metadata.tables) == 3
data = {'title': 'Test'}
from invenio_records.models import RecordMetadata as RM
# Create a record
with app.app_context():
assert RM.query.count() == 0
record_uuid = Record.create(data).id
db.session.commit()
assert RM.query.count() == 1
db.session.commit()
# Retrieve created record
with app.app_context():
record = Record.get_record(record_uuid)
assert record.dumps() == data
with pytest.raises(NoResultFound):
Record.get_record(uuid.uuid4())
record['field'] = True
record = record.patch([
{'op': 'add', 'path': '/hello', 'value': ['world']}
])
assert record['hello'] == ['world']
record.commit()
db.session.commit()
with app.app_context():
record2 = Record.get_record(record_uuid)
assert record2.model.version_id == 2
assert record2['field']
assert record2['hello'] == ['world']
db.session.commit()
# Cannot commit record without model (i.e. Record.create_record)
with app.app_context():
record3 = Record({'title': 'Not possible'})
with pytest.raises(RecordNotCommitableError):
record3.commit()
with app.app_context():
db.drop_all()
drop_database(db.engine.url)
def migrate_chunk(chunk, broken_output=None, dry_run=False):
from invenio_indexer.api import RecordIndexer
from ..pidstore.minters import inspire_recid_minter
indexer = RecordIndexer()
index_queue = []
for raw_record in chunk:
record = marc_create_record(raw_record, keep_singletons=False)
json_record = create_record(record)
if '$schema' in json_record:
json_record['$schema'] = url_for(
'invenio_jsonschemas.get_schema',
schema_path="records/{0}".format(json_record['$schema'])
)
rec_uuid = str(Record.create(json_record, id_=None).id)
# Create persistent identifier.
pid = inspire_recid_minter(rec_uuid, json_record)
index_queue.append(pid.object_uuid)
db.session.commit()
# Request record indexing
for i in index_queue:
indexer.index_by_id(i)
# Send task to migrate files.
return rec_uuid
def data_policies():
"""Load demo Data Policy records."""
from invenio_db import db
from invenio_records import Record
from invenio_indexer.api import RecordIndexer
from invenio_pidstore.errors import PIDDoesNotExistError, \
PersistentIdentifierError
from invenio_pidstore.models import PIDStatus, PersistentIdentifier
from invenio_pidstore.fetchers import recid_fetcher
from invenio_pidstore.minters import recid_minter
from cernopendata.modules.records.minters.recid import \
cernopendata_recid_minter
indexer = RecordIndexer()
schema = current_app.extensions['invenio-jsonschemas'].path_to_url(
'records/data-policies-v1.0.0.json'
)
data = pkg_resources.resource_filename('cernopendata',
'modules/fixtures/data')
data_policies_json = glob.glob(os.path.join(data, '*.json'))
for filename in data_policies_json:
with open(filename, 'rb') as source:
for data in json.load(source):
id = uuid.uuid4()
cernopendata_recid_minter(id, data)
record = Record.create(data, id_=id)
record['$schema'] = schema
db.session.commit()
indexer.index(record)
db.session.expunge_all()
def news():
"""Load demo news records."""
from invenio_db import db
from invenio_records import Record
from invenio_indexer.api import RecordIndexer
from cernopendata.modules.records.minters.artid import \
cernopendata_articleid_minter
indexer = RecordIndexer()
schema = current_app.extensions['invenio-jsonschemas'].path_to_url(
'records/article-v1.0.0.json'
)
data = pkg_resources.resource_filename('cernopendata',
'modules/fixtures/data')
articles_json = glob.glob(os.path.join(data, 'articles', 'news', '*.json'))
for filename in articles_json:
with open(filename, 'rb') as source:
for data in json.load(source):
if "collections" not in data and \
not isinstance(data.get("collections", None), basestring):
data["collections"] = []
data["collections"].append({"primary": "News"})
id = uuid.uuid4()
cernopendata_articleid_minter(id, data)
record = Record.create(data, id_=id)
record['$schema'] = schema
db.session.commit()
indexer.index(record)
db.session.expunge_all()
def create_record(ctx):
"""
Creates the record in the database.
:param ctx: The record metadata as a dictionary.
:return: the recid and the uuid
"""
record_information = create_data_structure(ctx)
record_id = uuid.uuid4()
pid = recid_minter(record_id, record_information)
record_information['recid'] = int(pid.pid_value)
record_information['uuid'] = str(record_id)
Record.create(record_information, id_=record_id)
db.session.commit()
return record_information
def test_change_record_schema_fails(app, test_records):
"""Test updating the $schema field fails."""
with app.app_context():
record = Record.get_record(test_records[0].record_id)
del record['$schema']
with pytest.raises(AlteredRecordError):
record.commit()
def test_record_patch_immutable_fields(app, test_records, test_users,
login_user):
"""Test invalid modification of record draft with HTTP PATCH."""
with app.app_context():
record = Record.get_record(test_records[0].record_id)
with app.test_client() as client:
user = test_users['admin']
login_user(user, client)
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
for path in IMMUTABLE_PATHS:
for command in [
{"op": "replace", "path": path, "value": ""},
{"op": "remove", "path": path},
{"op": "add", "path": path, "value": ""},
{"op": "copy", "from": "/title", "path": path, "value": ""},
{"op": "move", "from": "/title", "path": path, "value": ""},
]:
draft_patch_res = client.patch(
url_for('b2share_records_rest.b2rec_item',
pid_value=test_records[0].pid),
data=json.dumps([command]),
headers=headers)
assert draft_patch_res.status_code == 400
def revert_delete_record(pid):
"""Reverts deletion action of a record."""
pid_obj = PersistentIdentifier.query.filter_by(pid_value=pid).one()
record_uuid = pid_obj.object_uuid
deleted = Record.get_record(record_uuid, with_deleted=True)
if not deleted.is_deleted:
click.secho(f"Record {pid} was not deleted. Aborting.", fg="red")
return
# revert to previous revision
record = deleted.revert(deleted.revision_id - 1)
all_pid_objects = PersistentIdentifier.query.filter_by(
object_uuid=record_uuid)
# trying to get all the pid types (including legacy pids)
for pid_object in all_pid_objects:
pid_object.status = PIDStatus.REGISTERED
db.session.commit()
indexer_class = current_app_ils.indexer_by_pid_type(pid_obj.pid_type)
indexer_class.index(record)
click.secho(f"Record {pid} is reverted from deletion.", fg="green")
def store_record(obj, *args, **kwargs):
"""Create and index new record in main record space."""
assert "$schema" in obj.data, "No $schema attribute found!"
# Create record
# FIXME: Do some preprocessing of obj.data before creating a record so that
# we're sure that the schema will be validated without touching the full
# holdingpen stack.
record = Record.create(obj.data, id_=None)
# Create persistent identifier.
pid = inspire_recid_minter(str(record.id), record)
# Commit any changes to record
record.commit()
# Dump any changes to record
obj.data = record.dumps()
# Commit to DB before indexing
db.session.commit()
# Index record
indexer = RecordIndexer()
indexer.index_by_id(pid.object_uuid)
def test_record_replace_refs(app):
"""Test the replacement of JSON references using JSONResolver."""
with app.app_context():
record = Record.create({
'one': {'$ref': 'http://nest.ed/A'},
'three': {'$ref': 'http://nest.ed/ABC'}
})
app.extensions['invenio-records'].loader_cls = json_loader_factory(
JSONResolver(plugins=['demo.json_resolver']))
out_json = record.replace_refs()
expected_json = {
'one': {
'letter': 'A',
'next': '.',
},
'three': {
'letter': 'A',
'next': {
'letter': 'B',
'next': {
'letter': 'C',
'next': '.',
},
},
},
}
assert out_json == expected_json
def migrate_chunk(chunk, broken_output=None, dry_run=False):
from invenio_indexer.api import RecordIndexer
from ..pidstore.minters import inspire_recid_minter
indexer = RecordIndexer()
index_queue = []
for raw_record in chunk:
record = marc_create_record(raw_record, keep_singletons=False)
json_record = create_record(record)
if '$schema' in json_record:
json_record['$schema'] = url_for(
'invenio_jsonschemas.get_schema',
schema_path="records/{0}".format(json_record['$schema'])
)
rec_uuid = str(Record.create(json_record, id_=None).id)
# Create persistent identifier.
pid = inspire_recid_minter(rec_uuid, json_record)
index_queue.append(pid.object_uuid)
db.session.commit()
# Request record indexing
for i in index_queue:
indexer.index_by_id(i)
# Send task to migrate files.
return rec_uuid
def test_default_deduplication_validator(app, simple_record, mocker):
"""Make sure default deduplication validator works."""
from invenio_records import Record
mocker.patch('invenio_matcher.api.execute', duplicated_result)
with app.app_context():
record = Record(simple_record)
queries = [{'type': 'exact', 'match': 'title'}]
expected = [MatchResult(1, record, 1)]
result = list(
match(
record,
index="records",
doc_type="record",
queries=queries,
validator=None,
))
assert expected == result
# Test the same again to see if validator resets
expected = [MatchResult(1, record, 1)]
result = list(
match(
record,
index="records",
doc_type="record",
queries=queries,
validator=None,
))
assert expected == result
def test_preprocessor_mixin_record(app):
"""Test preprocessor mixin."""
with db.session.begin_nested():
recuuid = uuid.uuid4()
record = Record.create({
'title': 'test', 'aref': {'$ref': '#/title'}}, id_=recuuid)
record.model.created = datetime(2015, 10, 1, 11, 11, 11, 1)
pid = PersistentIdentifier.create(
'recid', '1', object_type='rec', object_uuid=recuuid,
status=PIDStatus.REGISTERED)
db.session.commit()
data = PreprocessorMixin().preprocess_record(pid, record)
for k in keys:
assert k in data
assert data['metadata']['title'] == 'test'
assert data['metadata']['aref'] == {'$ref': '#/title'}
assert data['created'] == '2015-10-01T11:11:11.000001+00:00'
assert data['revision'] == 1
data = PreprocessorMixin(replace_refs=True).preprocess_record(
pid, Record({'title': 'test2', 'aref': {'$ref': '#/title'}}))
assert data['created'] is None
assert data['updated'] is None
assert data['metadata']['aref'] == 'test2'
def test_permission_factory(app, db, action, permission_factory):
"""Test revisions."""
InvenioAccess(app)
rec_uuid = uuid.uuid4()
with db.session.begin_nested():
user_all = User(email='*****@*****.**')
user_one = User(email='*****@*****.**')
user_none = User(email='*****@*****.**')
db.session.add(user_all)
db.session.add(user_one)
db.session.add(user_none)
db.session.add(ActionUsers(action=action, user=user_all,
argument=None))
db.session.add(
ActionUsers(action=action, user=user_one, argument=str(rec_uuid)))
record = Record.create({'title': 'permission test'}, id_=rec_uuid)
# Create a record and assign permissions.
permission = permission_factory(record)
# Assert which permissions has access.
assert permission.allows(FakeIdentity(UserNeed(user_all.id)))
assert permission.allows(FakeIdentity(UserNeed(user_one.id)))
assert not permission.allows(FakeIdentity(UserNeed(user_none.id)))
def test_record_crud_2(self, load_entry_points, app, db, record_xml):
synchronizer = current_oai_client.providers["uk"].synchronizers["xoai"]
oai_sync = OAISync(provider_code="uk")
synchronizer.oai_sync = oai_sync
oai_identifier = "oai:dspace.cuni.cz:20.500.11956/2623"
# vytvoření záznamu
synchronizer.record_crud(oai_identifier=oai_identifier, xml=record_xml)
db.session.commit()
oai_rec = OAIRecord.get_record(oai_identifier)
assert oai_rec is not None
# smazání záznamu
synchronizer.record_crud(oai_rec=oai_rec, deleted=True)
db.session.commit()
oai_rec2 = OAIRecord.get_record(oai_identifier)
assert oai_rec == oai_rec2
# obnovení záznamu
synchronizer.record_crud(oai_rec=oai_rec,
timestamp='2050-10-22T08:18:08.567698+00:00',
xml=record_xml)
db.session.commit()
oai_rec3 = OAIRecord.get_record(oai_identifier)
assert oai_rec3 is not None
record = Record.get_record(oai_rec.id)
assert record is not None
def test_reindex(app, script_info):
"""Test reindex."""
# load records
with app.test_request_context():
runner = CliRunner()
rec_uuid = uuid.uuid4()
data = {'title': 'Test0'}
record = Record.create(data, id_=rec_uuid)
db.session.commit()
# Initialize queue
res = runner.invoke(cli.queue, ['init', 'purge'],
obj=script_info)
assert 0 == res.exit_code
res = runner.invoke(cli.reindex, ['--yes-i-know'], obj=script_info)
assert 0 == res.exit_code
res = runner.invoke(cli.run, [], obj=script_info)
assert 0 == res.exit_code
sleep(5)
indexer = RecordIndexer()
index, doc_type = indexer.record_to_index(record)
res = current_search_client.get(index=index, doc_type=doc_type,
id=rec_uuid)
assert res['found']
# Destroy queue
res = runner.invoke(cli.queue, ['delete'],
obj=script_info)
assert 0 == res.exit_code
def test_preprocessor_mixin_record(app, db):
"""Test preprocessor mixin."""
pid, record = create_record({'title': 'test', 'aref': {'$ref': '#/title'}})
record.model.created = datetime(2015, 10, 1, 11, 11, 11, 1)
db.session.commit()
data = PreprocessorMixin().preprocess_record(pid, record)
for k in keys:
assert k in data
assert data['metadata']['title'] == 'test'
assert data['metadata']['aref'] == {'$ref': '#/title'}
assert data['created'] == '2015-10-01T11:11:11.000001+00:00'
assert data['revision'] == 1
data = PreprocessorMixin(replace_refs=True).preprocess_record(
pid, Record({
'title': 'test2',
'aref': {
'$ref': '#/title'
}
}))
assert data['created'] is None
assert data['updated'] is None
assert data['metadata']['aref'] == 'test2'
def test_record_dump(app, db):
"""Test record dump method."""
with app.app_context():
record = Record.create({'foo': {'bar': 'Bazz', }, })
record_dump = record.dumps()
record_dump['foo']['bar'] = 'Spam'
assert record_dump['foo']['bar'] != record['foo']['bar']
def records(pids, db):
"""Fixture for the records."""
pid_versions = ['h1v1', 'h1v2', 'h2v1']
schema = {
'type': 'object',
'properties': {
'title': {
'type': 'string'
},
},
}
data = {
name: {
'title': 'Test version {}'.format(name),
'recid': pids[name].pid_value,
'$schema': schema
}
for name in pid_versions
}
records = dict()
for name in pid_versions:
record = Record.create(data[name])
pids[name].assign('rec', record.id)
records[name] = record
return records
def load_records(app, filename, schema, tries=5):
"""Try to index records."""
indexer = RecordIndexer()
records = []
with app.app_context():
with mock.patch('invenio_records.api.Record.validate',
return_value=None):
data_filename = pkg_resources.resource_filename(
'invenio_records', filename)
records_data = load(data_filename)
with db.session.begin_nested():
for item in records_data:
record_id = uuid.uuid4()
item_dict = dict(marc21.do(item))
item_dict['$schema'] = schema
recid_minter(record_id, item_dict)
oaiid_minter(record_id, item_dict)
record = Record.create(item_dict, id_=record_id)
indexer.index(record)
records.append(record.id)
db.session.commit()
# Wait for indexer to finish
for i in range(tries):
response = current_search_client.search()
if response['hits']['total'] >= len(records):
break
current_search.flush_and_refresh('_all')
return records
def store_record(obj, *args, **kwargs):
"""Create and index new record in main record space."""
assert "$schema" in obj.data, "No $schema attribute found!"
# Create record
# FIXME: Do some preprocessing of obj.data before creating a record so that
# we're sure that the schema will be validated without touching the full
# holdingpen stack.
record = Record.create(obj.data, id_=None)
# Create persistent identifier.
pid = inspire_recid_minter(str(record.id), record)
# Commit any changes to record
record.commit()
# Dump any changes to record
obj.data = record.dumps()
# Commit to DB before indexing
db.session.commit()
# Index record
indexer = RecordIndexer()
indexer.index_by_id(pid.object_uuid)
def create_record(ctx):
"""
Creates the record in the database.
:param ctx: The record metadata as a dictionary.
:return: the recid and the uuid
"""
record_information = create_data_structure(ctx)
record_id = uuid.uuid4()
pid = recid_minter(record_id, record_information)
record_information['recid'] = int(pid.pid_value)
record_information['uuid'] = str(record_id)
Record.create(record_information, id_=record_id)
db.session.commit()
return record_information
def test_record_dump(app, db):
"""Test record dump method."""
with app.app_context():
record = Record.create({"foo": {"bar": "Bazz"}})
record_dump = record.dumps()
record_dump["foo"]["bar"] = "Spam"
assert record_dump["foo"]["bar"] != record["foo"]["bar"]
def delete_model(self, model):
"""Delete a record."""
try:
if model.json is None:
return True
record = Record(model.json, model=model)
record.delete()
db.session.commit()
except SQLAlchemyError as e:
if not self.handle_view_exception(e):
flash('Failed to delete record. %s' % str(e), category='error')
db.session.rollback()
return False
return True
def test_record_replace_refs(app, db):
"""Test the replacement of JSON references using JSONResolver."""
record = Record.create({
'one': {
'$ref': 'http://nest.ed/A'
},
'three': {
'$ref': 'http://nest.ed/ABC'
}
})
app.extensions['invenio-records'].loader_cls = json_loader_factory(
JSONResolver(plugins=['demo.json_resolver']))
out_json = record.replace_refs()
expected_json = {
'one': {
'letter': 'A',
'next': '.',
},
'three': {
'letter': 'A',
'next': {
'letter': 'B',
'next': {
'letter': 'C',
'next': '.',
},
},
},
}
assert out_json == expected_json
def test_record_patch_immutable_fields(app, test_records, test_users,
login_user):
"""Test invalid modification of record draft with HTTP PATCH."""
with app.app_context():
record = Record.get_record(test_records[0].record_id)
with app.test_client() as client:
user = test_users['admin']
login_user(user, client)
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
for path in IMMUTABLE_PATHS:
for command in [
{"op": "replace", "path": path, "value": ""},
{"op": "remove", "path": path},
{"op": "add", "path": path, "value": ""},
{"op": "copy", "from": "/title", "path": path, "value": ""},
{"op": "move", "from": "/title", "path": path, "value": ""},
]:
draft_patch_res = client.patch(
url_for('b2share_records_rest.b2rec_item',
pid_value=test_records[0].pid),
data=json.dumps([command]),
headers=headers)
assert draft_patch_res.status_code == 400
data = json.loads(draft_patch_res.get_data(as_text=True))
assert data['errors'][0]['message'] == \
'The field "{}" is immutable.'.format(path)
def test_permission_factory(app, action, permission_factory):
"""Test revisions."""
InvenioAccess(app)
with app.app_context():
rec_uuid = uuid.uuid4()
with db.session.begin_nested():
user_all = User(email='*****@*****.**')
user_one = User(email='*****@*****.**')
user_none = User(email='*****@*****.**')
db.session.add(user_all)
db.session.add(user_one)
db.session.add(user_none)
db.session.add(ActionUsers(action=action,
user=user_all, argument=None))
db.session.add(ActionUsers(action=action,
user=user_one, argument=str(rec_uuid)))
record = Record.create({'title': 'permission test'}, id_=rec_uuid)
# Create a record and assign permissions.
permission = permission_factory(record)
# Assert which permissions has access.
assert permission.allows(FakeIdentity(UserNeed(user_all.id)))
assert permission.allows(FakeIdentity(UserNeed(user_one.id)))
assert not permission.allows(FakeIdentity(UserNeed(user_none.id)))
def files_permission_factory(obj=None, action=None):
"""Permission for files are always based on the type of bucket.
1. Record bucket: Read access only with open and restricted access.
2. Any other bucket is restricted to admins only.
"""
# Extract bucket id
bucket_id = None
if isinstance(obj, Bucket):
bucket_id = str(obj.id)
elif isinstance(obj, ObjectVersion):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, MultipartObject):
bucket_id = str(obj.bucket_id)
elif isinstance(obj, FileObject):
bucket_id = str(obj.bucket_id)
# Retrieve record
if bucket_id is not None:
record_bucket = RecordsBuckets.query.filter_by(bucket_id=bucket_id).one_or_none()
if record_bucket is not None:
record = Record.get_record(record_bucket.record_id)
return FilePermission.create(record, action)
return AdminPermission.create(obj, action)
def load_records(app, filename, schema, tries=5):
"""Try to index records."""
indexer = RecordIndexer()
records = []
with app.app_context():
with mock.patch('invenio_records.api.Record.validate',
return_value=None):
data_filename = pkg_resources.resource_filename(
'invenio_records', filename)
records_data = load(data_filename)
with db.session.begin_nested():
for item in records_data:
record_id = uuid.uuid4()
item_dict = dict(marc21.do(item))
item_dict['$schema'] = schema
recid_minter(record_id, item_dict)
oaiid_minter(record_id, item_dict)
record = Record.create(item_dict, id_=record_id)
indexer.index(record)
records.append(record.id)
db.session.commit()
# Wait for indexer to finish
for i in range(tries):
response = current_search_client.search()
if response['hits']['total'] >= len(records):
break
sleep(5)
return records
def _update_event_bucket(event):
"""Update event's payload with correct bucket of deposit."""
depid = event.payload['deposit_id']
dep_uuid = str(PersistentIdentifier.get('depid', depid).object_uuid)
deposit_bucket = Record.get_record(dep_uuid)['_buckets']['deposit']
event.payload['bucket_id'] = deposit_bucket
flag_modified(event, 'payload')
def test_metadata_extraction_video_mp4(app, db, depid, bucket, video_mp4):
"""Test metadata extraction video mp4."""
# Extract metadata
obj = ObjectVersion.create(bucket=bucket, key='video.mp4')
video_metadata_extraction.s(uri=video_mp4,
object_version=str(obj.version_id),
deposit_id=str(depid)).delay()
# Check that deposit's metadata got updated
recid = PersistentIdentifier.get('depid', depid).object_uuid
record = Record.get_record(recid)
assert 'extracted_metadata' in record['_deposit']
assert record['_deposit']['extracted_metadata']
# Check that ObjectVersionTags were added
obj = ObjectVersion.query.first()
tags = obj.get_tags()
assert tags['duration'] == '60.095000'
assert tags['bit_rate'] == '612177'
assert tags['size'] == '5510872'
assert tags['avg_frame_rate'] == '288000/12019'
assert tags['codec_name'] == 'h264'
assert tags['width'] == '640'
assert tags['height'] == '360'
assert tags['nb_frames'] == '1440'
assert tags['display_aspect_ratio'] == '0:1'
assert tags['color_range'] == 'tv'
def record_upsert(json):
"""Insert or update a record."""
control_number = json.get('control_number', json.get('recid'))
if control_number:
control_number = int(control_number)
pid_type = InspireRecordIdProvider.schema_to_pid_type(json['$schema'])
try:
pid = PersistentIdentifier.get(pid_type, control_number)
record = Record.get_record(pid.object_uuid)
record.update(json)
record.commit()
except PIDDoesNotExistError:
record = Record.create(json, id_=None)
# Create persistent identifier.
inspire_recid_minter(str(record.id), json)
return record
def _get_record_from_workflow(workflow):
assert len(workflow.objects) == 1
workflow_object = workflow.objects[0]
recid = workflow_object.data['control_number']
pid = PersistentIdentifier.get('recid', recid)
return Record.get_record(pid.object_uuid)
def create_record(data):
"""Create a test record."""
with db.session.begin_nested():
data = copy.deepcopy(data)
rec_uuid = uuid.uuid4()
pid = current_pidstore.minters['recid'](rec_uuid, data)
record = Record.create(data, id_=rec_uuid)
return pid, record
def create_object(bucket, file_name, stream):
"""Object creation inside the bucket using the file and its content."""
obj = ObjectVersion.create(bucket, file_name, stream=stream)
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
data = {
'pid_value': provider.pid.pid_value,
'files': [
{
'uri': '/files/{0}/{1}'.format(str(bucket.id), file_name),
'key': file_name,
'size': obj.file.size,
'bucket': str(bucket.id),
'local': True
}
]
}
Record.create(data, id_=rec_uuid)
def test_invalid_patch(app):
"""Test INVALID record patch request (PATCH .../records/<record_id>)."""
with app.app_context():
InvenioRecordsREST(app)
with app.test_client() as client:
# check that PATCH with non existing id will return 404
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
res = client.patch(url_for(blueprint.name + '.' +
RecordResource.view_name, record_id=0),
data=json.dumps(test_patch),
headers=headers)
assert res.status_code == 404
# create the record using the internal API
internal_record = Record.create(test_data)
# check that PATCH with non accepted format will return 406
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'video/mp4')]
res = client.patch(url_for(blueprint.name + '.' +
RecordResource.view_name,
record_id=internal_record.model.id),
data=json.dumps(test_patch),
headers=headers)
assert res.status_code == 406
# check that PATCH with non-json Content-Type will return 400
headers = [('Content-Type', 'video/mp4'),
('Accept', 'application/json')]
res = client.patch(url_for(blueprint.name + '.' +
RecordResource.view_name,
record_id=internal_record.model.id),
data=json.dumps(test_patch),
headers=headers)
assert res.status_code == 415
# check that PATCH with invalid json-patch will return 400
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
res = client.patch(url_for(blueprint.name + '.' +
RecordResource.view_name,
record_id=internal_record.model.id),
data=json.dumps([
{'invalid': 'json-patch'}
]),
headers=headers)
assert res.status_code == 400
# check that PATCH with invalid json will return 400
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
res = client.patch(url_for(blueprint.name + '.' +
RecordResource.view_name,
record_id=internal_record.model.id),
data='{sdfsdf',
headers=headers)
assert res.status_code == 400
def update_authors_recid(record_id, uuid, profile_recid):
"""Update author profile for a given signature.
The method receives UUIDs representing record and signature
respectively together with an author profile recid.
The new recid will be placed in the signature with the given
UUID.
:param record_id:
A string representing UUID of a given record.
Example:
record_id = "a5afb151-8f75-4e91-8dc1-05e7e8e8c0b8"
:param uuid:
A string representing UUID of a given signature.
Example:
uuid = "c2f432bd-2f52-4c16-ac66-096f168c762f"
:param profile_recid:
A string representing author profile recid, that
updated signature should point to.
Example:
profile_recid = "1"
"""
try:
record = Record.get_record(record_id)
update_flag = False
for author in record['authors']:
if author['uuid'] == uuid:
author['recid'] = str(profile_recid)
update_flag = True
if update_flag:
# Disconnect the signal on insert of a new record.
before_record_index.disconnect(append_updated_record_to_queue)
# Update the record in the database.
record.commit()
db.session.commit()
# Update the record in Elasticsearch.
indexer = RecordIndexer()
indexer.index_by_id(record.id)
except StaleDataError as exc:
raise update_authors_recid.retry(exc=exc)
finally:
# Reconnect the disconnected signal.
before_record_index.connect(append_updated_record_to_queue)
# Report.
logger.info("Updated signature %s with profile %s",
uuid, profile_recid)
def get_primary_arxiv_category(self):
try:
pid = PersistentIdentifier.get('recid', self.control_number)
record = Record.get_record(pid.object_uuid)
return get_arxiv_primary_category(record)
except PIDDoesNotExistError:
# records imported from Inspire won't be found
pass
return None
def test_record_replace_refs(app, db):
"""Test the replacement of JSON references using JSONResolver."""
record = Record.create({"one": {"$ref": "http://nest.ed/A"}, "three": {"$ref": "http://nest.ed/ABC"}})
app.extensions["invenio-records"].loader_cls = json_loader_factory(JSONResolver(plugins=["demo.json_resolver"]))
out_json = record.replace_refs()
expected_json = {
"one": {"letter": "A", "next": "."},
"three": {"letter": "A", "next": {"letter": "B", "next": {"letter": "C", "next": "."}}},
}
assert out_json == expected_json