def test_valid_delete(app): """Test VALID record delete request (DELETE .../records/<record_id>).""" with app.app_context(): # create the record using the internal API pid, record = create_record(test_data) with app.test_client() as client: headers = [('Accept', 'application/json')] res = client.delete(url_for('invenio_records_rest.recid_item', pid_value=pid.pid_value), headers=headers) assert res.status_code == 204 # check database state with pytest.raises(NoResultFound): Record.get_record(record.id) assert pid.is_deleted() # check that DELETE with non JSON accepted format will work # as it returns nothing pid, record = create_record(test_data) headers = [('Accept', 'video/mp4')] res = client.delete(url_for('invenio_records_rest.recid_item', pid_value=pid.pid_value), headers=headers) assert res.status_code == 204
def test_valid_put(app): """Test VALID record patch request (PATCH .../records/<record_id>).""" with app.app_context(): InvenioRecordsREST(app) # create the record using the internal API internal_record = Record.create(test_data) with app.test_client() as client: headers = [('Content-Type', 'application/json'), ('Accept', 'application/json')] res = client.put(url_for(blueprint.name + '.' + RecordResource.view_name, record_id=internal_record.model.id), data=json.dumps(test_data_patched), headers=headers) assert res.status_code == 200 # check that the returned record matches the given data response_data = json.loads(res.get_data(as_text=True)) assert response_data['data'] == test_data_patched # check that an internal record returned id and that it contains # the same data assert 'id' in response_data.keys() internal_record = Record.get_record(response_data['id']) assert internal_record == response_data['data'] # check that the returned self link returns the same data subtest_self_link(response_data, res.headers, client)
def test_validate_with_format(app, db): """Test that validation can accept custom format rules.""" with app.app_context(): checker = FormatChecker() checker.checks("foo")(lambda el: el.startswith("foo")) data = {"bar": "foo", "$schema": {"properties": {"bar": {"format": "foo"}}}} # test record creation with valid data assert data == Record.create(data) record = Record.create(data, format_checker=checker) # test direct call to validate with valid data assert record.validate(format_checker=checker) is None # test commit with valid data record.commit(format_checker=checker) record["bar"] = "bar" # test direct call to validate with invalid data with pytest.raises(ValidationError) as excinfo: record.validate(format_checker=checker) assert "'bar' is not a 'foo'" in str(excinfo.value) # test commit with invalid data with pytest.raises(ValidationError) as excinfo: record.commit(format_checker=checker) assert "'bar' is not a 'foo'" in str(excinfo.value) data["bar"] = "bar" # test record creation with invalid data with pytest.raises(ValidationError) as excinfo: record = Record.create(data, format_checker=checker) assert "'bar' is not a 'foo'" in str(excinfo.value)
def test_valid_delete(app): """Test VALID record delete request (DELETE .../records/<record_id>).""" with app.app_context(): # create the record using the internal API pid, record = create_record(test_data) with app.test_client() as client: headers = [('Accept', 'application/json')] res = client.delete(url_for('invenio_records_rest.recid_item', pid_value=pid.pid_value), headers=headers) assert res.status_code == 204 # check database state with pytest.raises(NoResultFound): Record.get_record(record.id) assert pid.is_deleted() # check that DELETE with non JSON accepted format will work # as it returns nothing pid, record = create_record(test_data) headers = [('Accept', 'video/mp4')] res = client.delete(url_for('invenio_records_rest.recid_item', pid_value=pid.pid_value), headers=headers) assert res.status_code == 204
def test_valid_put(app): """Test VALID record patch request (PATCH .../records/<record_id>).""" with app.app_context(): InvenioRecordsREST(app) # create the record using the internal API internal_record = Record.create(test_data) with app.test_client() as client: headers = [('Content-Type', 'application/json'), ('Accept', 'application/json')] res = client.put(url_for(blueprint.name + '.' + RecordResource.view_name, record_id=internal_record.model.id), data=json.dumps(test_data_patched), headers=headers) assert res.status_code == 200 # check that the returned record matches the given data response_data = json.loads(res.get_data(as_text=True)) assert response_data['data'] == test_data_patched # check that an internal record returned id and that it contains # the same data assert 'id' in response_data.keys() internal_record = Record.get_record(response_data['id']) assert internal_record == response_data['data'] # check that the returned self link returns the same data subtest_self_link(response_data, res.headers, client)
def test_record_update_mutable(app, db): """Test updating mutables in a record.""" recid = uuid.UUID("262d2748-ba41-456f-a844-4d043a419a6f") # Create a new record with two mutables, a list and a dict rec = Record.create({"title": "Title", "list": ["foo"], "dict": {"moo": "boo"}}, id_=recid) # Make sure mutables are there before and after commit assert rec == {"title": "Title", "list": ["foo"], "dict": {"moo": "boo"}} db.session.commit() db.session.expunge_all() rec = Record.get_record(recid) assert rec == {"title": "Title", "list": ["foo"], "dict": {"moo": "boo"}} # Set the mutables under key rec["list"] = ["bar"] rec["dict"] = {"eggs": "bacon"} rec.commit() # Make sure it commits to DB assert rec == {"title": "Title", "list": ["bar"], "dict": {"eggs": "bacon"}} db.session.commit() db.session.expunge_all() rec = Record.get_record(recid) assert rec == {"title": "Title", "list": ["bar"], "dict": {"eggs": "bacon"}} # Update the mutables under key rec["list"].append("spam") rec["dict"]["ham"] = "chicken" rec.commit() # Make sure it commits to DB assert rec == {"title": "Title", "list": ["bar", "spam"], "dict": {"eggs": "bacon", "ham": "chicken"}} db.session.commit() db.session.expunge_all() rec = Record.get_record(recid) assert rec == {"title": "Title", "list": ["bar", "spam"], "dict": {"eggs": "bacon", "ham": "chicken"}}
def test_validate_partial(app, db): """Test partial validation.""" schema = {"properties": {"a": {"type": "string"}, "b": {"type": "string"}}, "required": ["b"]} data = {"a": "hello", "$schema": schema} with app.app_context(): # Test validation on create() # normal validation should fail because 'b' is required with pytest.raises(ValidationError) as exc_info: Record.create(data) assert "'b' is a required property" == exc_info.value.message # validate with a less restrictive validator record = Record.create(data, validator=PartialDraft4Validator) # set wrong data types should fails in any case data_incorrect = copy.deepcopy(data) data_incorrect["a"] = 1 with pytest.raises(ValidationError) as exc_info: Record.create(data_incorrect, validator=PartialDraft4Validator) assert "1 is not of type 'string'" == exc_info.value.message # Test validation on commit() # validation not passing with normal validator with pytest.raises(ValidationError) as exc_info: record.commit() assert "'b' is a required property" == exc_info.value.message # validation passing with less restrictive validator assert data == record.commit(validator=PartialDraft4Validator) # set wrong data types should fails in any case record["a"] = 1 with pytest.raises(ValidationError) as exc_info: record.commit(validator=PartialDraft4Validator)
def test_serialize(): """Test JSON serialize.""" data = json.loads( JSONLDSerializer(CONTEXT, schema_class=_TestSchema).serialize( PersistentIdentifier(pid_type='recid', pid_value='2'), Record({'title': 'mytitle', 'recid': '2'}) ) ) assert data == { '@id': 'http://localhost/record/2', 'http://purl.org/dc/terms/title': [{'@value': 'mytitle'}] } data = json.loads(JSONLDSerializer( CONTEXT, schema_class=_TestSchema, expanded=False).serialize( PersistentIdentifier(pid_type='recid', pid_value='2'), Record({'title': 'mytitle', 'recid': '2'}) ) ) assert data == { '@context': { '@base': 'http://localhost/record/', 'dct': 'http://purl.org/dc/terms/', 'recid': '@id', 'title': 'dct:title' }, 'recid': '2', 'title': 'mytitle' }
def record_upsert(json): """Insert or update a record.""" control_number = json.get('control_number', json.get('recid')) if control_number: control_number = int(control_number) pid_type = InspireRecordIdProvider.schema_to_pid_type(json['$schema']) try: pid = PersistentIdentifier.get(pid_type, control_number) record = Record.get_record(pid.object_uuid) record.update(json) record.commit() except PIDDoesNotExistError: record = Record.create(json, id_=None) # Create persistent identifier. inspire_recid_minter(str(record.id), json) if json.get('deleted'): new_recid = get_recid_from_ref(json.get('new_record')) if new_recid: merged_record = get_db_record(pid_type, new_recid) merge_pidstores_of_two_merged_records(merged_record.id, record.id) else: soft_delete_pidstore_for_record(record.id) return record
def test_rest_delete_record(app, db, es, es_acl_prepare, test_users): with app.test_client() as client: with db.session.begin_nested(): acl = DefaultACL(name='default', schemas=[RECORD_SCHEMA], priority=0, originator=test_users.u1, operation='update') actor = UserActor(name='u1', users=[test_users.u1], acl=acl, originator=test_users.u1) db.session.add(acl) db.session.add(actor) pid, record = create_record({'keywords': ['blah']}, clz=SchemaEnforcingRecord) RecordIndexer().index(record) current_search_client.indices.refresh() current_search_client.indices.flush() login(client, test_users.u2) response = client.delete(record_url(pid)) assert response.status_code == 403 login(client, test_users.u1) response = client.delete(record_url(pid)) assert response.status_code == 204 with pytest.raises(NoResultFound): Record.get_record(pid.object_uuid)
def test_metadata_extraction_video(app, db, cds_depid, bucket, video): """Test metadata extraction video mp4.""" recid = PersistentIdentifier.get('depid', cds_depid).object_uuid # simulate a no fully filled record record = Record.get_record(recid) if 'title' in record: del record['title'] validator = 'cds.modules.records.validators.PartialDraft4Validator' with pytest.raises(ValidationError): record.commit() record.commit(validator=import_string(validator)) # Extract metadata obj = ObjectVersion.create(bucket=bucket, key='video.mp4') obj_id = str(obj.version_id) dep_id = str(cds_depid) task_s = ExtractMetadataTask().s(uri=video, version_id=obj_id, deposit_id=dep_id) task_s.delay() # Check that deposit's metadata got updated record = Record.get_record(recid) assert 'extracted_metadata' in record['_cds'] assert record['_cds']['extracted_metadata'] # Check that ObjectVersionTags were added tags = ObjectVersion.query.first().get_tags() assert tags['duration'] == '60.095000' assert tags['bit_rate'] == '612177' assert tags['avg_frame_rate'] == '288000/12019' assert tags['codec_name'] == 'h264' assert tags['codec_long_name'] == 'H.264 / AVC / MPEG-4 AVC / ' \ 'MPEG-4 part 10' assert tags['width'] == '640' assert tags['height'] == '360' assert tags['nb_frames'] == '1440' assert tags['display_aspect_ratio'] == '16:9' assert tags['color_range'] == 'tv' # Undo ExtractMetadataTask().clean(deposit_id=dep_id, version_id=obj_id) # Check that deposit's metadata got reverted record = Record.get_record(recid) assert 'extracted_metadata' not in record['_cds'] # Check that ObjectVersionTags are still there (attached to the old obj) tags = ObjectVersion.query.first().get_tags() assert len(tags) == 11 # Simulate failed task, no extracted_metadata ExtractMetadataTask().clean(deposit_id=dep_id, version_id=obj_id) record = Record.get_record(recid) assert 'extracted_metadata' not in record['_cds'] # check again tags tags = ObjectVersion.query.first().get_tags() assert len(tags) == 11
def acl_changed_reindex(acl_id): """ ACL has been changed so reindex all the documents in the given index. :param acl_id: id of ACL instance """ logger.info('Reindexing started for ACL=%s', acl_id) timestamp = datetime.datetime.now().astimezone().isoformat() acl = ACL.query.filter_by(id=acl_id).one_or_none() if not acl: # deleted in the meanwhile, so just return return # pragma no cover # make sure all indices are flushed so that no resource is obsolete in index for schema in acl.schemas: current_search_client.indices.flush(index=schema_to_index(schema)[0]) indexer = RecordIndexer() updated_count = 0 removed_count = 0 for id in acl.get_matching_resources(): try: rec = Record.get_record(id) except: # pragma no cover # record removed in the meanwhile by another thread/process, # indexer should have been called to remove it from ES # won't test this so pragma no cover continue try: indexer.index(rec) updated_count += 1 except Exception as e: # pragma no cover logger.exception('Error indexing ACL for resource %s: %s', id, e) # reindex the resources those were indexed by this acl but no longer should be for id in acl.used_in_records(older_than_timestamp=timestamp): try: rec = Record.get_record(id) except NoResultFound: # pragma no cover continue except: # pragma no cover logger.exception('Unexpected exception in record reindexing') continue try: removed_count += 1 indexer.index(rec) except: # pragma no cover logger.exception('Error indexing ACL for obsolete resource %s', id) logger.info( 'Reindexing finished for ACL=%s, acl applied to %s records, acl removed from %s records', acl_id, updated_count, removed_count)
def records(db): """Create sample records with references.""" return [ Record.create({'$ref': 'ref1'}), Record.create({'$ref': 'ref2'}), Record.create({'reflist': [ {'$ref': 'ref3'}, {'$ref': 'ref4'} ]}) ]
def test_file_permissions(app, db, test_object, # fixtures user, access_right, expected): """Test file permissions.""" # Create test users admin = User(email='*****@*****.**', password='******') owner = User(email='*****@*****.**', password='******') auth = User(email='*****@*****.**', password='******') db.session.add_all([admin, owner, auth]) db.session.add( ActionUsers.allow(ActionNeed('admin-access'), user=admin) ) # Create test record rec_uuid = uuid.uuid4() PersistentIdentifier.create( 'recid', '1', object_type='rec', object_uuid=rec_uuid, status=PIDStatus.REGISTERED ) Record.create({ 'recid': 1, 'owners': [2], 'access_right': access_right, '_files': [ { 'key': test_object.key, 'bucket': str(test_object.bucket_id), 'checksum': 'invalid' }, ] }, id_=rec_uuid) db.session.add( RecordsBuckets(record_id=rec_uuid, bucket_id=test_object.bucket_id) ) file_url = url_for( 'invenio_records_ui.recid_files', pid_value='1', filename=test_object.key ) db.session.commit() with app.test_client() as client: if user: # Login as user with client.session_transaction() as sess: sess['user_id'] = User.query.filter_by( email='{}@zenodo.org'.format(user)).one().id sess['_fresh'] = True res = client.get(file_url) assert res.status_code == expected
def test_undelete_with_get(testapp, db): """Test undelete a record.""" record = Record.create({'title': 'test'}) db.session.commit() record.delete() db.session.commit() record = Record.get_record(record.id, with_deleted=True) record.undelete() record.commit() db.session.commit() assert record == {}
def test_record_deletion(test_record): recid = test_record.get('control_number') assert recid test_record.delete() db.session.commit() pid = PersistentIdentifier.get('recid', recid) assert pid.status == PIDStatus.DELETED with raises(NoResultFound): Record.get_record(pid.object_uuid)
def files(): """Load files.""" data_path = os.path.join(os.path.dirname(__file__), 'data') # Create location loc = Location(name='local', uri=data_path, default=True) db.session.commit() # Bucket bucket = Bucket.create(loc) # Example files from the data folder example_files = ( 'markdown.md', 'csvfile.csv', 'zipfile.zip', 'jsonfile.json', 'xmlfile.xml', 'notebook.ipynb', 'jpgfile.jpg', 'pngfile.png', ) # Create single file records for f in example_files: with open(os.path.join(data_path, f), 'rb') as fp: create_object(bucket, f, fp) # Create a multi-file record rec_uuid = uuid4() provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid) data = { 'pid_value': provider.pid.pid_value, 'files': [] } # Template to create different files template_file = { 'uri': '/files/{0}/{1}', 'key': '', 'bucket': str(bucket.id), 'local': True } for filename in example_files: file_data = template_file.copy() file_data['uri'] = file_data['uri'].format(str(bucket.id), filename) file_data['key'] = filename data['files'].append(file_data) Record.create(data, id_=rec_uuid) db.session.commit()
def test_change_record_community(app, test_records): """Test updating the community field fails.""" with app.app_context(): record = Record.get_record(test_records[0].record_id) del record['community'] with pytest.raises(AlteredRecordError): record.commit() with app.app_context(): record = Record.get_record(test_records[0].record_id) record['community'] = str(uuid.uuid4()) with pytest.raises(AlteredRecordError): record.commit()
def files(): """Load files.""" data_path = os.path.join(os.path.dirname(__file__), 'data') # Create location loc = Location(name='local', uri=data_path, default=True) db.session.commit() # Bucket bucket = Bucket.create(loc) # Example files from the data folder example_files = ( 'markdown.md', 'csvfile.csv', 'zipfile.zip', 'jsonfile.json', 'xmlfile.xml', 'notebook.ipynb', 'jpgfile.jpg', 'pngfile.png', ) # Create single file records for f in example_files: with open(os.path.join(data_path, f), 'rb') as fp: create_object(bucket, f, fp) # Create a multi-file record rec_uuid = uuid4() provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid) data = {'pid_value': provider.pid.pid_value, 'files': []} # Template to create different files template_file = { 'uri': '/files/{0}/{1}', 'key': '', 'bucket': str(bucket.id), 'local': True } for filename in example_files: file_data = template_file.copy() file_data['uri'] = file_data['uri'].format(str(bucket.id), filename) file_data['key'] = filename data['files'].append(file_data) Record.create(data, id_=rec_uuid) db.session.commit()
def test_db(): """Test database backend.""" app = Flask(__name__) app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get( 'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db' ) FlaskCLI(app) InvenioDB(app) InvenioRecords(app) with app.app_context(): create_database(db.engine.url) db.create_all() assert len(db.metadata.tables) == 3 data = {'title': 'Test'} from invenio_records.models import RecordMetadata as RM # Create a record with app.app_context(): assert RM.query.count() == 0 record_uuid = Record.create(data).id db.session.commit() assert RM.query.count() == 1 db.session.commit() # Retrieve created record with app.app_context(): record = Record.get_record(record_uuid) assert record.dumps() == data with pytest.raises(NoResultFound): Record.get_record(uuid.uuid4()) record['field'] = True record = record.patch([ {'op': 'add', 'path': '/hello', 'value': ['world']} ]) assert record['hello'] == ['world'] record.commit() db.session.commit() with app.app_context(): record2 = Record.get_record(record_uuid) assert record2.model.version_id == 2 assert record2['field'] assert record2['hello'] == ['world'] db.session.commit() # Cannot commit record without model (i.e. Record.create_record) with app.app_context(): record3 = Record({'title': 'Not possible'}) with pytest.raises(RecordNotCommitableError): record3.commit() with app.app_context(): db.drop_all() drop_database(db.engine.url)
def migrate_chunk(chunk, broken_output=None, dry_run=False): from invenio_indexer.api import RecordIndexer from ..pidstore.minters import inspire_recid_minter indexer = RecordIndexer() index_queue = [] for raw_record in chunk: record = marc_create_record(raw_record, keep_singletons=False) json_record = create_record(record) if '$schema' in json_record: json_record['$schema'] = url_for( 'invenio_jsonschemas.get_schema', schema_path="records/{0}".format(json_record['$schema']) ) rec_uuid = str(Record.create(json_record, id_=None).id) # Create persistent identifier. pid = inspire_recid_minter(rec_uuid, json_record) index_queue.append(pid.object_uuid) db.session.commit() # Request record indexing for i in index_queue: indexer.index_by_id(i) # Send task to migrate files. return rec_uuid
def data_policies(): """Load demo Data Policy records.""" from invenio_db import db from invenio_records import Record from invenio_indexer.api import RecordIndexer from invenio_pidstore.errors import PIDDoesNotExistError, \ PersistentIdentifierError from invenio_pidstore.models import PIDStatus, PersistentIdentifier from invenio_pidstore.fetchers import recid_fetcher from invenio_pidstore.minters import recid_minter from cernopendata.modules.records.minters.recid import \ cernopendata_recid_minter indexer = RecordIndexer() schema = current_app.extensions['invenio-jsonschemas'].path_to_url( 'records/data-policies-v1.0.0.json' ) data = pkg_resources.resource_filename('cernopendata', 'modules/fixtures/data') data_policies_json = glob.glob(os.path.join(data, '*.json')) for filename in data_policies_json: with open(filename, 'rb') as source: for data in json.load(source): id = uuid.uuid4() cernopendata_recid_minter(id, data) record = Record.create(data, id_=id) record['$schema'] = schema db.session.commit() indexer.index(record) db.session.expunge_all()
def news(): """Load demo news records.""" from invenio_db import db from invenio_records import Record from invenio_indexer.api import RecordIndexer from cernopendata.modules.records.minters.artid import \ cernopendata_articleid_minter indexer = RecordIndexer() schema = current_app.extensions['invenio-jsonschemas'].path_to_url( 'records/article-v1.0.0.json' ) data = pkg_resources.resource_filename('cernopendata', 'modules/fixtures/data') articles_json = glob.glob(os.path.join(data, 'articles', 'news', '*.json')) for filename in articles_json: with open(filename, 'rb') as source: for data in json.load(source): if "collections" not in data and \ not isinstance(data.get("collections", None), basestring): data["collections"] = [] data["collections"].append({"primary": "News"}) id = uuid.uuid4() cernopendata_articleid_minter(id, data) record = Record.create(data, id_=id) record['$schema'] = schema db.session.commit() indexer.index(record) db.session.expunge_all()
def create_record(ctx): """ Creates the record in the database. :param ctx: The record metadata as a dictionary. :return: the recid and the uuid """ record_information = create_data_structure(ctx) record_id = uuid.uuid4() pid = recid_minter(record_id, record_information) record_information['recid'] = int(pid.pid_value) record_information['uuid'] = str(record_id) Record.create(record_information, id_=record_id) db.session.commit() return record_information
def test_change_record_schema_fails(app, test_records): """Test updating the $schema field fails.""" with app.app_context(): record = Record.get_record(test_records[0].record_id) del record['$schema'] with pytest.raises(AlteredRecordError): record.commit()
def test_record_patch_immutable_fields(app, test_records, test_users, login_user): """Test invalid modification of record draft with HTTP PATCH.""" with app.app_context(): record = Record.get_record(test_records[0].record_id) with app.test_client() as client: user = test_users['admin'] login_user(user, client) headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] for path in IMMUTABLE_PATHS: for command in [ {"op": "replace", "path": path, "value": ""}, {"op": "remove", "path": path}, {"op": "add", "path": path, "value": ""}, {"op": "copy", "from": "/title", "path": path, "value": ""}, {"op": "move", "from": "/title", "path": path, "value": ""}, ]: draft_patch_res = client.patch( url_for('b2share_records_rest.b2rec_item', pid_value=test_records[0].pid), data=json.dumps([command]), headers=headers) assert draft_patch_res.status_code == 400
def revert_delete_record(pid): """Reverts deletion action of a record.""" pid_obj = PersistentIdentifier.query.filter_by(pid_value=pid).one() record_uuid = pid_obj.object_uuid deleted = Record.get_record(record_uuid, with_deleted=True) if not deleted.is_deleted: click.secho(f"Record {pid} was not deleted. Aborting.", fg="red") return # revert to previous revision record = deleted.revert(deleted.revision_id - 1) all_pid_objects = PersistentIdentifier.query.filter_by( object_uuid=record_uuid) # trying to get all the pid types (including legacy pids) for pid_object in all_pid_objects: pid_object.status = PIDStatus.REGISTERED db.session.commit() indexer_class = current_app_ils.indexer_by_pid_type(pid_obj.pid_type) indexer_class.index(record) click.secho(f"Record {pid} is reverted from deletion.", fg="green")
def store_record(obj, *args, **kwargs): """Create and index new record in main record space.""" assert "$schema" in obj.data, "No $schema attribute found!" # Create record # FIXME: Do some preprocessing of obj.data before creating a record so that # we're sure that the schema will be validated without touching the full # holdingpen stack. record = Record.create(obj.data, id_=None) # Create persistent identifier. pid = inspire_recid_minter(str(record.id), record) # Commit any changes to record record.commit() # Dump any changes to record obj.data = record.dumps() # Commit to DB before indexing db.session.commit() # Index record indexer = RecordIndexer() indexer.index_by_id(pid.object_uuid)
def test_record_replace_refs(app): """Test the replacement of JSON references using JSONResolver.""" with app.app_context(): record = Record.create({ 'one': {'$ref': 'http://nest.ed/A'}, 'three': {'$ref': 'http://nest.ed/ABC'} }) app.extensions['invenio-records'].loader_cls = json_loader_factory( JSONResolver(plugins=['demo.json_resolver'])) out_json = record.replace_refs() expected_json = { 'one': { 'letter': 'A', 'next': '.', }, 'three': { 'letter': 'A', 'next': { 'letter': 'B', 'next': { 'letter': 'C', 'next': '.', }, }, }, } assert out_json == expected_json
def migrate_chunk(chunk, broken_output=None, dry_run=False): from invenio_indexer.api import RecordIndexer from ..pidstore.minters import inspire_recid_minter indexer = RecordIndexer() index_queue = [] for raw_record in chunk: record = marc_create_record(raw_record, keep_singletons=False) json_record = create_record(record) if '$schema' in json_record: json_record['$schema'] = url_for( 'invenio_jsonschemas.get_schema', schema_path="records/{0}".format(json_record['$schema']) ) rec_uuid = str(Record.create(json_record, id_=None).id) # Create persistent identifier. pid = inspire_recid_minter(rec_uuid, json_record) index_queue.append(pid.object_uuid) db.session.commit() # Request record indexing for i in index_queue: indexer.index_by_id(i) # Send task to migrate files. return rec_uuid
def test_default_deduplication_validator(app, simple_record, mocker): """Make sure default deduplication validator works.""" from invenio_records import Record mocker.patch('invenio_matcher.api.execute', duplicated_result) with app.app_context(): record = Record(simple_record) queries = [{'type': 'exact', 'match': 'title'}] expected = [MatchResult(1, record, 1)] result = list( match( record, index="records", doc_type="record", queries=queries, validator=None, )) assert expected == result # Test the same again to see if validator resets expected = [MatchResult(1, record, 1)] result = list( match( record, index="records", doc_type="record", queries=queries, validator=None, )) assert expected == result
def test_preprocessor_mixin_record(app): """Test preprocessor mixin.""" with db.session.begin_nested(): recuuid = uuid.uuid4() record = Record.create({ 'title': 'test', 'aref': {'$ref': '#/title'}}, id_=recuuid) record.model.created = datetime(2015, 10, 1, 11, 11, 11, 1) pid = PersistentIdentifier.create( 'recid', '1', object_type='rec', object_uuid=recuuid, status=PIDStatus.REGISTERED) db.session.commit() data = PreprocessorMixin().preprocess_record(pid, record) for k in keys: assert k in data assert data['metadata']['title'] == 'test' assert data['metadata']['aref'] == {'$ref': '#/title'} assert data['created'] == '2015-10-01T11:11:11.000001+00:00' assert data['revision'] == 1 data = PreprocessorMixin(replace_refs=True).preprocess_record( pid, Record({'title': 'test2', 'aref': {'$ref': '#/title'}})) assert data['created'] is None assert data['updated'] is None assert data['metadata']['aref'] == 'test2'
def test_permission_factory(app, db, action, permission_factory): """Test revisions.""" InvenioAccess(app) rec_uuid = uuid.uuid4() with db.session.begin_nested(): user_all = User(email='*****@*****.**') user_one = User(email='*****@*****.**') user_none = User(email='*****@*****.**') db.session.add(user_all) db.session.add(user_one) db.session.add(user_none) db.session.add(ActionUsers(action=action, user=user_all, argument=None)) db.session.add( ActionUsers(action=action, user=user_one, argument=str(rec_uuid))) record = Record.create({'title': 'permission test'}, id_=rec_uuid) # Create a record and assign permissions. permission = permission_factory(record) # Assert which permissions has access. assert permission.allows(FakeIdentity(UserNeed(user_all.id))) assert permission.allows(FakeIdentity(UserNeed(user_one.id))) assert not permission.allows(FakeIdentity(UserNeed(user_none.id)))
def test_record_crud_2(self, load_entry_points, app, db, record_xml): synchronizer = current_oai_client.providers["uk"].synchronizers["xoai"] oai_sync = OAISync(provider_code="uk") synchronizer.oai_sync = oai_sync oai_identifier = "oai:dspace.cuni.cz:20.500.11956/2623" # vytvořenà záznamu synchronizer.record_crud(oai_identifier=oai_identifier, xml=record_xml) db.session.commit() oai_rec = OAIRecord.get_record(oai_identifier) assert oai_rec is not None # smazánà záznamu synchronizer.record_crud(oai_rec=oai_rec, deleted=True) db.session.commit() oai_rec2 = OAIRecord.get_record(oai_identifier) assert oai_rec == oai_rec2 # obnovenà záznamu synchronizer.record_crud(oai_rec=oai_rec, timestamp='2050-10-22T08:18:08.567698+00:00', xml=record_xml) db.session.commit() oai_rec3 = OAIRecord.get_record(oai_identifier) assert oai_rec3 is not None record = Record.get_record(oai_rec.id) assert record is not None
def test_reindex(app, script_info): """Test reindex.""" # load records with app.test_request_context(): runner = CliRunner() rec_uuid = uuid.uuid4() data = {'title': 'Test0'} record = Record.create(data, id_=rec_uuid) db.session.commit() # Initialize queue res = runner.invoke(cli.queue, ['init', 'purge'], obj=script_info) assert 0 == res.exit_code res = runner.invoke(cli.reindex, ['--yes-i-know'], obj=script_info) assert 0 == res.exit_code res = runner.invoke(cli.run, [], obj=script_info) assert 0 == res.exit_code sleep(5) indexer = RecordIndexer() index, doc_type = indexer.record_to_index(record) res = current_search_client.get(index=index, doc_type=doc_type, id=rec_uuid) assert res['found'] # Destroy queue res = runner.invoke(cli.queue, ['delete'], obj=script_info) assert 0 == res.exit_code
def test_preprocessor_mixin_record(app, db): """Test preprocessor mixin.""" pid, record = create_record({'title': 'test', 'aref': {'$ref': '#/title'}}) record.model.created = datetime(2015, 10, 1, 11, 11, 11, 1) db.session.commit() data = PreprocessorMixin().preprocess_record(pid, record) for k in keys: assert k in data assert data['metadata']['title'] == 'test' assert data['metadata']['aref'] == {'$ref': '#/title'} assert data['created'] == '2015-10-01T11:11:11.000001+00:00' assert data['revision'] == 1 data = PreprocessorMixin(replace_refs=True).preprocess_record( pid, Record({ 'title': 'test2', 'aref': { '$ref': '#/title' } })) assert data['created'] is None assert data['updated'] is None assert data['metadata']['aref'] == 'test2'
def test_record_dump(app, db): """Test record dump method.""" with app.app_context(): record = Record.create({'foo': {'bar': 'Bazz', }, }) record_dump = record.dumps() record_dump['foo']['bar'] = 'Spam' assert record_dump['foo']['bar'] != record['foo']['bar']
def records(pids, db): """Fixture for the records.""" pid_versions = ['h1v1', 'h1v2', 'h2v1'] schema = { 'type': 'object', 'properties': { 'title': { 'type': 'string' }, }, } data = { name: { 'title': 'Test version {}'.format(name), 'recid': pids[name].pid_value, '$schema': schema } for name in pid_versions } records = dict() for name in pid_versions: record = Record.create(data[name]) pids[name].assign('rec', record.id) records[name] = record return records
def load_records(app, filename, schema, tries=5): """Try to index records.""" indexer = RecordIndexer() records = [] with app.app_context(): with mock.patch('invenio_records.api.Record.validate', return_value=None): data_filename = pkg_resources.resource_filename( 'invenio_records', filename) records_data = load(data_filename) with db.session.begin_nested(): for item in records_data: record_id = uuid.uuid4() item_dict = dict(marc21.do(item)) item_dict['$schema'] = schema recid_minter(record_id, item_dict) oaiid_minter(record_id, item_dict) record = Record.create(item_dict, id_=record_id) indexer.index(record) records.append(record.id) db.session.commit() # Wait for indexer to finish for i in range(tries): response = current_search_client.search() if response['hits']['total'] >= len(records): break current_search.flush_and_refresh('_all') return records
def store_record(obj, *args, **kwargs): """Create and index new record in main record space.""" assert "$schema" in obj.data, "No $schema attribute found!" # Create record # FIXME: Do some preprocessing of obj.data before creating a record so that # we're sure that the schema will be validated without touching the full # holdingpen stack. record = Record.create(obj.data, id_=None) # Create persistent identifier. pid = inspire_recid_minter(str(record.id), record) # Commit any changes to record record.commit() # Dump any changes to record obj.data = record.dumps() # Commit to DB before indexing db.session.commit() # Index record indexer = RecordIndexer() indexer.index_by_id(pid.object_uuid)
def create_record(ctx): """ Creates the record in the database. :param ctx: The record metadata as a dictionary. :return: the recid and the uuid """ record_information = create_data_structure(ctx) record_id = uuid.uuid4() pid = recid_minter(record_id, record_information) record_information['recid'] = int(pid.pid_value) record_information['uuid'] = str(record_id) Record.create(record_information, id_=record_id) db.session.commit() return record_information
def test_record_dump(app, db): """Test record dump method.""" with app.app_context(): record = Record.create({"foo": {"bar": "Bazz"}}) record_dump = record.dumps() record_dump["foo"]["bar"] = "Spam" assert record_dump["foo"]["bar"] != record["foo"]["bar"]
def delete_model(self, model): """Delete a record.""" try: if model.json is None: return True record = Record(model.json, model=model) record.delete() db.session.commit() except SQLAlchemyError as e: if not self.handle_view_exception(e): flash('Failed to delete record. %s' % str(e), category='error') db.session.rollback() return False return True
def test_record_replace_refs(app, db): """Test the replacement of JSON references using JSONResolver.""" record = Record.create({ 'one': { '$ref': 'http://nest.ed/A' }, 'three': { '$ref': 'http://nest.ed/ABC' } }) app.extensions['invenio-records'].loader_cls = json_loader_factory( JSONResolver(plugins=['demo.json_resolver'])) out_json = record.replace_refs() expected_json = { 'one': { 'letter': 'A', 'next': '.', }, 'three': { 'letter': 'A', 'next': { 'letter': 'B', 'next': { 'letter': 'C', 'next': '.', }, }, }, } assert out_json == expected_json
def test_record_patch_immutable_fields(app, test_records, test_users, login_user): """Test invalid modification of record draft with HTTP PATCH.""" with app.app_context(): record = Record.get_record(test_records[0].record_id) with app.test_client() as client: user = test_users['admin'] login_user(user, client) headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] for path in IMMUTABLE_PATHS: for command in [ {"op": "replace", "path": path, "value": ""}, {"op": "remove", "path": path}, {"op": "add", "path": path, "value": ""}, {"op": "copy", "from": "/title", "path": path, "value": ""}, {"op": "move", "from": "/title", "path": path, "value": ""}, ]: draft_patch_res = client.patch( url_for('b2share_records_rest.b2rec_item', pid_value=test_records[0].pid), data=json.dumps([command]), headers=headers) assert draft_patch_res.status_code == 400 data = json.loads(draft_patch_res.get_data(as_text=True)) assert data['errors'][0]['message'] == \ 'The field "{}" is immutable.'.format(path)
def test_permission_factory(app, action, permission_factory): """Test revisions.""" InvenioAccess(app) with app.app_context(): rec_uuid = uuid.uuid4() with db.session.begin_nested(): user_all = User(email='*****@*****.**') user_one = User(email='*****@*****.**') user_none = User(email='*****@*****.**') db.session.add(user_all) db.session.add(user_one) db.session.add(user_none) db.session.add(ActionUsers(action=action, user=user_all, argument=None)) db.session.add(ActionUsers(action=action, user=user_one, argument=str(rec_uuid))) record = Record.create({'title': 'permission test'}, id_=rec_uuid) # Create a record and assign permissions. permission = permission_factory(record) # Assert which permissions has access. assert permission.allows(FakeIdentity(UserNeed(user_all.id))) assert permission.allows(FakeIdentity(UserNeed(user_one.id))) assert not permission.allows(FakeIdentity(UserNeed(user_none.id)))
def files_permission_factory(obj=None, action=None): """Permission for files are always based on the type of bucket. 1. Record bucket: Read access only with open and restricted access. 2. Any other bucket is restricted to admins only. """ # Extract bucket id bucket_id = None if isinstance(obj, Bucket): bucket_id = str(obj.id) elif isinstance(obj, ObjectVersion): bucket_id = str(obj.bucket_id) elif isinstance(obj, MultipartObject): bucket_id = str(obj.bucket_id) elif isinstance(obj, FileObject): bucket_id = str(obj.bucket_id) # Retrieve record if bucket_id is not None: record_bucket = RecordsBuckets.query.filter_by(bucket_id=bucket_id).one_or_none() if record_bucket is not None: record = Record.get_record(record_bucket.record_id) return FilePermission.create(record, action) return AdminPermission.create(obj, action)
def load_records(app, filename, schema, tries=5): """Try to index records.""" indexer = RecordIndexer() records = [] with app.app_context(): with mock.patch('invenio_records.api.Record.validate', return_value=None): data_filename = pkg_resources.resource_filename( 'invenio_records', filename) records_data = load(data_filename) with db.session.begin_nested(): for item in records_data: record_id = uuid.uuid4() item_dict = dict(marc21.do(item)) item_dict['$schema'] = schema recid_minter(record_id, item_dict) oaiid_minter(record_id, item_dict) record = Record.create(item_dict, id_=record_id) indexer.index(record) records.append(record.id) db.session.commit() # Wait for indexer to finish for i in range(tries): response = current_search_client.search() if response['hits']['total'] >= len(records): break sleep(5) return records
def _update_event_bucket(event): """Update event's payload with correct bucket of deposit.""" depid = event.payload['deposit_id'] dep_uuid = str(PersistentIdentifier.get('depid', depid).object_uuid) deposit_bucket = Record.get_record(dep_uuid)['_buckets']['deposit'] event.payload['bucket_id'] = deposit_bucket flag_modified(event, 'payload')
def test_metadata_extraction_video_mp4(app, db, depid, bucket, video_mp4): """Test metadata extraction video mp4.""" # Extract metadata obj = ObjectVersion.create(bucket=bucket, key='video.mp4') video_metadata_extraction.s(uri=video_mp4, object_version=str(obj.version_id), deposit_id=str(depid)).delay() # Check that deposit's metadata got updated recid = PersistentIdentifier.get('depid', depid).object_uuid record = Record.get_record(recid) assert 'extracted_metadata' in record['_deposit'] assert record['_deposit']['extracted_metadata'] # Check that ObjectVersionTags were added obj = ObjectVersion.query.first() tags = obj.get_tags() assert tags['duration'] == '60.095000' assert tags['bit_rate'] == '612177' assert tags['size'] == '5510872' assert tags['avg_frame_rate'] == '288000/12019' assert tags['codec_name'] == 'h264' assert tags['width'] == '640' assert tags['height'] == '360' assert tags['nb_frames'] == '1440' assert tags['display_aspect_ratio'] == '0:1' assert tags['color_range'] == 'tv'
def record_upsert(json): """Insert or update a record.""" control_number = json.get('control_number', json.get('recid')) if control_number: control_number = int(control_number) pid_type = InspireRecordIdProvider.schema_to_pid_type(json['$schema']) try: pid = PersistentIdentifier.get(pid_type, control_number) record = Record.get_record(pid.object_uuid) record.update(json) record.commit() except PIDDoesNotExistError: record = Record.create(json, id_=None) # Create persistent identifier. inspire_recid_minter(str(record.id), json) return record
def _get_record_from_workflow(workflow): assert len(workflow.objects) == 1 workflow_object = workflow.objects[0] recid = workflow_object.data['control_number'] pid = PersistentIdentifier.get('recid', recid) return Record.get_record(pid.object_uuid)
def create_record(data): """Create a test record.""" with db.session.begin_nested(): data = copy.deepcopy(data) rec_uuid = uuid.uuid4() pid = current_pidstore.minters['recid'](rec_uuid, data) record = Record.create(data, id_=rec_uuid) return pid, record
def create_object(bucket, file_name, stream): """Object creation inside the bucket using the file and its content.""" obj = ObjectVersion.create(bucket, file_name, stream=stream) rec_uuid = uuid4() provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid) data = { 'pid_value': provider.pid.pid_value, 'files': [ { 'uri': '/files/{0}/{1}'.format(str(bucket.id), file_name), 'key': file_name, 'size': obj.file.size, 'bucket': str(bucket.id), 'local': True } ] } Record.create(data, id_=rec_uuid)
def test_invalid_patch(app): """Test INVALID record patch request (PATCH .../records/<record_id>).""" with app.app_context(): InvenioRecordsREST(app) with app.test_client() as client: # check that PATCH with non existing id will return 404 headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] res = client.patch(url_for(blueprint.name + '.' + RecordResource.view_name, record_id=0), data=json.dumps(test_patch), headers=headers) assert res.status_code == 404 # create the record using the internal API internal_record = Record.create(test_data) # check that PATCH with non accepted format will return 406 headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'video/mp4')] res = client.patch(url_for(blueprint.name + '.' + RecordResource.view_name, record_id=internal_record.model.id), data=json.dumps(test_patch), headers=headers) assert res.status_code == 406 # check that PATCH with non-json Content-Type will return 400 headers = [('Content-Type', 'video/mp4'), ('Accept', 'application/json')] res = client.patch(url_for(blueprint.name + '.' + RecordResource.view_name, record_id=internal_record.model.id), data=json.dumps(test_patch), headers=headers) assert res.status_code == 415 # check that PATCH with invalid json-patch will return 400 headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] res = client.patch(url_for(blueprint.name + '.' + RecordResource.view_name, record_id=internal_record.model.id), data=json.dumps([ {'invalid': 'json-patch'} ]), headers=headers) assert res.status_code == 400 # check that PATCH with invalid json will return 400 headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] res = client.patch(url_for(blueprint.name + '.' + RecordResource.view_name, record_id=internal_record.model.id), data='{sdfsdf', headers=headers) assert res.status_code == 400
def update_authors_recid(record_id, uuid, profile_recid): """Update author profile for a given signature. The method receives UUIDs representing record and signature respectively together with an author profile recid. The new recid will be placed in the signature with the given UUID. :param record_id: A string representing UUID of a given record. Example: record_id = "a5afb151-8f75-4e91-8dc1-05e7e8e8c0b8" :param uuid: A string representing UUID of a given signature. Example: uuid = "c2f432bd-2f52-4c16-ac66-096f168c762f" :param profile_recid: A string representing author profile recid, that updated signature should point to. Example: profile_recid = "1" """ try: record = Record.get_record(record_id) update_flag = False for author in record['authors']: if author['uuid'] == uuid: author['recid'] = str(profile_recid) update_flag = True if update_flag: # Disconnect the signal on insert of a new record. before_record_index.disconnect(append_updated_record_to_queue) # Update the record in the database. record.commit() db.session.commit() # Update the record in Elasticsearch. indexer = RecordIndexer() indexer.index_by_id(record.id) except StaleDataError as exc: raise update_authors_recid.retry(exc=exc) finally: # Reconnect the disconnected signal. before_record_index.connect(append_updated_record_to_queue) # Report. logger.info("Updated signature %s with profile %s", uuid, profile_recid)
def get_primary_arxiv_category(self): try: pid = PersistentIdentifier.get('recid', self.control_number) record = Record.get_record(pid.object_uuid) return get_arxiv_primary_category(record) except PIDDoesNotExistError: # records imported from Inspire won't be found pass return None
def test_record_replace_refs(app, db): """Test the replacement of JSON references using JSONResolver.""" record = Record.create({"one": {"$ref": "http://nest.ed/A"}, "three": {"$ref": "http://nest.ed/ABC"}}) app.extensions["invenio-records"].loader_cls = json_loader_factory(JSONResolver(plugins=["demo.json_resolver"])) out_json = record.replace_refs() expected_json = { "one": {"letter": "A", "next": "."}, "three": {"letter": "A", "next": {"letter": "B", "next": {"letter": "C", "next": "."}}}, } assert out_json == expected_json