def _create_record_from_filepath(path, rec_uuid, indexer, versions, verbose): with open(path) as record_file: record_str = record_file.read() record_str = resolve_community_id(record_str) record_str = resolve_block_schema_id(record_str) json_data = json.loads(record_str) b2share_deposit_uuid_minter(rec_uuid, data=json_data) deposit = Deposit.create(json_data, id_=rec_uuid) ObjectVersion.create(deposit.files.bucket, 'myfile', stream=BytesIO(b'mycontent')) deposit.publish() pid, record = deposit.fetch_published() indexer.index(record) if verbose > 0: click.secho('created new record: {}'.format(str(rec_uuid))) last_id = pid.pid_value for i in range(2 * versions): rec_uuid = uuid4() json_data = json.loads(record_str) b2share_deposit_uuid_minter(rec_uuid, data=json_data) deposit2 = Deposit.create(json_data, id_=rec_uuid, version_of=last_id) ObjectVersion.create(deposit2.files.bucket, 'myfile-ver{}'.format(i), stream=BytesIO(b'mycontent')) deposit2.publish() pid, record2 = deposit2.fetch_published() indexer.index(record2) last_id = pid.pid_value if verbose > 0: click.secho('created new version: {}'.format(str(rec_uuid))) return record, deposit
def _create_record_from_filepath(path, rec_uuid, indexer, versions, verbose): with open(path) as record_file: record_str = record_file.read() record_str = resolve_community_id(record_str) record_str = resolve_block_schema_id(record_str) json_data = json.loads(record_str) b2share_deposit_uuid_minter(rec_uuid, data=json_data) deposit = Deposit.create(json_data, id_=rec_uuid) ObjectVersion.create(deposit.files.bucket, 'myfile', stream=BytesIO(b'mycontent')) deposit.publish() pid, record = deposit.fetch_published() indexer.index(record) if verbose > 0: click.secho('created new record: {}'.format(str(rec_uuid))) last_id = pid.pid_value for i in range(2*versions): rec_uuid = uuid4() json_data = json.loads(record_str) b2share_deposit_uuid_minter(rec_uuid, data=json_data) deposit2 = Deposit.create(json_data, id_=rec_uuid, version_of=last_id) ObjectVersion.create(deposit2.files.bucket, 'myfile-ver{}'.format(i), stream=BytesIO(b'mycontent')) deposit2.publish() pid, record2 = deposit2.fetch_published() indexer.index(record2) last_id = pid.pid_value if verbose > 0: click.secho('created new version: {}'.format(str(rec_uuid))) return record, deposit
def test_deposit_create_with_invalid_fields_fails(app, test_records_data): """Test deposit creation without or with an invalid field fails.""" data = test_records_data[0] with app.app_context(): data['publication_state'] = 'published' with pytest.raises(InvalidDepositError): deposit = Deposit.create(deepcopy(data)) with app.app_context(): data['$schema'] = '__garbage__' with pytest.raises(InvalidDepositError): deposit = Deposit.create(deepcopy(data))
def test_unpublished_deposit_unindex(app, test_users, draft_deposits, script_info, login_user): """Check that deleting an unpublished deposit also removes it from the search index.""" creator = test_users['deposits_creator'] with app.app_context(): Deposit.get_record(draft_deposits[0].deposit_id).delete() # execute scheduled tasks synchronously process_bulk_queue.delay() # flush the indices so that indexed records are searchable current_search_client.indices.flush('*') # deleted record should not be searchable subtest_record_search(app, creator, [], draft_deposits[1:], login_user)
def test_published_deposit_unindex(app, test_users, test_records, script_info, login_user): """Check that deleting a published deposit also removes it from the search index.""" creator = test_users['deposits_creator'] with app.app_context(): Deposit.get_record(test_records[0].deposit_id).delete() # execute scheduled tasks synchronously process_bulk_queue.delay() # flush the indices so that indexed records are searchable current_search_client.indices.flush('*') # deleted record should not be searchable subtest_record_search(app, creator, test_records, test_records[1:], login_user)
def test_deposit_create_with_invalid_community_fails(app, test_records_data): """Test deposit creation without or with an invalid community fails.""" data = test_records_data[0] with app.app_context(): # test with no community del data['community'] with pytest.raises(ValidationError): deposit = Deposit.create(deepcopy(data)) with app.app_context(): # test with an invalid community data['community'] = str(uuid.uuid4()) with pytest.raises(InvalidDepositError): deposit = Deposit.create(deepcopy(data))
def test_change_deposit_community(app, draft_deposits): """Test deposit creation without or with an invalid community fails.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) # test removing the community id del deposit['community'] with pytest.raises(AlteredRecordError): deposit.commit() with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) # test changing the community id deposit['community'] = str(uuid.uuid4()) with pytest.raises(InvalidDepositError): deposit.commit()
def test_change_deposit_community(app, draft_deposits): """Test deposit creation without or with an invalid community fails.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].id) # test removing the community id del deposit['community'] with pytest.raises(AlteredRecordError): deposit.commit() with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].id) # test changing the community id deposit['community'] = str(uuid.uuid4()) with pytest.raises(InvalidDepositError): deposit.commit()
def test_generation_of_external_pids(app, records_data_with_external_pids, deposit_with_external_pids, test_users): """Test the generate_external_pids function.""" expected_output = records_data_with_external_pids['external_pids'][:] with app.app_context(): deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) output = generate_external_pids(deposit) assert output == expected_output all_files = list(deposit.files) for f in all_files: if f.obj.key == \ records_data_with_external_pids['external_pids'][0]['key']: f.delete(f.obj.bucket, f.obj.key) output = generate_external_pids(deposit) expected_output_sorted = \ records_data_with_external_pids['external_pids'][1:] expected_output_sorted.sort(key=lambda f: f['key']) assert output == expected_output_sorted records_data_with_external_pids['external_pids'].reverse() deposit2 = create_deposit(records_data_with_external_pids, test_users['deposits_creator']) output = generate_external_pids(deposit2) assert output == expected_output
def test_missing_field(field): with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] draft_patch_res = client.patch( url_for('b2share_deposit_rest.b2dep_item', pid_value=deposit.pid.pid_value), data=json.dumps([{ "op": "remove", "path": '/' + field, }, { "op": "replace", "path": "/publication_state", "value": PublicationStates.submitted.name }]), headers=headers) assert draft_patch_res.status_code == 400 draft_patch_error = json.loads( draft_patch_res.get_data(as_text=True)) assert draft_patch_error['message'] == "Validation error." assert draft_patch_error['errors'][0]['field'] == field return draft_patch_error['errors'][0]['message']
def test_change_deposit_schema_fails(app, draft_deposits): """Test updating the $schema field fails.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].id) del deposit['$schema'] with pytest.raises(AlteredRecordError): deposit.commit()
def test_deposit_create(app, draft_deposits): """Test deposit creation.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) assert (deposit['publication_state'] == PublicationStates.draft.name) assert (deposit['_deposit']['status'] == 'draft')
def patch(self, *args, **kwargs): """PATCH the deposit.""" pid, record = request.view_args['pid_value'].data result = super(DepositResource, self).patch(*args, **kwargs) record = Deposit.get_record(record['_deposit']['id']) self._index_record(record) return result
def test_change_deposit_schema_fails(app, draft_deposits): """Test updating the $schema field fails.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) del deposit['$schema'] with pytest.raises(AlteredRecordError): deposit.commit()
def test_deposit_patch_immutable_fields(app, draft_deposits, test_users, login_user): """Test invalid modification of record draft with HTTP PATCH.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] for path in IMMUTABLE_PATHS: for command in [ {"op": "replace", "path": path, "value": ""}, {"op": "remove", "path": path}, {"op": "add", "path": path, "value": ""}, {"op": "copy", "from": "/title", "path": path, "value": ""}, {"op": "move", "from": "/title", "path": path, "value": ""}, ]: draft_patch_res = client.patch( url_for('b2share_deposit_rest.b2dep_item', pid_value=deposit.pid.pid_value), data=json.dumps([command]), headers=headers) assert draft_patch_res.status_code == 400
def test_generation_of_external_pids(app, records_data_with_external_pids, deposit_with_external_pids, test_users): """Test the generate_external_pids function.""" expected_output = records_data_with_external_pids[ 'external_pids'][:] with app.app_context(): deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) output = generate_external_pids(deposit) assert output == expected_output all_files = list(deposit.files) for f in all_files: if f.obj.key == \ records_data_with_external_pids['external_pids'][0]['key']: f.delete(f.obj.bucket, f.obj.key) output = generate_external_pids(deposit) expected_output_sorted = \ records_data_with_external_pids['external_pids'][1:] expected_output_sorted.sort(key=lambda f: f['key']) assert output == expected_output_sorted records_data_with_external_pids['external_pids'].reverse() deposit2 = create_deposit(records_data_with_external_pids, test_users['deposits_creator']) output = generate_external_pids(deposit2) assert output == expected_output
def test_deposit_patch_immutable_fields(app, draft_deposits, test_users, login_user): """Test invalid modification of record draft with HTTP PATCH.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].id) with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] for path in IMMUTABLE_PATHS: for command in [ {"op": "replace", "path": path, "value": ""}, {"op": "remove", "path": path}, {"op": "add", "path": path, "value": ""}, {"op": "copy", "from": "/title", "path": path, "value": ""}, {"op": "move", "from": "/title", "path": path, "value": ""}, ]: draft_patch_res = client.patch( url_for('b2share_deposit_rest.b2dep_item', pid_value=deposit.pid.pid_value), data=json.dumps([command]), headers=headers) assert draft_patch_res.status_code == 400
def process_v1_record(directory, indexer, base_url, logfile): """ Parse a downloaded file containing records """ with open(os.path.join(directory, '___record___.json'), 'r') as f: file_content = f.read() record_json = json.loads(file_content) recid = str(record_json.get('record_id')) if not record_json.get('domain'): click.secho('Record {} "{}" has no domain, '.format(recid, record_json.get('title')), fg='red') logfile.write("\n********************\n") logfile.write("\nERROR: record {} has no domain, is in limbo\n".format(recid)) logfile.write("\n********************\n") click.secho('Processing record {} "{}"'.format(recid, record_json.get('title'))) record = _process_record(record_json) if record is not None: user = get_or_create_user(record_json['uploaded_by']) with current_app.test_request_context('/', base_url=base_url): current_app.login_manager.reload_user(user) try: deposit = Deposit.create(record) _create_bucket(deposit, record_json, directory, logfile) deposit.publish() _, record = deposit.fetch_published() # index the record indexer.index(record) db.session.commit() except: logfile.write("\n********************") logfile.write("\nERROR while creating record {}\n".format(recid)) logfile.write(traceback.format_exc()) logfile.write("\n********************") click.secho("Finished processing {}".format(record['titles'][0]['title']))
def test_record_publish_adds_no_handles_for_external_files(app, records_data_with_external_pids, test_records_data): """Test that no handle PIDs are created for external files.""" for metadata in test_records_data: with app.app_context(): app.config.update({'FAKE_EPIC_PID': True}) external_pids = records_data_with_external_pids['external_pids'] external_dict = {x['key']: x['ePIC_PID'] for x in external_pids} data = deepcopy(metadata) data['external_pids'] = deepcopy(external_pids) record_uuid = uuid.uuid4() b2share_deposit_uuid_minter(record_uuid, data=data) deposit = Deposit.create(data, id_=record_uuid) ObjectVersion.create(deposit.files.bucket, 'real_file_1.txt', stream=BytesIO(b'mycontent')) ObjectVersion.create(deposit.files.bucket, 'real_file_2.txt', stream=BytesIO(b'mycontent')) deposit.submit() deposit.publish() deposit.commit() _, record = deposit.fetch_published() # external files don't get a handle PID, they already have one # which is stored in record['_deposit']['external_pids'] for f in record.files: if f['key'] in external_dict: assert f.get('ePIC_PID') is None else: assert '0000' in f['ePIC_PID'] # is a new fake PID
def test_deposit_update_unknown_publication_state(app, draft_deposits): """Test deposit submission by updating the "publication_state" field.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].id) deposit.update({'publication_state': 'invalid_state'}) with pytest.raises(InvalidPublicationStateError): deposit.commit()
def test_deposit_create(app, draft_deposits): """Test deposit creation.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].id) assert (deposit['publication_state'] == PublicationStates.draft.name) assert (deposit['_deposit']['status'] == 'draft')
def test_missing_field(field): with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] draft_patch_res = client.patch( url_for('b2share_deposit_rest.b2dep_item', pid_value=deposit.pid.pid_value), data=json.dumps([{ "op": "remove", "path": '/'+field, }, { "op": "replace", "path": "/publication_state", "value": PublicationStates.submitted.name }]), headers=headers) assert draft_patch_res.status_code == 400 draft_patch_error = json.loads( draft_patch_res.get_data(as_text=True)) assert draft_patch_error['message'] == "Validation error." assert draft_patch_error['errors'][0]['field'] == field return draft_patch_error['errors'][0]['message']
def test_adding_external_files(app, records_data_with_external_pids, deposit_with_external_pids): """Test the addition of external files.""" with app.app_context(): # renaming a file to have a duplicate key records_data_with_external_pids['external_pids'][0]['key'] = \ 'file2.txt' deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) # this should be caught and return an InvalidDepositError with pytest.raises(InvalidDepositError): deposit = deposit.patch([ {'op': 'replace', 'path': '/external_pids', 'value': records_data_with_external_pids['external_pids']} ]) deposit.commit() with app.app_context(): records_data_with_external_pids['external_pids'][0]['key'] = \ 'file1.txt' records_data_with_external_pids['external_pids'].append({ "key": "file3.txt", "ePIC_PID": "http://hdl.handle.net/11304/0d8dbdec-74e4-4774-954e-1a98e5c0cfa3" }) deposit = deposit.patch([ {'op': 'replace', 'path': '/external_pids', 'value': records_data_with_external_pids['external_pids']} ]) deposit.commit() assert_external_files(deposit, records_data_with_external_pids['external_pids'])
def test_deposit_update_unknown_publication_state(app, draft_deposits): """Test deposit submission by updating the "publication_state" field.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) deposit.update({'publication_state': 'invalid_state'}) with pytest.raises(InvalidPublicationStateError): deposit.commit()
def test_deposit_submit(app, draft_deposits): """Test deposit submission.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].id) deposit.submit() assert ( deposit['publication_state'] == PublicationStates.submitted.name) assert (deposit['_deposit']['status'] == 'draft')
def test_deposit_submit(app, draft_deposits): """Test deposit submission.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) deposit.submit() assert (deposit['publication_state'] == PublicationStates.submitted.name) assert (deposit['_deposit']['status'] == 'draft')
def create_deposit(data, creator, files=None): """Create a deposit with the given user as creator.""" with authenticated_user(creator): deposit = Deposit.create(data=deepcopy(data)) if files is not None: for key, value in files.items(): deposit.files[key] = BytesIO(value) return deposit
def test_deposit_submit(app, test_records_data, draft_deposits, test_users, login_user): """Test record draft submit with HTTP PATCH.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) record_data = test_records_data[0] with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] draft_patch_res = client.patch(url_for( 'b2share_deposit_rest.b2dep_item', pid_value=deposit.pid.pid_value), data=json.dumps([{ "op": "replace", "path": "/publication_state", "value": PublicationStates.submitted.name }]), headers=headers) assert draft_patch_res.status_code == 200 draft_patch_data = json.loads( draft_patch_res.get_data(as_text=True)) expected_metadata = build_expected_metadata( record_data, PublicationStates.draft.name, owners=[user.id], draft=True, ) with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) expected_metadata['publication_state'] = \ PublicationStates.submitted.name assert expected_metadata == draft_patch_data['metadata'] assert (deposit['publication_state'] == PublicationStates.submitted.name) subtest_self_link(draft_patch_data, draft_patch_res.headers, client)
def test_deposit_publish(app, draft_deposits): """Test deposit submission by updating the "publication_state" field.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].id) deposit.submit() deposit.publish() assert ( deposit['publication_state'] == PublicationStates.published.name) assert (deposit['_deposit']['status'] == 'published')
def test_deposit_create_with_incomplete_metadata(app, test_incomplete_records_data): """Test deposit creation with incomplete metadata succeeds.""" with app.app_context(): for data in test_incomplete_records_data: deposit = Deposit.create(data.incomplete_data) assert ( deposit['publication_state'] == PublicationStates.draft.name) assert (deposit['_deposit']['status'] == 'draft')
def test_deposit_update_and_submit(app, draft_deposits): """Test deposit submission by updating the "publication_state" field.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].id) deposit.update({'publication_state': PublicationStates.submitted.name}) deposit.commit() assert ( deposit['publication_state'] == PublicationStates.submitted.name) assert (deposit['_deposit']['status'] == 'draft')
def test_deposit_publish(app, draft_deposits): """Test deposit submission by updating the "publication_state" field.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) deposit.submit() deposit.publish() assert (deposit['publication_state'] == PublicationStates.published.name) assert (deposit['_deposit']['status'] == 'published')
def test_deposit_update_and_submit(app, draft_deposits): """Test deposit submission by updating the "publication_state" field.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) deposit.update({'publication_state': PublicationStates.submitted.name}) deposit.commit() assert (deposit['publication_state'] == PublicationStates.submitted.name) assert (deposit['_deposit']['status'] == 'draft')
def test_deposit_submit(app, test_records_data, draft_deposits, test_users, login_user): """Test record draft submit with HTTP PATCH.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) record_data = test_records_data[0] with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] draft_patch_res = client.patch( url_for('b2share_deposit_rest.b2dep_item', pid_value=deposit.pid.pid_value), data=json.dumps([{ "op": "replace", "path": "/publication_state", "value": PublicationStates.submitted.name }]), headers=headers) assert draft_patch_res.status_code == 200 draft_patch_data = json.loads( draft_patch_res.get_data(as_text=True)) expected_metadata = build_expected_metadata( record_data, PublicationStates.draft.name, owners=[user.id], draft=True, ) with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) expected_metadata['publication_state'] = \ PublicationStates.submitted.name assert expected_metadata == draft_patch_data['metadata'] assert (deposit['publication_state'] == PublicationStates.submitted.name) subtest_self_link(draft_patch_data, draft_patch_res.headers, client)
def create(data): data = deepcopy(data) record_uuid = uuid.uuid4() # Create persistent identifier b2share_deposit_uuid_minter(record_uuid, data=data) deposit = Deposit.create(data=data, id_=record_uuid, version_of=version_of) if files is not None: for key, value in files.items(): deposit.files[key] = BytesIO(value) return deposit
def test_deposit_submit_with_incomplete_metadata(app, test_incomplete_records_data): """Test deposit submission with incomplete metadata fails.""" for data in test_incomplete_records_data: with app.app_context(): deposit = Deposit.create(data.complete_data) deposit.commit() # make the data incomplete deposit = deposit.patch(data.patch) with pytest.raises(ValidationError): deposit.submit()
def create_deposits(app, test_records_data, creator): """Create test deposits.""" DepositInfo = namedtuple('DepositInfo', ['id', 'data', 'deposit']) with authenticated_user(creator): deposits = [Deposit.create(data=data) for data in deepcopy(test_records_data)] for deposit in deposits: deposit.commit() deposit.commit() return [DepositInfo(dep.id, dep.dumps(), dep) for dep in deposits]
def test_modify_external_files(app, deposit_with_external_pids, new_external_pids): """Test changing PID, renaming and deletion of external files.""" with app.app_context(): deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) deposit = deposit.patch([ {'op': 'replace', 'path': '/external_pids', 'value': new_external_pids} ]) deposit.commit() assert_external_files(deposit, new_external_pids)
def test_modify_external_files(app, deposit_with_external_pids, new_external_pids): """Test changing PID, renaming and deletion of external files.""" with app.app_context(): deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) deposit = deposit.patch([{ 'op': 'replace', 'path': '/external_pids', 'value': new_external_pids }]) deposit.commit() assert_external_files(deposit, new_external_pids)
def create_deposits(app, test_records_data, creator): """Create test deposits.""" DepositInfo = namedtuple( 'DepositInfo', [ 'deposit_id', 'data', 'deposit', # FIXME: replaced by get_deposit, remove it later 'get_deposit' ]) deposits = [] with authenticated_user(creator): for data in deepcopy(test_records_data): record_uuid = uuid.uuid4() # Create persistent identifier b2share_deposit_uuid_minter(record_uuid, data=data) deposits.append(B2ShareDeposit.create(data=data, id_=record_uuid)) return [DepositInfo( dep.id, dep.dumps(), dep, (lambda id: lambda: B2ShareDeposit.get_record(id))(dep.id) ) for dep in deposits]
def test_deposit_publish_with_incomplete_metadata( app, test_incomplete_records_data): """Test publication of an incomplete deposit fails.""" for data in test_incomplete_records_data: with app.app_context(): deposit = Deposit.create(data.complete_data) deposit.submit() deposit.commit() # make the data incomplete deposit = deposit.patch(data.patch) with pytest.raises(ValidationError): deposit.publish()
def create_deposits(app, test_records_data, creator): """Create test deposits.""" DepositInfo = namedtuple('DepositInfo', ['deposit_id', 'data', 'deposit']) deposits = [] with authenticated_user(creator): for data in deepcopy(test_records_data): record_uuid = uuid.uuid4() # Create persistent identifier b2share_deposit_uuid_minter(record_uuid, data=data) deposits.append(Deposit.create(data=data, id_=record_uuid)) return [DepositInfo(dep.id, dep.dumps(), dep) for dep in deposits]
def test_new_deposit_versions_preserve_schema(app, test_records, test_users): """Creating new versions of existing records.""" with app.app_context(): # Retrieve a record which will be the first version, and its deposit v1_rec = B2ShareRecord.get_record(test_records[0].record_id) _, v1_id = pid_of(test_records[0].data) # update the community's schema community_id = v1_rec.model.json['community'] old_schema = CommunitySchema.get_community_schema(community_id) json_schema = json.loads(old_schema.model.community_schema) new_schema = CommunitySchema.create_version(community_id, json_schema) assert new_schema.version > old_schema.version # create new, unversioned, draft and records data = copy_data_from_previous(v1_rec.model.json) unver_dep = create_deposit(data, test_users['deposits_creator']) unver_dep.submit() unver_dep.publish() _, unver_rec = unver_dep.fetch_published() # the unversioned draft or record in a version chain have the updated schema assert unver_dep['$schema'] != Deposit._build_deposit_schema( v1_rec.model.json) assert unver_rec.model.json['$schema'] != v1_rec.model.json['$schema'] # create new draft and record in the version chain of v1 data = copy_data_from_previous(v1_rec.model.json) v2_dep = create_deposit(data, test_users['deposits_creator'], version_of=v1_id) v2_dep.submit() v2_dep.publish() _, v2_rec = v2_dep.fetch_published() # the new draft and record in a version chain preserve the version of the schema assert v2_dep['$schema'] == Deposit._build_deposit_schema( v1_rec.model.json) assert v2_rec.model.json['$schema'] == v1_rec.model.json['$schema']
def test_record_commit_with_incomplete_metadata(app, test_incomplete_records_data): """Test commit of an incomplete record fails.""" for data in test_incomplete_records_data: with app.app_context(): deposit = Deposit.create(data.complete_data) deposit.submit() deposit.publish() deposit.commit() pid, record = deposit.fetch_published() # make the data incomplete record = record.patch(data.patch) with pytest.raises(ValidationError): record.commit()
def create_deposits(app, test_records_data, creator): """Create test deposits.""" DepositInfo = namedtuple( 'DepositInfo', [ 'deposit_id', 'data', 'deposit', # FIXME: replaced by get_deposit, remove it later 'get_deposit' ]) deposits = [] with authenticated_user(creator): for data in deepcopy(test_records_data): record_uuid = uuid.uuid4() # Create persistent identifier b2share_deposit_uuid_minter(record_uuid, data=data) deposits.append(B2ShareDeposit.create(data=data, id_=record_uuid)) return [ DepositInfo(dep.id, dep.dumps(), dep, (lambda id: lambda: B2ShareDeposit.get_record(id))(dep.id)) for dep in deposits ]
def test_deposit_put_is_disabled(app, draft_deposits, test_users, login_user): """Test invalid modification of record draft with HTTP PUT.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) headers = [('Content-Type', 'application/json'), ('Accept', 'application/json')] draft_put_res = client.put(url_for( 'b2share_deposit_rest.b2dep_item', pid_value=deposit.pid.pid_value), data='{}', headers=headers) assert draft_put_res.status_code == 405
def test_record_commit_with_incomplete_metadata(app, test_incomplete_records_data): """Test commit of an incomplete record fails.""" for metadata in test_incomplete_records_data: with app.app_context(): data = deepcopy(metadata.complete_data) record_uuid = uuid.uuid4() b2share_deposit_uuid_minter(record_uuid, data=data) deposit = Deposit.create(data, id_=record_uuid) deposit.submit() deposit.publish() deposit.commit() pid, record = deposit.fetch_published() # make the data incomplete record = record.patch(metadata.patch) with pytest.raises(ValidationError): record.commit()
def test_deposit_put_is_disabled(app, draft_deposits, test_users, login_user): """Test invalid modification of record draft with HTTP PUT.""" with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) with app.test_client() as client: user = test_users['deposits_creator'] login_user(user, client) headers = [('Content-Type', 'application/json'), ('Accept', 'application/json')] draft_put_res = client.put( url_for('b2share_deposit_rest.b2dep_item', pid_value=deposit.pid.pid_value), data='{}', headers=headers) assert draft_put_res.status_code == 405
def test_missing_handle_prefix(app, test_users, login_user, records_data_with_external_pids, deposit_with_external_pids): """Test external files that are registered without a handle prefix.""" with app.app_context(): no_prefix_external_pid = \ [{"key": "no_prefix.txt", "ePIC_PID": "11304/0d8dbdec-74e4-4774-954e-1a98e5c0cfa3"}] correct_external_pid = \ [{"key": "no_prefix.txt", "ePIC_PID": "http://hdl.handle.net/11304/0d8dbdec-74e4-4774-954e-1a98e5c0cfa3"}] deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) deposit = deposit.patch([ {'op': 'replace', 'path': '/external_pids', 'value': no_prefix_external_pid} ]) deposit.commit() assert_external_files(deposit, correct_external_pid) headers = [('Content-Type', 'application/json'), ('Accept', 'application/json')] def create_record(client, record_data): return client.post( url_for('b2share_records_rest.b2rec_list'), data=json.dumps(record_data), headers=headers ) with app.app_context(): with app.test_client() as client: user = test_users['normal'] login_user(user, client) # create the deposit with the missing prefix records_data_with_external_pids['external_pids'] = \ no_prefix_external_pid draft_create_res = create_record( client, records_data_with_external_pids) assert draft_create_res.status_code == 201 draft_create_data = json.loads( draft_create_res.get_data(as_text=True)) # assert that when returned the external pid has the prefix assert draft_create_data['metadata']['external_pids'] == \ correct_external_pid
def test_embargoed_records_with_external_pids(app, test_users, login_user, deposit_with_external_pids): """Test that embargoed records handle external pids correctly.""" with app.app_context(): deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) deposit = deposit.patch([{ 'op': 'add', 'path': '/embargo_date', 'value': (datetime.utcnow() + timedelta(days=1)).isoformat() }]) deposit = deposit.patch([{ 'op': 'replace', 'path': '/open_access', 'value': False }]) deposit.commit() deposit.submit() deposit.publish() with app.app_context(): with app.test_client() as client: # anonymous user tries to download the external file for file in deposit.files: ext_file_url = url_for('invenio_files_rest.object_api', bucket_id=file.obj.bucket, key=file.obj.key) resp = client.get(ext_file_url) # 404 assert resp.status_code == 404 # test that when getting the record the external_pids field # is not visible in the metadata with app.app_context(): with app.test_client() as client: # the owner can download the external file login_user(test_users['deposits_creator'], client) for file in deposit.files: ext_file_url = url_for('invenio_files_rest.object_api', bucket_id=file.obj.bucket, key=file.obj.key) resp = client.get(ext_file_url) assert resp.status_code == 302
def test_embargoed_records_with_external_pids(app, test_users, login_user, deposit_with_external_pids): """Test that embargoed records handle external pids correctly.""" with app.app_context(): deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) deposit = deposit.patch([ {'op': 'add', 'path': '/embargo_date', 'value': (datetime.utcnow() + timedelta(days=1)).isoformat()} ]) deposit = deposit.patch([ {'op': 'replace', 'path': '/open_access', 'value': False} ]) deposit.commit() deposit.submit() deposit.publish() with app.app_context(): with app.test_client() as client: # anonymous user tries to download the external file for file in deposit.files: ext_file_url = url_for('invenio_files_rest.object_api', bucket_id=file.obj.bucket, key=file.obj.key) resp = client.get(ext_file_url) # 404 assert resp.status_code == 404 # test that when getting the record the external_pids field # is not visible in the metadata with app.app_context(): with app.test_client() as client: # the owner can download the external file login_user(test_users['deposits_creator'], client) for file in deposit.files: ext_file_url = url_for('invenio_files_rest.object_api', bucket_id=file.obj.bucket, key=file.obj.key) resp = client.get(ext_file_url) assert resp.status_code == 302
def test_deposit_add_unknown_fields(app, draft_deposits): """Test adding unknown fields in deposit. It should fail.""" for path in [ # all the following paths point to "object" fields in # in the root JSON Schema '/new_field', '/community_specific/new_field', '/creators/0/new_field', '/titles/0/new_field', '/contributors/0/new_field', '/resource_types/0/new_field', '/alternate_identifiers/0/new_field', '/descriptions/0/new_field', '/license/new_field', ]: with app.app_context(): deposit = Deposit.get_record(draft_deposits[0].deposit_id) deposit = deposit.patch([ {'op': 'add', 'path': path, 'value': 'any value'} ]) with pytest.raises(ValidationError): deposit.commit()
def test_getting_record_with_external_pids(app, login_user, test_users, deposit_with_external_pids, records_data_with_external_pids): """External pids are serialized in the metadata when it is allowed.""" def test_get_deposit(deposit_pid_value, user): with app.test_client() as client: login_user(user, client) deposit_url = url_for('b2share_deposit_rest.b2dep_item', pid_value=deposit_pid_value) resp = client.get(deposit_url) deposit_data = json.loads( resp.get_data(as_text=True)) return deposit_data def test_get_record(record_pid_value, user): with app.test_client() as client: login_user(user, client) record_url = url_for('b2share_records_rest.b2rec_item', pid_value=record_pid_value) resp = client.get(record_url) record_data = json.loads( resp.get_data(as_text=True)) return record_data def test_search_deposits(user): with app.test_client() as client: login_user(user, client) search_deposits_url = url_for( 'b2share_records_rest.b2rec_list', drafts=1, size=100) headers = [('Content-Type', 'application/json'), ('Accept', 'application/json')] resp = client.get( search_deposits_url, headers=headers) deposit_search_res = json.loads( resp.get_data(as_text=True)) return deposit_search_res def test_search_records(user): with app.test_client() as client: login_user(user, client) search_records_url = url_for( 'b2share_records_rest.b2rec_list', size=100) headers = [('Content-Type', 'application/json'), ('Accept', 'application/json')] resp = client.get( search_records_url, headers=headers) record_search_res = json.loads( resp.get_data(as_text=True)) return record_search_res with app.app_context(): deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) with app.app_context(): deposit_data = test_get_deposit(deposit.pid.pid_value, test_users['deposits_creator']) # assert that the external_pids are visible # when getting a specific deposit assert_external_files( deposit, deposit_data['metadata']['external_pids']) current_search_client.indices.refresh('*') deposit_search_data = test_search_deposits( test_users['deposits_creator']) assert deposit_search_data['hits']['total'] == 1 # external_pids are not shown in a deposit search because it would use # too much resources to generate it for each search hit. assert 'external_pids' not in deposit_search_data[ 'hits']['hits'][0]['metadata'] with app.app_context(): deposit = Deposit.get_record(deposit_with_external_pids.deposit_id) deposit.submit() deposit.publish() record_resolver = Resolver( pid_type='b2rec', object_type='rec', getter=B2ShareRecord.get_record, ) record_pid, record = record_resolver.resolve(deposit.pid.pid_value) current_search_client.indices.refresh('*') record_data = test_get_record( record_pid.pid_value, test_users['deposits_creator']) # when getting a specific record the owner sees the external_pids assert_external_files(record, record_data['metadata']['external_pids']) with app.app_context(): record_data = test_get_record( record_pid.pid_value, test_users['normal']) # and all other users as well if it is open access assert_external_files(record, record_data['metadata']['external_pids']) deposit_search_data = test_search_deposits( test_users['deposits_creator']) assert deposit_search_data['hits']['total'] == 1 # external_pids are not shown in deposit search even when published assert 'external_pids' not in deposit_search_data[ 'hits']['hits'][0]['metadata'] record_search_data = test_search_records( test_users['deposits_creator']) assert record_search_data['hits']['total'] == 1 # external_pids are shown for record search if they are open access # for all users assert 'external_pids' in record_search_data[ 'hits']['hits'][0]['metadata'] record_search_data = test_search_records(test_users['normal']) assert record_search_data['hits']['total'] == 1 assert 'external_pids' in record_search_data[ 'hits']['hits'][0]['metadata'] with app.app_context(): deposit2 = create_deposit(records_data_with_external_pids, test_users['deposits_creator']) deposit2 = deposit2.patch([ {'op': 'add', 'path': '/embargo_date', 'value': (datetime.utcnow() + timedelta(days=1)).isoformat()}, {'op': 'replace', 'path': '/open_access', 'value': False} ]) deposit2.commit() deposit2.submit() deposit2.publish() record_resolver = Resolver( pid_type='b2rec', object_type='rec', getter=B2ShareRecord.get_record, ) record_pid, record = record_resolver.resolve(deposit2.pid.pid_value) record_data = test_get_record( record_pid.pid_value, test_users['deposits_creator']) # owners of records have access to files and # external_pids of embargoed records assert_external_files(record, record_data['metadata']['external_pids']) with app.app_context(): record_data = test_get_record( record_pid.pid_value, test_users['normal']) # normal users shouldn't have access to the # files and external_pids of an embargoed record assert 'metadata' in record_data assert 'external_pids' not in record_data['metadata']
def alembic_upgrade_database_data(alembic, verbose): """Migrate the database data from v2.0.0 to 2.1.0.""" ### Add versioning PIDs ### # Reserve the record PID and versioning PID for unpublished deposits # Hack: disable record indexing during record migration from invenio_indexer.api import RecordIndexer old_index_fn = RecordIndexer.index RecordIndexer.index = lambda s, record: None if verbose: click.secho('migrating deposits and records...') with db.session.begin_nested(): # Migrate published records records_pids = PersistentIdentifier.query.filter( PersistentIdentifier.pid_type == RecordUUIDProvider.pid_type, PersistentIdentifier.status == PIDStatus.REGISTERED, ).all() for rec_pid in records_pids: if verbose: click.secho(' record {}'.format(rec_pid.pid_value)) try: record = Record.get_record(rec_pid.object_uuid) except NoResultFound: # The record is deleted but not the PID. Fix it. rec_pid.status = PIDStatus.DELETED continue # Create parent version PID parent_pid = RecordUUIDProvider.create().pid version_master = PIDVersioning(parent=parent_pid) version_master.insert_draft_child(child=rec_pid) version_master.update_redirect() migrate_record_metadata( Record.get_record(rec_pid.object_uuid), parent_pid ) # Migrate deposits deposit_pids = PersistentIdentifier.query.filter( PersistentIdentifier.pid_type == DepositUUIDProvider.pid_type, PersistentIdentifier.status == PIDStatus.REGISTERED, ).all() for dep_pid in deposit_pids: if verbose: click.secho(' deposit {}'.format(dep_pid.pid_value)) try: deposit = Deposit.get_record(dep_pid.object_uuid) if deposit['publication_state'] != \ PublicationStates.published.name: # The record is not published yet. Reserve the PID. rec_pid = RecordUUIDProvider.create( object_type='rec', pid_value=dep_pid.pid_value, ).pid # Create parent version PID parent_pid = RecordUUIDProvider.create().pid assert parent_pid version_master = PIDVersioning(parent=parent_pid) version_master.insert_draft_child(child=rec_pid) else: # Retrieve previously created version PID rec_pid = RecordUUIDProvider.get(dep_pid.pid_value).pid version_master = PIDVersioning(child=rec_pid) parent_pid = version_master.parent if not parent_pid: click.secho(' record {} was deleted, but the deposit has not been removed'.format(rec_pid.pid_value), fg='red') if parent_pid: migrate_record_metadata( Deposit.get_record(dep_pid.object_uuid), parent_pid ) except NoResultFound: # The deposit is deleted but not the PID. Fix it. dep_pid.status = PIDStatus.DELETED if verbose: click.secho('done migrating deposits.') RecordIndexer.index = old_index_fn
def test_deposit_publish(app, test_users, test_communities, login_user): """Test record draft publication with HTTP PATCH.""" with app.app_context(): community_name = 'MyTestCommunity1' creator = test_users['deposits_creator'] record_data = generate_record_data(community=community_name) community = Community.get(name=community_name) com_admin = create_user('com_admin', roles=[community.admin_role]) deposit = create_deposit(record_data, creator) deposit_id = deposit.id deposit.submit() db.session.commit() with app.test_client() as client: login_user(com_admin, client) headers = [('Content-Type', 'application/json-patch+json'), ('Accept', 'application/json')] draft_patch_res = client.patch( url_for('b2share_deposit_rest.b2dep_item', pid_value=deposit.pid.pid_value), data=json.dumps([{ "op": "replace", "path": "/publication_state", "value": PublicationStates.published.name }]), headers=headers) assert draft_patch_res.status_code == 200 draft_patch_data = json.loads( draft_patch_res.get_data(as_text=True)) expected_metadata = build_expected_metadata( record_data, PublicationStates.published.name, owners=[creator.id], draft=True, PID=draft_patch_data['metadata'].get('ePIC_PID'), DOI=draft_patch_data['metadata'].get('DOI'), ) with app.app_context(): deposit = Deposit.get_record(deposit_id) with app.test_client() as client: login_user(creator, client) assert expected_metadata == draft_patch_data['metadata'] assert (deposit['publication_state'] == PublicationStates.published.name) subtest_self_link(draft_patch_data, draft_patch_res.headers, client) pid, published = deposit.fetch_published() # check that the published record and the deposit are equal except # for the schema cleaned_deposit = {f: v for f, v in deposit.items() if f != '$schema'} cleaned_published = {f: v for f, v in deposit.items() if f != '$schema'} assert cleaned_published == cleaned_deposit # check "published" link assert draft_patch_data['links']['publication'] == \ url_for('b2share_records_rest.{0}_item'.format( RecordUUIDProvider.pid_type ), pid_value=pid.pid_value, _external=True) # check that the published record content match the deposit headers = [('Accept', 'application/json')] self_response = client.get( draft_patch_data['links']['publication'], headers=headers ) assert self_response.status_code == 200 published_data = json.loads(self_response.get_data( as_text=True)) # we don't want to compare the links and dates cleaned_published_data = deepcopy(published_data) # the published record has an extra empty 'files' array assert cleaned_published_data['files'] == [] del cleaned_published_data['files'] cleaned_draft_data = deepcopy(draft_patch_data) for item in [cleaned_published_data, cleaned_draft_data]: del item['links'] del item['created'] del item['updated'] del item['metadata']['$schema'] assert cleaned_draft_data == cleaned_published_data