def _create_record_from_filepath(path, rec_uuid, indexer, versions, verbose):
with open(path) as record_file:
record_str = record_file.read()
record_str = resolve_community_id(record_str)
record_str = resolve_block_schema_id(record_str)
json_data = json.loads(record_str)
b2share_deposit_uuid_minter(rec_uuid, data=json_data)
deposit = Deposit.create(json_data, id_=rec_uuid)
ObjectVersion.create(deposit.files.bucket,
'myfile',
stream=BytesIO(b'mycontent'))
deposit.publish()
pid, record = deposit.fetch_published()
indexer.index(record)
if verbose > 0:
click.secho('created new record: {}'.format(str(rec_uuid)))
last_id = pid.pid_value
for i in range(2 * versions):
rec_uuid = uuid4()
json_data = json.loads(record_str)
b2share_deposit_uuid_minter(rec_uuid, data=json_data)
deposit2 = Deposit.create(json_data, id_=rec_uuid, version_of=last_id)
ObjectVersion.create(deposit2.files.bucket,
'myfile-ver{}'.format(i),
stream=BytesIO(b'mycontent'))
deposit2.publish()
pid, record2 = deposit2.fetch_published()
indexer.index(record2)
last_id = pid.pid_value
if verbose > 0:
click.secho('created new version: {}'.format(str(rec_uuid)))
return record, deposit
def _create_record_from_filepath(path, rec_uuid, indexer, versions, verbose):
with open(path) as record_file:
record_str = record_file.read()
record_str = resolve_community_id(record_str)
record_str = resolve_block_schema_id(record_str)
json_data = json.loads(record_str)
b2share_deposit_uuid_minter(rec_uuid, data=json_data)
deposit = Deposit.create(json_data, id_=rec_uuid)
ObjectVersion.create(deposit.files.bucket, 'myfile',
stream=BytesIO(b'mycontent'))
deposit.publish()
pid, record = deposit.fetch_published()
indexer.index(record)
if verbose > 0:
click.secho('created new record: {}'.format(str(rec_uuid)))
last_id = pid.pid_value
for i in range(2*versions):
rec_uuid = uuid4()
json_data = json.loads(record_str)
b2share_deposit_uuid_minter(rec_uuid, data=json_data)
deposit2 = Deposit.create(json_data, id_=rec_uuid,
version_of=last_id)
ObjectVersion.create(deposit2.files.bucket, 'myfile-ver{}'.format(i),
stream=BytesIO(b'mycontent'))
deposit2.publish()
pid, record2 = deposit2.fetch_published()
indexer.index(record2)
last_id = pid.pid_value
if verbose > 0:
click.secho('created new version: {}'.format(str(rec_uuid)))
return record, deposit
def test_deposit_create_with_invalid_fields_fails(app, test_records_data):
"""Test deposit creation without or with an invalid field fails."""
data = test_records_data[0]
with app.app_context():
data['publication_state'] = 'published'
with pytest.raises(InvalidDepositError):
deposit = Deposit.create(deepcopy(data))
with app.app_context():
data['$schema'] = '__garbage__'
with pytest.raises(InvalidDepositError):
deposit = Deposit.create(deepcopy(data))
def test_unpublished_deposit_unindex(app, test_users, draft_deposits,
script_info, login_user):
"""Check that deleting an unpublished deposit also removes it from the search index."""
creator = test_users['deposits_creator']
with app.app_context():
Deposit.get_record(draft_deposits[0].deposit_id).delete()
# execute scheduled tasks synchronously
process_bulk_queue.delay()
# flush the indices so that indexed records are searchable
current_search_client.indices.flush('*')
# deleted record should not be searchable
subtest_record_search(app, creator, [], draft_deposits[1:], login_user)
def test_published_deposit_unindex(app, test_users, test_records, script_info,
login_user):
"""Check that deleting a published deposit also removes it from the search index."""
creator = test_users['deposits_creator']
with app.app_context():
Deposit.get_record(test_records[0].deposit_id).delete()
# execute scheduled tasks synchronously
process_bulk_queue.delay()
# flush the indices so that indexed records are searchable
current_search_client.indices.flush('*')
# deleted record should not be searchable
subtest_record_search(app, creator, test_records, test_records[1:],
login_user)
def test_deposit_create_with_invalid_community_fails(app, test_records_data):
"""Test deposit creation without or with an invalid community fails."""
data = test_records_data[0]
with app.app_context():
# test with no community
del data['community']
with pytest.raises(ValidationError):
deposit = Deposit.create(deepcopy(data))
with app.app_context():
# test with an invalid community
data['community'] = str(uuid.uuid4())
with pytest.raises(InvalidDepositError):
deposit = Deposit.create(deepcopy(data))
def test_change_deposit_community(app, draft_deposits):
"""Test deposit creation without or with an invalid community fails."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
# test removing the community id
del deposit['community']
with pytest.raises(AlteredRecordError):
deposit.commit()
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
# test changing the community id
deposit['community'] = str(uuid.uuid4())
with pytest.raises(InvalidDepositError):
deposit.commit()
def test_change_deposit_community(app, draft_deposits):
"""Test deposit creation without or with an invalid community fails."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].id)
# test removing the community id
del deposit['community']
with pytest.raises(AlteredRecordError):
deposit.commit()
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].id)
# test changing the community id
deposit['community'] = str(uuid.uuid4())
with pytest.raises(InvalidDepositError):
deposit.commit()
def test_generation_of_external_pids(app, records_data_with_external_pids,
deposit_with_external_pids, test_users):
"""Test the generate_external_pids function."""
expected_output = records_data_with_external_pids['external_pids'][:]
with app.app_context():
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
output = generate_external_pids(deposit)
assert output == expected_output
all_files = list(deposit.files)
for f in all_files:
if f.obj.key == \
records_data_with_external_pids['external_pids'][0]['key']:
f.delete(f.obj.bucket, f.obj.key)
output = generate_external_pids(deposit)
expected_output_sorted = \
records_data_with_external_pids['external_pids'][1:]
expected_output_sorted.sort(key=lambda f: f['key'])
assert output == expected_output_sorted
records_data_with_external_pids['external_pids'].reverse()
deposit2 = create_deposit(records_data_with_external_pids,
test_users['deposits_creator'])
output = generate_external_pids(deposit2)
assert output == expected_output
def test_missing_field(field):
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
draft_patch_res = client.patch(
url_for('b2share_deposit_rest.b2dep_item',
pid_value=deposit.pid.pid_value),
data=json.dumps([{
"op": "remove",
"path": '/' + field,
}, {
"op": "replace",
"path": "/publication_state",
"value": PublicationStates.submitted.name
}]),
headers=headers)
assert draft_patch_res.status_code == 400
draft_patch_error = json.loads(
draft_patch_res.get_data(as_text=True))
assert draft_patch_error['message'] == "Validation error."
assert draft_patch_error['errors'][0]['field'] == field
return draft_patch_error['errors'][0]['message']
def test_change_deposit_schema_fails(app, draft_deposits):
"""Test updating the $schema field fails."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].id)
del deposit['$schema']
with pytest.raises(AlteredRecordError):
deposit.commit()
def test_deposit_create(app, draft_deposits):
"""Test deposit creation."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
assert (deposit['publication_state']
== PublicationStates.draft.name)
assert (deposit['_deposit']['status'] == 'draft')
def patch(self, *args, **kwargs):
"""PATCH the deposit."""
pid, record = request.view_args['pid_value'].data
result = super(DepositResource, self).patch(*args, **kwargs)
record = Deposit.get_record(record['_deposit']['id'])
self._index_record(record)
return result
def test_change_deposit_schema_fails(app, draft_deposits):
"""Test updating the $schema field fails."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
del deposit['$schema']
with pytest.raises(AlteredRecordError):
deposit.commit()
def test_deposit_patch_immutable_fields(app, draft_deposits, test_users,
login_user):
"""Test invalid modification of record draft with HTTP PATCH."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
for path in IMMUTABLE_PATHS:
for command in [
{"op": "replace", "path": path, "value": ""},
{"op": "remove", "path": path},
{"op": "add", "path": path, "value": ""},
{"op": "copy", "from": "/title", "path": path, "value": ""},
{"op": "move", "from": "/title", "path": path, "value": ""},
]:
draft_patch_res = client.patch(
url_for('b2share_deposit_rest.b2dep_item',
pid_value=deposit.pid.pid_value),
data=json.dumps([command]),
headers=headers)
assert draft_patch_res.status_code == 400
def test_generation_of_external_pids(app, records_data_with_external_pids,
deposit_with_external_pids, test_users):
"""Test the generate_external_pids function."""
expected_output = records_data_with_external_pids[
'external_pids'][:]
with app.app_context():
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
output = generate_external_pids(deposit)
assert output == expected_output
all_files = list(deposit.files)
for f in all_files:
if f.obj.key == \
records_data_with_external_pids['external_pids'][0]['key']:
f.delete(f.obj.bucket, f.obj.key)
output = generate_external_pids(deposit)
expected_output_sorted = \
records_data_with_external_pids['external_pids'][1:]
expected_output_sorted.sort(key=lambda f: f['key'])
assert output == expected_output_sorted
records_data_with_external_pids['external_pids'].reverse()
deposit2 = create_deposit(records_data_with_external_pids,
test_users['deposits_creator'])
output = generate_external_pids(deposit2)
assert output == expected_output
def test_deposit_patch_immutable_fields(app, draft_deposits, test_users,
login_user):
"""Test invalid modification of record draft with HTTP PATCH."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].id)
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
for path in IMMUTABLE_PATHS:
for command in [
{"op": "replace", "path": path, "value": ""},
{"op": "remove", "path": path},
{"op": "add", "path": path, "value": ""},
{"op": "copy", "from": "/title", "path": path, "value": ""},
{"op": "move", "from": "/title", "path": path, "value": ""},
]:
draft_patch_res = client.patch(
url_for('b2share_deposit_rest.b2dep_item',
pid_value=deposit.pid.pid_value),
data=json.dumps([command]),
headers=headers)
assert draft_patch_res.status_code == 400
def process_v1_record(directory, indexer, base_url, logfile):
"""
Parse a downloaded file containing records
"""
with open(os.path.join(directory, '___record___.json'), 'r') as f:
file_content = f.read()
record_json = json.loads(file_content)
recid = str(record_json.get('record_id'))
if not record_json.get('domain'):
click.secho('Record {} "{}" has no domain, '.format(recid, record_json.get('title')),
fg='red')
logfile.write("\n********************\n")
logfile.write("\nERROR: record {} has no domain, is in limbo\n".format(recid))
logfile.write("\n********************\n")
click.secho('Processing record {} "{}"'.format(recid, record_json.get('title')))
record = _process_record(record_json)
if record is not None:
user = get_or_create_user(record_json['uploaded_by'])
with current_app.test_request_context('/', base_url=base_url):
current_app.login_manager.reload_user(user)
try:
deposit = Deposit.create(record)
_create_bucket(deposit, record_json, directory, logfile)
deposit.publish()
_, record = deposit.fetch_published()
# index the record
indexer.index(record)
db.session.commit()
except:
logfile.write("\n********************")
logfile.write("\nERROR while creating record {}\n".format(recid))
logfile.write(traceback.format_exc())
logfile.write("\n********************")
click.secho("Finished processing {}".format(record['titles'][0]['title']))
def test_record_publish_adds_no_handles_for_external_files(app,
records_data_with_external_pids,
test_records_data):
"""Test that no handle PIDs are created for external files."""
for metadata in test_records_data:
with app.app_context():
app.config.update({'FAKE_EPIC_PID': True})
external_pids = records_data_with_external_pids['external_pids']
external_dict = {x['key']: x['ePIC_PID'] for x in external_pids}
data = deepcopy(metadata)
data['external_pids'] = deepcopy(external_pids)
record_uuid = uuid.uuid4()
b2share_deposit_uuid_minter(record_uuid, data=data)
deposit = Deposit.create(data, id_=record_uuid)
ObjectVersion.create(deposit.files.bucket, 'real_file_1.txt',
stream=BytesIO(b'mycontent'))
ObjectVersion.create(deposit.files.bucket, 'real_file_2.txt',
stream=BytesIO(b'mycontent'))
deposit.submit()
deposit.publish()
deposit.commit()
_, record = deposit.fetch_published()
# external files don't get a handle PID, they already have one
# which is stored in record['_deposit']['external_pids']
for f in record.files:
if f['key'] in external_dict:
assert f.get('ePIC_PID') is None
else:
assert '0000' in f['ePIC_PID'] # is a new fake PID
def test_deposit_update_unknown_publication_state(app, draft_deposits):
"""Test deposit submission by updating the "publication_state" field."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].id)
deposit.update({'publication_state': 'invalid_state'})
with pytest.raises(InvalidPublicationStateError):
deposit.commit()
def test_deposit_create(app, draft_deposits):
"""Test deposit creation."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].id)
assert (deposit['publication_state']
== PublicationStates.draft.name)
assert (deposit['_deposit']['status'] == 'draft')
def test_missing_field(field):
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
draft_patch_res = client.patch(
url_for('b2share_deposit_rest.b2dep_item',
pid_value=deposit.pid.pid_value),
data=json.dumps([{
"op": "remove", "path": '/'+field,
}, {
"op": "replace", "path": "/publication_state",
"value": PublicationStates.submitted.name
}]),
headers=headers)
assert draft_patch_res.status_code == 400
draft_patch_error = json.loads(
draft_patch_res.get_data(as_text=True))
assert draft_patch_error['message'] == "Validation error."
assert draft_patch_error['errors'][0]['field'] == field
return draft_patch_error['errors'][0]['message']
def test_adding_external_files(app, records_data_with_external_pids,
deposit_with_external_pids):
"""Test the addition of external files."""
with app.app_context():
# renaming a file to have a duplicate key
records_data_with_external_pids['external_pids'][0]['key'] = \
'file2.txt'
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
# this should be caught and return an InvalidDepositError
with pytest.raises(InvalidDepositError):
deposit = deposit.patch([
{'op': 'replace', 'path': '/external_pids',
'value': records_data_with_external_pids['external_pids']}
])
deposit.commit()
with app.app_context():
records_data_with_external_pids['external_pids'][0]['key'] = \
'file1.txt'
records_data_with_external_pids['external_pids'].append({
"key": "file3.txt",
"ePIC_PID":
"http://hdl.handle.net/11304/0d8dbdec-74e4-4774-954e-1a98e5c0cfa3"
})
deposit = deposit.patch([
{'op': 'replace', 'path': '/external_pids',
'value': records_data_with_external_pids['external_pids']}
])
deposit.commit()
assert_external_files(deposit,
records_data_with_external_pids['external_pids'])
def test_deposit_update_unknown_publication_state(app, draft_deposits):
"""Test deposit submission by updating the "publication_state" field."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
deposit.update({'publication_state':
'invalid_state'})
with pytest.raises(InvalidPublicationStateError):
deposit.commit()
def test_deposit_submit(app, draft_deposits):
"""Test deposit submission."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].id)
deposit.submit()
assert (
deposit['publication_state'] == PublicationStates.submitted.name)
assert (deposit['_deposit']['status'] == 'draft')
def test_deposit_submit(app, draft_deposits):
"""Test deposit submission."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
deposit.submit()
assert (deposit['publication_state']
== PublicationStates.submitted.name)
assert (deposit['_deposit']['status'] == 'draft')
def create_deposit(data, creator, files=None):
"""Create a deposit with the given user as creator."""
with authenticated_user(creator):
deposit = Deposit.create(data=deepcopy(data))
if files is not None:
for key, value in files.items():
deposit.files[key] = BytesIO(value)
return deposit
def test_deposit_submit(app, test_records_data, draft_deposits, test_users,
login_user):
"""Test record draft submit with HTTP PATCH."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
record_data = test_records_data[0]
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
draft_patch_res = client.patch(url_for(
'b2share_deposit_rest.b2dep_item',
pid_value=deposit.pid.pid_value),
data=json.dumps([{
"op":
"replace",
"path":
"/publication_state",
"value":
PublicationStates.submitted.name
}]),
headers=headers)
assert draft_patch_res.status_code == 200
draft_patch_data = json.loads(
draft_patch_res.get_data(as_text=True))
expected_metadata = build_expected_metadata(
record_data,
PublicationStates.draft.name,
owners=[user.id],
draft=True,
)
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
expected_metadata['publication_state'] = \
PublicationStates.submitted.name
assert expected_metadata == draft_patch_data['metadata']
assert (deposit['publication_state'] ==
PublicationStates.submitted.name)
subtest_self_link(draft_patch_data, draft_patch_res.headers,
client)
def test_deposit_publish(app, draft_deposits):
"""Test deposit submission by updating the "publication_state" field."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].id)
deposit.submit()
deposit.publish()
assert (
deposit['publication_state'] == PublicationStates.published.name)
assert (deposit['_deposit']['status'] == 'published')
def test_deposit_create_with_incomplete_metadata(app,
test_incomplete_records_data):
"""Test deposit creation with incomplete metadata succeeds."""
with app.app_context():
for data in test_incomplete_records_data:
deposit = Deposit.create(data.incomplete_data)
assert (
deposit['publication_state'] == PublicationStates.draft.name)
assert (deposit['_deposit']['status'] == 'draft')
def test_deposit_update_and_submit(app, draft_deposits):
"""Test deposit submission by updating the "publication_state" field."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].id)
deposit.update({'publication_state': PublicationStates.submitted.name})
deposit.commit()
assert (
deposit['publication_state'] == PublicationStates.submitted.name)
assert (deposit['_deposit']['status'] == 'draft')
def test_deposit_publish(app, draft_deposits):
"""Test deposit submission by updating the "publication_state" field."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
deposit.submit()
deposit.publish()
assert (deposit['publication_state']
== PublicationStates.published.name)
assert (deposit['_deposit']['status'] == 'published')
def test_deposit_update_and_submit(app, draft_deposits):
"""Test deposit submission by updating the "publication_state" field."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
deposit.update({'publication_state':
PublicationStates.submitted.name})
deposit.commit()
assert (deposit['publication_state']
== PublicationStates.submitted.name)
assert (deposit['_deposit']['status'] == 'draft')
def test_deposit_submit(app, test_records_data, draft_deposits, test_users,
login_user):
"""Test record draft submit with HTTP PATCH."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
record_data = test_records_data[0]
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
draft_patch_res = client.patch(
url_for('b2share_deposit_rest.b2dep_item',
pid_value=deposit.pid.pid_value),
data=json.dumps([{
"op": "replace", "path": "/publication_state",
"value": PublicationStates.submitted.name
}]),
headers=headers)
assert draft_patch_res.status_code == 200
draft_patch_data = json.loads(
draft_patch_res.get_data(as_text=True))
expected_metadata = build_expected_metadata(
record_data,
PublicationStates.draft.name,
owners=[user.id],
draft=True,
)
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
expected_metadata['publication_state'] = \
PublicationStates.submitted.name
assert expected_metadata == draft_patch_data['metadata']
assert (deposit['publication_state']
== PublicationStates.submitted.name)
subtest_self_link(draft_patch_data,
draft_patch_res.headers,
client)
def create(data):
data = deepcopy(data)
record_uuid = uuid.uuid4()
# Create persistent identifier
b2share_deposit_uuid_minter(record_uuid, data=data)
deposit = Deposit.create(data=data, id_=record_uuid,
version_of=version_of)
if files is not None:
for key, value in files.items():
deposit.files[key] = BytesIO(value)
return deposit
def test_deposit_submit_with_incomplete_metadata(app,
test_incomplete_records_data):
"""Test deposit submission with incomplete metadata fails."""
for data in test_incomplete_records_data:
with app.app_context():
deposit = Deposit.create(data.complete_data)
deposit.commit()
# make the data incomplete
deposit = deposit.patch(data.patch)
with pytest.raises(ValidationError):
deposit.submit()
def create_deposits(app, test_records_data, creator):
"""Create test deposits."""
DepositInfo = namedtuple('DepositInfo', ['id', 'data', 'deposit'])
with authenticated_user(creator):
deposits = [Deposit.create(data=data)
for data in deepcopy(test_records_data)]
for deposit in deposits:
deposit.commit()
deposit.commit()
return [DepositInfo(dep.id, dep.dumps(), dep) for dep in deposits]
def test_modify_external_files(app, deposit_with_external_pids,
new_external_pids):
"""Test changing PID, renaming and deletion of external files."""
with app.app_context():
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
deposit = deposit.patch([
{'op': 'replace', 'path': '/external_pids',
'value': new_external_pids}
])
deposit.commit()
assert_external_files(deposit, new_external_pids)
def test_modify_external_files(app, deposit_with_external_pids,
new_external_pids):
"""Test changing PID, renaming and deletion of external files."""
with app.app_context():
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
deposit = deposit.patch([{
'op': 'replace',
'path': '/external_pids',
'value': new_external_pids
}])
deposit.commit()
assert_external_files(deposit, new_external_pids)
def create_deposits(app, test_records_data, creator):
"""Create test deposits."""
DepositInfo = namedtuple(
'DepositInfo', [
'deposit_id',
'data',
'deposit', # FIXME: replaced by get_deposit, remove it later
'get_deposit'
])
deposits = []
with authenticated_user(creator):
for data in deepcopy(test_records_data):
record_uuid = uuid.uuid4()
# Create persistent identifier
b2share_deposit_uuid_minter(record_uuid, data=data)
deposits.append(B2ShareDeposit.create(data=data, id_=record_uuid))
return [DepositInfo(
dep.id, dep.dumps(), dep,
(lambda id: lambda: B2ShareDeposit.get_record(id))(dep.id)
) for dep in deposits]
def test_deposit_publish_with_incomplete_metadata(
app, test_incomplete_records_data):
"""Test publication of an incomplete deposit fails."""
for data in test_incomplete_records_data:
with app.app_context():
deposit = Deposit.create(data.complete_data)
deposit.submit()
deposit.commit()
# make the data incomplete
deposit = deposit.patch(data.patch)
with pytest.raises(ValidationError):
deposit.publish()
def create_deposits(app, test_records_data, creator):
"""Create test deposits."""
DepositInfo = namedtuple('DepositInfo', ['deposit_id', 'data', 'deposit'])
deposits = []
with authenticated_user(creator):
for data in deepcopy(test_records_data):
record_uuid = uuid.uuid4()
# Create persistent identifier
b2share_deposit_uuid_minter(record_uuid, data=data)
deposits.append(Deposit.create(data=data, id_=record_uuid))
return [DepositInfo(dep.id, dep.dumps(), dep) for dep in deposits]
def test_new_deposit_versions_preserve_schema(app, test_records, test_users):
"""Creating new versions of existing records."""
with app.app_context():
# Retrieve a record which will be the first version, and its deposit
v1_rec = B2ShareRecord.get_record(test_records[0].record_id)
_, v1_id = pid_of(test_records[0].data)
# update the community's schema
community_id = v1_rec.model.json['community']
old_schema = CommunitySchema.get_community_schema(community_id)
json_schema = json.loads(old_schema.model.community_schema)
new_schema = CommunitySchema.create_version(community_id, json_schema)
assert new_schema.version > old_schema.version
# create new, unversioned, draft and records
data = copy_data_from_previous(v1_rec.model.json)
unver_dep = create_deposit(data, test_users['deposits_creator'])
unver_dep.submit()
unver_dep.publish()
_, unver_rec = unver_dep.fetch_published()
# the unversioned draft or record in a version chain have the updated schema
assert unver_dep['$schema'] != Deposit._build_deposit_schema(
v1_rec.model.json)
assert unver_rec.model.json['$schema'] != v1_rec.model.json['$schema']
# create new draft and record in the version chain of v1
data = copy_data_from_previous(v1_rec.model.json)
v2_dep = create_deposit(data,
test_users['deposits_creator'],
version_of=v1_id)
v2_dep.submit()
v2_dep.publish()
_, v2_rec = v2_dep.fetch_published()
# the new draft and record in a version chain preserve the version of the schema
assert v2_dep['$schema'] == Deposit._build_deposit_schema(
v1_rec.model.json)
assert v2_rec.model.json['$schema'] == v1_rec.model.json['$schema']
def test_record_commit_with_incomplete_metadata(app,
test_incomplete_records_data):
"""Test commit of an incomplete record fails."""
for data in test_incomplete_records_data:
with app.app_context():
deposit = Deposit.create(data.complete_data)
deposit.submit()
deposit.publish()
deposit.commit()
pid, record = deposit.fetch_published()
# make the data incomplete
record = record.patch(data.patch)
with pytest.raises(ValidationError):
record.commit()
def create_deposits(app, test_records_data, creator):
"""Create test deposits."""
DepositInfo = namedtuple(
'DepositInfo',
[
'deposit_id',
'data',
'deposit', # FIXME: replaced by get_deposit, remove it later
'get_deposit'
])
deposits = []
with authenticated_user(creator):
for data in deepcopy(test_records_data):
record_uuid = uuid.uuid4()
# Create persistent identifier
b2share_deposit_uuid_minter(record_uuid, data=data)
deposits.append(B2ShareDeposit.create(data=data, id_=record_uuid))
return [
DepositInfo(dep.id, dep.dumps(), dep,
(lambda id: lambda: B2ShareDeposit.get_record(id))(dep.id))
for dep in deposits
]
def test_deposit_put_is_disabled(app, draft_deposits, test_users, login_user):
"""Test invalid modification of record draft with HTTP PUT."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
headers = [('Content-Type', 'application/json'),
('Accept', 'application/json')]
draft_put_res = client.put(url_for(
'b2share_deposit_rest.b2dep_item',
pid_value=deposit.pid.pid_value),
data='{}',
headers=headers)
assert draft_put_res.status_code == 405
def test_record_commit_with_incomplete_metadata(app,
test_incomplete_records_data):
"""Test commit of an incomplete record fails."""
for metadata in test_incomplete_records_data:
with app.app_context():
data = deepcopy(metadata.complete_data)
record_uuid = uuid.uuid4()
b2share_deposit_uuid_minter(record_uuid, data=data)
deposit = Deposit.create(data, id_=record_uuid)
deposit.submit()
deposit.publish()
deposit.commit()
pid, record = deposit.fetch_published()
# make the data incomplete
record = record.patch(metadata.patch)
with pytest.raises(ValidationError):
record.commit()
def test_deposit_put_is_disabled(app, draft_deposits, test_users,
login_user):
"""Test invalid modification of record draft with HTTP PUT."""
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
with app.test_client() as client:
user = test_users['deposits_creator']
login_user(user, client)
headers = [('Content-Type', 'application/json'),
('Accept', 'application/json')]
draft_put_res = client.put(
url_for('b2share_deposit_rest.b2dep_item',
pid_value=deposit.pid.pid_value),
data='{}',
headers=headers)
assert draft_put_res.status_code == 405
def test_missing_handle_prefix(app, test_users, login_user,
records_data_with_external_pids,
deposit_with_external_pids):
"""Test external files that are registered without a handle prefix."""
with app.app_context():
no_prefix_external_pid = \
[{"key": "no_prefix.txt",
"ePIC_PID": "11304/0d8dbdec-74e4-4774-954e-1a98e5c0cfa3"}]
correct_external_pid = \
[{"key": "no_prefix.txt",
"ePIC_PID":
"http://hdl.handle.net/11304/0d8dbdec-74e4-4774-954e-1a98e5c0cfa3"}]
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
deposit = deposit.patch([
{'op': 'replace', 'path': '/external_pids',
'value': no_prefix_external_pid}
])
deposit.commit()
assert_external_files(deposit, correct_external_pid)
headers = [('Content-Type', 'application/json'),
('Accept', 'application/json')]
def create_record(client, record_data):
return client.post(
url_for('b2share_records_rest.b2rec_list'),
data=json.dumps(record_data),
headers=headers
)
with app.app_context():
with app.test_client() as client:
user = test_users['normal']
login_user(user, client)
# create the deposit with the missing prefix
records_data_with_external_pids['external_pids'] = \
no_prefix_external_pid
draft_create_res = create_record(
client, records_data_with_external_pids)
assert draft_create_res.status_code == 201
draft_create_data = json.loads(
draft_create_res.get_data(as_text=True))
# assert that when returned the external pid has the prefix
assert draft_create_data['metadata']['external_pids'] == \
correct_external_pid
def test_embargoed_records_with_external_pids(app, test_users, login_user,
deposit_with_external_pids):
"""Test that embargoed records handle external pids correctly."""
with app.app_context():
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
deposit = deposit.patch([{
'op':
'add',
'path':
'/embargo_date',
'value': (datetime.utcnow() + timedelta(days=1)).isoformat()
}])
deposit = deposit.patch([{
'op': 'replace',
'path': '/open_access',
'value': False
}])
deposit.commit()
deposit.submit()
deposit.publish()
with app.app_context():
with app.test_client() as client:
# anonymous user tries to download the external file
for file in deposit.files:
ext_file_url = url_for('invenio_files_rest.object_api',
bucket_id=file.obj.bucket,
key=file.obj.key)
resp = client.get(ext_file_url)
# 404
assert resp.status_code == 404
# test that when getting the record the external_pids field
# is not visible in the metadata
with app.app_context():
with app.test_client() as client:
# the owner can download the external file
login_user(test_users['deposits_creator'], client)
for file in deposit.files:
ext_file_url = url_for('invenio_files_rest.object_api',
bucket_id=file.obj.bucket,
key=file.obj.key)
resp = client.get(ext_file_url)
assert resp.status_code == 302
def test_embargoed_records_with_external_pids(app, test_users,
login_user,
deposit_with_external_pids):
"""Test that embargoed records handle external pids correctly."""
with app.app_context():
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
deposit = deposit.patch([
{'op': 'add', 'path': '/embargo_date',
'value': (datetime.utcnow() + timedelta(days=1)).isoformat()}
])
deposit = deposit.patch([
{'op': 'replace', 'path': '/open_access',
'value': False}
])
deposit.commit()
deposit.submit()
deposit.publish()
with app.app_context():
with app.test_client() as client:
# anonymous user tries to download the external file
for file in deposit.files:
ext_file_url = url_for('invenio_files_rest.object_api',
bucket_id=file.obj.bucket,
key=file.obj.key)
resp = client.get(ext_file_url)
# 404
assert resp.status_code == 404
# test that when getting the record the external_pids field
# is not visible in the metadata
with app.app_context():
with app.test_client() as client:
# the owner can download the external file
login_user(test_users['deposits_creator'], client)
for file in deposit.files:
ext_file_url = url_for('invenio_files_rest.object_api',
bucket_id=file.obj.bucket,
key=file.obj.key)
resp = client.get(ext_file_url)
assert resp.status_code == 302
def test_deposit_add_unknown_fields(app, draft_deposits):
"""Test adding unknown fields in deposit. It should fail."""
for path in [
# all the following paths point to "object" fields in
# in the root JSON Schema
'/new_field',
'/community_specific/new_field',
'/creators/0/new_field',
'/titles/0/new_field',
'/contributors/0/new_field',
'/resource_types/0/new_field',
'/alternate_identifiers/0/new_field',
'/descriptions/0/new_field',
'/license/new_field',
]:
with app.app_context():
deposit = Deposit.get_record(draft_deposits[0].deposit_id)
deposit = deposit.patch([
{'op': 'add', 'path': path, 'value': 'any value'}
])
with pytest.raises(ValidationError):
deposit.commit()
def test_getting_record_with_external_pids(app, login_user, test_users,
deposit_with_external_pids,
records_data_with_external_pids):
"""External pids are serialized in the metadata when it is allowed."""
def test_get_deposit(deposit_pid_value, user):
with app.test_client() as client:
login_user(user, client)
deposit_url = url_for('b2share_deposit_rest.b2dep_item',
pid_value=deposit_pid_value)
resp = client.get(deposit_url)
deposit_data = json.loads(
resp.get_data(as_text=True))
return deposit_data
def test_get_record(record_pid_value, user):
with app.test_client() as client:
login_user(user, client)
record_url = url_for('b2share_records_rest.b2rec_item',
pid_value=record_pid_value)
resp = client.get(record_url)
record_data = json.loads(
resp.get_data(as_text=True))
return record_data
def test_search_deposits(user):
with app.test_client() as client:
login_user(user, client)
search_deposits_url = url_for(
'b2share_records_rest.b2rec_list', drafts=1, size=100)
headers = [('Content-Type', 'application/json'),
('Accept', 'application/json')]
resp = client.get(
search_deposits_url,
headers=headers)
deposit_search_res = json.loads(
resp.get_data(as_text=True))
return deposit_search_res
def test_search_records(user):
with app.test_client() as client:
login_user(user, client)
search_records_url = url_for(
'b2share_records_rest.b2rec_list', size=100)
headers = [('Content-Type', 'application/json'),
('Accept', 'application/json')]
resp = client.get(
search_records_url,
headers=headers)
record_search_res = json.loads(
resp.get_data(as_text=True))
return record_search_res
with app.app_context():
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
with app.app_context():
deposit_data = test_get_deposit(deposit.pid.pid_value,
test_users['deposits_creator'])
# assert that the external_pids are visible
# when getting a specific deposit
assert_external_files(
deposit,
deposit_data['metadata']['external_pids'])
current_search_client.indices.refresh('*')
deposit_search_data = test_search_deposits(
test_users['deposits_creator'])
assert deposit_search_data['hits']['total'] == 1
# external_pids are not shown in a deposit search because it would use
# too much resources to generate it for each search hit.
assert 'external_pids' not in deposit_search_data[
'hits']['hits'][0]['metadata']
with app.app_context():
deposit = Deposit.get_record(deposit_with_external_pids.deposit_id)
deposit.submit()
deposit.publish()
record_resolver = Resolver(
pid_type='b2rec',
object_type='rec',
getter=B2ShareRecord.get_record,
)
record_pid, record = record_resolver.resolve(deposit.pid.pid_value)
current_search_client.indices.refresh('*')
record_data = test_get_record(
record_pid.pid_value, test_users['deposits_creator'])
# when getting a specific record the owner sees the external_pids
assert_external_files(record, record_data['metadata']['external_pids'])
with app.app_context():
record_data = test_get_record(
record_pid.pid_value, test_users['normal'])
# and all other users as well if it is open access
assert_external_files(record, record_data['metadata']['external_pids'])
deposit_search_data = test_search_deposits(
test_users['deposits_creator'])
assert deposit_search_data['hits']['total'] == 1
# external_pids are not shown in deposit search even when published
assert 'external_pids' not in deposit_search_data[
'hits']['hits'][0]['metadata']
record_search_data = test_search_records(
test_users['deposits_creator'])
assert record_search_data['hits']['total'] == 1
# external_pids are shown for record search if they are open access
# for all users
assert 'external_pids' in record_search_data[
'hits']['hits'][0]['metadata']
record_search_data = test_search_records(test_users['normal'])
assert record_search_data['hits']['total'] == 1
assert 'external_pids' in record_search_data[
'hits']['hits'][0]['metadata']
with app.app_context():
deposit2 = create_deposit(records_data_with_external_pids,
test_users['deposits_creator'])
deposit2 = deposit2.patch([
{'op': 'add', 'path': '/embargo_date',
'value': (datetime.utcnow() + timedelta(days=1)).isoformat()},
{'op': 'replace', 'path': '/open_access',
'value': False}
])
deposit2.commit()
deposit2.submit()
deposit2.publish()
record_resolver = Resolver(
pid_type='b2rec',
object_type='rec',
getter=B2ShareRecord.get_record,
)
record_pid, record = record_resolver.resolve(deposit2.pid.pid_value)
record_data = test_get_record(
record_pid.pid_value, test_users['deposits_creator'])
# owners of records have access to files and
# external_pids of embargoed records
assert_external_files(record, record_data['metadata']['external_pids'])
with app.app_context():
record_data = test_get_record(
record_pid.pid_value, test_users['normal'])
# normal users shouldn't have access to the
# files and external_pids of an embargoed record
assert 'metadata' in record_data
assert 'external_pids' not in record_data['metadata']
def alembic_upgrade_database_data(alembic, verbose):
"""Migrate the database data from v2.0.0 to 2.1.0."""
### Add versioning PIDs ###
# Reserve the record PID and versioning PID for unpublished deposits
# Hack: disable record indexing during record migration
from invenio_indexer.api import RecordIndexer
old_index_fn = RecordIndexer.index
RecordIndexer.index = lambda s, record: None
if verbose:
click.secho('migrating deposits and records...')
with db.session.begin_nested():
# Migrate published records
records_pids = PersistentIdentifier.query.filter(
PersistentIdentifier.pid_type == RecordUUIDProvider.pid_type,
PersistentIdentifier.status == PIDStatus.REGISTERED,
).all()
for rec_pid in records_pids:
if verbose:
click.secho(' record {}'.format(rec_pid.pid_value))
try:
record = Record.get_record(rec_pid.object_uuid)
except NoResultFound:
# The record is deleted but not the PID. Fix it.
rec_pid.status = PIDStatus.DELETED
continue
# Create parent version PID
parent_pid = RecordUUIDProvider.create().pid
version_master = PIDVersioning(parent=parent_pid)
version_master.insert_draft_child(child=rec_pid)
version_master.update_redirect()
migrate_record_metadata(
Record.get_record(rec_pid.object_uuid),
parent_pid
)
# Migrate deposits
deposit_pids = PersistentIdentifier.query.filter(
PersistentIdentifier.pid_type == DepositUUIDProvider.pid_type,
PersistentIdentifier.status == PIDStatus.REGISTERED,
).all()
for dep_pid in deposit_pids:
if verbose:
click.secho(' deposit {}'.format(dep_pid.pid_value))
try:
deposit = Deposit.get_record(dep_pid.object_uuid)
if deposit['publication_state'] != \
PublicationStates.published.name:
# The record is not published yet. Reserve the PID.
rec_pid = RecordUUIDProvider.create(
object_type='rec',
pid_value=dep_pid.pid_value,
).pid
# Create parent version PID
parent_pid = RecordUUIDProvider.create().pid
assert parent_pid
version_master = PIDVersioning(parent=parent_pid)
version_master.insert_draft_child(child=rec_pid)
else:
# Retrieve previously created version PID
rec_pid = RecordUUIDProvider.get(dep_pid.pid_value).pid
version_master = PIDVersioning(child=rec_pid)
parent_pid = version_master.parent
if not parent_pid:
click.secho(' record {} was deleted, but the deposit has not been removed'.format(rec_pid.pid_value), fg='red')
if parent_pid:
migrate_record_metadata(
Deposit.get_record(dep_pid.object_uuid),
parent_pid
)
except NoResultFound:
# The deposit is deleted but not the PID. Fix it.
dep_pid.status = PIDStatus.DELETED
if verbose:
click.secho('done migrating deposits.')
RecordIndexer.index = old_index_fn
def test_deposit_publish(app, test_users, test_communities,
login_user):
"""Test record draft publication with HTTP PATCH."""
with app.app_context():
community_name = 'MyTestCommunity1'
creator = test_users['deposits_creator']
record_data = generate_record_data(community=community_name)
community = Community.get(name=community_name)
com_admin = create_user('com_admin', roles=[community.admin_role])
deposit = create_deposit(record_data, creator)
deposit_id = deposit.id
deposit.submit()
db.session.commit()
with app.test_client() as client:
login_user(com_admin, client)
headers = [('Content-Type', 'application/json-patch+json'),
('Accept', 'application/json')]
draft_patch_res = client.patch(
url_for('b2share_deposit_rest.b2dep_item',
pid_value=deposit.pid.pid_value),
data=json.dumps([{
"op": "replace", "path": "/publication_state",
"value": PublicationStates.published.name
}]),
headers=headers)
assert draft_patch_res.status_code == 200
draft_patch_data = json.loads(
draft_patch_res.get_data(as_text=True))
expected_metadata = build_expected_metadata(
record_data,
PublicationStates.published.name,
owners=[creator.id],
draft=True,
PID=draft_patch_data['metadata'].get('ePIC_PID'),
DOI=draft_patch_data['metadata'].get('DOI'),
)
with app.app_context():
deposit = Deposit.get_record(deposit_id)
with app.test_client() as client:
login_user(creator, client)
assert expected_metadata == draft_patch_data['metadata']
assert (deposit['publication_state']
== PublicationStates.published.name)
subtest_self_link(draft_patch_data,
draft_patch_res.headers,
client)
pid, published = deposit.fetch_published()
# check that the published record and the deposit are equal except
# for the schema
cleaned_deposit = {f: v for f, v in deposit.items()
if f != '$schema'}
cleaned_published = {f: v for f, v in deposit.items()
if f != '$schema'}
assert cleaned_published == cleaned_deposit
# check "published" link
assert draft_patch_data['links']['publication'] == \
url_for('b2share_records_rest.{0}_item'.format(
RecordUUIDProvider.pid_type
), pid_value=pid.pid_value, _external=True)
# check that the published record content match the deposit
headers = [('Accept', 'application/json')]
self_response = client.get(
draft_patch_data['links']['publication'],
headers=headers
)
assert self_response.status_code == 200
published_data = json.loads(self_response.get_data(
as_text=True))
# we don't want to compare the links and dates
cleaned_published_data = deepcopy(published_data)
# the published record has an extra empty 'files' array
assert cleaned_published_data['files'] == []
del cleaned_published_data['files']
cleaned_draft_data = deepcopy(draft_patch_data)
for item in [cleaned_published_data, cleaned_draft_data]:
del item['links']
del item['created']
del item['updated']
del item['metadata']['$schema']
assert cleaned_draft_data == cleaned_published_data