def test_autoadd_explicit_newversion(
app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Explicitly the autoadded communities in a new version."""
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2['communities'] = ['ecfunded', 'grants_comm', 'zenodo']
deposit_v2['grants'] = [{'title': 'SomeGrant'}, ]
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
assert record_v1.get('communities', []) == ['grants_comm', ]
assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
assert record_v2.get('communities', []) == ['grants_comm', ]
assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
def test_autoadd_explicit_newversion(
app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Explicitly the autoadded communities in a new version."""
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2['communities'] = ['ecfunded', 'grants_comm', 'zenodo']
deposit_v2['grants'] = [{'title': 'SomeGrant'}, ]
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
assert record_v1.get('communities', []) == ['grants_comm', ]
assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
assert record_v2.get('communities', []) == ['grants_comm', ]
assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
def test_basic_api(app, db, communities, deposit, deposit_file):
"""Test basic workflow using Deposit and Communities API."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c2' through deposit v2
deposit_v2['communities'] = [
'c1',
'c2',
]
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
recid_v2_value = recid_v2.pid_value
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
# Inclusion requests should be visible for both records
assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
# Accept to 'c1' through record_v2 (as originally requested),
# and 'c2' through record_v1 (version)
c1_api.accept_record(record_v2, pid=recid_v2)
c2_api.accept_record(record_v1, pid=recid_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
# Accepting individual record to a community should propagate the changes
# to all versions
assert record_v1['communities'] == record_v2['communities'] == \
['c1', 'c2', ]
# Removing 'c1' from deposit_v1 should remove it from two published records
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1 = deposit_v1.edit()
deposit_v1['communities'] = []
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
def test_basic_api(app, db, communities, deposit, deposit_file):
"""Test basic workflow using Deposit and Communities API."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c2' through deposit v2
deposit_v2['communities'] = ['c1', 'c2', ]
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
recid_v2_value = recid_v2.pid_value
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
# Inclusion requests should be visible for both records
assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
# Accept to 'c1' through record_v2 (as originally requested),
# and 'c2' through record_v1 (version)
c1_api.accept_record(record_v2, pid=recid_v2)
c2_api.accept_record(record_v1, pid=recid_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
# Accepting individual record to a community should propagate the changes
# to all versions
assert record_v1['communities'] == record_v2['communities'] == \
['c1', 'c2', ]
# Removing 'c1' from deposit_v1 should remove it from two published records
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1 = deposit_v1.edit()
deposit_v1['communities'] = []
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
def test_communities_newversion_while_ir_pending_bug(app, db, users,
communities, deposit,
deposit_file):
"""Make sure that pending IRs remain after a new version (bug)."""
deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
# Two inclusion requests are pending
assert InclusionRequest.query.count() == 2
# Accept one community
c1_api = ZenodoCommunity('c1')
c1_api.accept_record(record_v1, pid=recid_v1)
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
# Make sure there is still IR to community 'c2' after newversion
assert InclusionRequest.query.count() == 1
assert InclusionRequest.query.one().id_community == 'c2'
assert record_v1.get('communities', []) == [
'c1',
]
assert deposit_v1.get('communities', []) == [
'c1',
'c2',
]
assert record_v2.get('communities', []) == [
'c1',
]
assert deposit_v2.get('communities', []) == [
'c1',
'c2',
]
def test_propagation_with_newversion_open(
app, db, users, communities, deposit, deposit_file):
"""Adding old versions to a community should propagate to all drafts."""
# deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
deposit_v1 = deposit_v1.edit()
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
# New version in 'deposit_v2' has not been published yet
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
# depid_v1_value = deposit_v1['_deposit']['id']
# depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1['communities'] = ['c1', 'c2', ]
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
c1_api = ZenodoCommunity('c1')
c1_api.accept_record(record_v1, pid=recid_v1)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
assert deposit_v2['communities'] == ['c1', 'c2']
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
assert record_v2['communities'] == ['c1', ]
def test_propagation_with_newversion_open(
app, db, users, communities, deposit, deposit_file):
"""Adding old versions to a community should propagate to all drafts."""
# deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
deposit_v1 = deposit_v1.edit()
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
# New version in 'deposit_v2' has not been published yet
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
# depid_v1_value = deposit_v1['_deposit']['id']
# depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1['communities'] = ['c1', 'c2', ]
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
c1_api = ZenodoCommunity('c1')
c1_api.accept_record(record_v1, pid=recid_v1)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
assert deposit_v2['communities'] == ['c1', 'c2']
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
assert record_v2['communities'] == ['c1', ]
def test_rat_deposit_files_access(app, db, api_client, deposit, deposit_file,
deposit_url, json_auth_headers,
license_record, rat_generate_token):
"""Test deposit files access via RATs."""
client = api_client
depid = deposit['_deposit']['id']
deposit['owners'] = [rat_generate_token.user_id]
deposit['_deposit']['owners'] = [rat_generate_token.user_id]
deposit.commit()
db.session.commit()
rat_token = jwt.encode(
payload={
'iat': datetime.utcnow(),
'sub': {
'deposit_id': depid,
'access': 'read',
},
},
key=rat_generate_token.access_token,
algorithm='HS256',
headers={'kid': str(rat_generate_token.id)},
)
deposit_url += '/' + str(depid)
file_url = '/files/{}/test.txt'.format(deposit['_buckets']['deposit'])
publish_url = deposit_url + '/actions/publish'
res = client.get(file_url)
assert res.status_code == 404
res = client.get(file_url, query_string={'token': rat_token})
assert res.status_code == 200
# Try other forbidden operations using the RAT
res = client.get(deposit_url, query_string={'token': rat_token})
assert res.status_code == 401
data = json.dumps(get_data())
res = client.put(deposit_url, data=data, query_string={'token': rat_token})
assert res.status_code == 401
res = client.put(file_url, data=data, query_string={'token': rat_token})
assert res.status_code == 404
res = client.post(publish_url, query_string={'token': rat_token})
assert res.status_code == 401
# Change record owner
depid, deposit = deposit_resolver.resolve(depid)
deposit['owners'] = [123]
deposit['_deposit']['owners'] = [123]
deposit.commit()
db.session.commit()
res = client.get(file_url)
assert res.status_code == 404
res = client.get(file_url, query_string={'token': rat_token})
assert res.status_code == 404
def test_communities_newversion_addition(app, db, users, communities, deposit,
deposit_file):
"""Make sure that new version of record synchronizes the communities."""
deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
c1_api.accept_record(record_v1, pid=recid_v1)
c2_api.accept_record(record_v1, pid=recid_v1)
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
# Remove 'c2' and request for 'c5'. Make sure that communities from
# previous record version are preserved/removed properly
deposit_v2['communities'] = ['c1', 'c5']
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
assert record_v1.get('communities', []) == [
'c1',
]
assert deposit_v1.get('communities', []) == [
'c1',
'c5',
]
assert record_v2.get('communities', []) == [
'c1',
]
assert deposit_v2.get('communities', []) == [
'c1',
'c5',
]
def test_extra_formats_buckets_permissions(api, api_client, minimal_deposit,
deposit_url, db, es, users,
locations, json_extra_auth_headers,
extra_auth_headers, license_record,
user_email, status):
"""Test Files-REST permissions for the extra formats bucket and files."""
# Create deposit
response = api_client.post(deposit_url,
json=minimal_deposit,
headers=json_extra_auth_headers)
data = response.json
# Get identifier and links
depid = data['record_id']
links = data['links']
# Upload 1 files
response = api_client.put(
links['bucket'] + '/test1.txt',
data='ctx',
headers=extra_auth_headers,
)
# Add extra_formats bucket with a file
response = api_client.put(
'/deposit/depositions/{0}/formats'.format(depid),
data='foo file',
headers=[('Content-Type', 'application/foo+xml')] + extra_auth_headers)
dep_uuid, deposit = deposit_resolver.resolve(depid)
if user_email:
# Login as user
login_user_via_session(api_client, email=user_email)
response = api_client.get('/files/' + str(deposit.extra_formats.bucket.id))
assert response.status_code == status
response = api_client.put('/files/' +
str(deposit.extra_formats.bucket.id) +
'/application/foo+xml',
data='ctx')
assert response.status_code == status
# Publish deposition
response = api_client.post(links['publish'], headers=extra_auth_headers)
if user_email:
# Login as user
login_user_via_session(api_client, email=user_email)
response = api_client.get('/files/' + str(deposit.extra_formats.bucket.id))
assert response.status_code == status
response = api_client.put('/files/' +
str(deposit.extra_formats.bucket.id) +
'/application/foo+xml',
data='ctx')
assert response.status_code == status
def test_communities_newversion_while_ir_pending_bug(
app, db, users, communities, deposit, deposit_file):
"""Make sure that pending IRs remain after a new version (bug)."""
deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
# Two inclusion requests are pending
assert InclusionRequest.query.count() == 2
# Accept one community
c1_api = ZenodoCommunity('c1')
c1_api.accept_record(record_v1, pid=recid_v1)
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
# Make sure there is still IR to community 'c2' after newversion
assert InclusionRequest.query.count() == 1
assert InclusionRequest.query.one().id_community == 'c2'
assert record_v1.get('communities', []) == ['c1', ]
assert deposit_v1.get('communities', []) == ['c1', 'c2', ]
assert record_v2.get('communities', []) == ['c1', ]
assert deposit_v2.get('communities', []) == ['c1', 'c2', ]
def test_communities_newversion_addition(
app, db, users, communities, deposit, deposit_file):
"""Make sure that new version of record synchronizes the communities."""
deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
c1_api.accept_record(record_v1, pid=recid_v1)
c2_api.accept_record(record_v1, pid=recid_v1)
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
# Remove 'c2' and request for 'c5'. Make sure that communities from
# previous record version are preserved/removed properly
deposit_v2['communities'] = ['c1', 'c5']
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
assert record_v1.get('communities', []) == ['c1', ]
assert deposit_v1.get('communities', []) == ['c1', 'c5', ]
assert record_v2.get('communities', []) == ['c1', ]
assert deposit_v2.get('communities', []) == ['c1', 'c5', ]
def test_relations_serialization(app, db, deposit, deposit_file):
"""Serialize PID relations."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
expected = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 1
}
]
}
assert serialize_relations(recid_v1) == expected
deposit_v1.newversion()
# Should contain "draft_child_deposit" information
expected = {
"version": [
{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 1,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert serialize_relations(recid_v1) == expected
# Publish the new version
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
# Should no longer contain "draft_child_deposit" info after publishing
# and no longer be the last child
expected = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}
]
}
assert serialize_relations(recid_v1) == expected
# New version should be the last child now
expected = {
"version": [
{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert serialize_relations(recid_v2) == expected
def test_related_identifiers_serialization(app, db, deposit, deposit_file):
"""Serialize PID Relations to related identifiers."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c2' through deposit v2
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
rids = serialize_related_identifiers(recid_v1)
expected_v1 = [
{
'scheme': 'doi',
'identifier': '10.5072/zenodo.1',
'relation': 'isPartOf'
}
# TODO: serialization of new version realtions is disabled
# {
# 'scheme': 'doi',
# 'identifier': '10.5072/zenodo.3',
# 'relation': 'isPreviousVersionOf'
# }
]
assert rids == expected_v1
rids = serialize_related_identifiers(recid_v2)
expected_v2 = [
{
'scheme': 'doi',
'identifier': '10.5072/zenodo.1',
'relation': 'isPartOf'
}
# TODO: serialization of new version realtions is disabled
# {
# 'scheme': 'doi',
# 'identifier': '10.5072/zenodo.2',
# 'relation': 'isNewVersionOf'
# }
]
assert rids == expected_v2
parent_pid = PersistentIdentifier.get('recid', '1')
rids = serialize_related_identifiers(parent_pid)
expected_parent = [
{
'relation': 'hasPart',
'scheme': 'doi',
'identifier': '10.5072/zenodo.2'
},
{
'relation': 'hasPart',
'scheme': 'doi',
'identifier': '10.5072/zenodo.3'
}
]
assert rids == expected_parent
def delete_record(record_uuid, reason, user):
"""Delete the record and it's PIDs.
:param record_uuid: UUID of the record to be removed.
:param reason: Reason for removal. Either one of: 'spam', 'uploader',
'takedown' (see 'ZENODO_REMOVAL_REASONS' variable in config),
otherwise using it as a verbatim "Reason" string.
:param user: ID or email of the Zenodo user (admin)
responsible for the removal.
"""
from invenio_github.models import ReleaseStatus
if isinstance(user, text_type):
user_id = User.query.filter_by(email=user).one().id
elif isinstance(user, int):
user_id = User.query.get(user).id
else:
raise TypeError(
"User cannot be determined from argument: {0}".format(user))
record = ZenodoRecord.get_record(record_uuid)
# Remove the record from versioning and delete the recid
recid = PersistentIdentifier.get('recid', record['recid'])
pv = PIDVersioning(child=recid)
pv.remove_child(recid)
pv.update_redirect()
recid.delete()
# Remove the record from index
try:
RecordIndexer().delete(record)
except NotFoundError:
pass
# Remove buckets
record_bucket = record.files.bucket
RecordsBuckets.query.filter_by(record_id=record.id).delete()
record_bucket.locked = False
record_bucket.remove()
removal_reasons = dict(current_app.config['ZENODO_REMOVAL_REASONS'])
if reason in removal_reasons:
reason = removal_reasons[reason]
depid, deposit = deposit_resolver.resolve(record['_deposit']['id'])
try:
doi = PersistentIdentifier.get('doi', record['doi'])
except PIDDoesNotExistError:
doi = None
# Record OpenAIRE info
try:
original_id = openaire_original_id(record, openaire_type(record))[1]
datasource_id = openaire_datasource_id(record)
except PIDDoesNotExistError:
original_id = None
datasource_id = None
if pv.children.count() == 0:
conceptrecid = PersistentIdentifier.get('recid',
record['conceptrecid'])
conceptrecid.delete()
new_last_child = None
else:
new_last_child = (pv.last_child.pid_value,
str(pv.last_child.object_uuid))
if 'conceptdoi' in record:
conceptdoi_value = record['conceptdoi']
else:
conceptdoi_value = None
# Completely delete the deposit
# Deposit will be removed from index
deposit.delete(delete_published=True)
# Clear the record and put the deletion information
record.clear()
record.update({
'removal_reason': reason,
'removed_by': user_id,
})
record.commit()
# Mark the relevant GitHub Release as deleted
for ghr in record.model.github_releases:
ghr.status = ReleaseStatus.DELETED
if not is_local_doi(doi.pid_value):
db.session.delete(doi)
db.session.commit()
# After successful DB commit, sync the DOIs with DataCite
if is_local_doi(doi.pid_value):
datacite_inactivate.delay(doi.pid_value)
if conceptdoi_value:
if new_last_child:
# Update last child (updates also conceptdoi)
pid_value, rec_uuid = new_last_child
datacite_register.delay(pid_value, rec_uuid)
else:
datacite_inactivate.delay(conceptdoi_value)
# Also delete from OpenAIRE index
if current_app.config['OPENAIRE_DIRECT_INDEXING_ENABLED'] and original_id \
and datasource_id:
openaire_delete.delay(original_id=original_id,
datasource_id=datasource_id)
def test_extra_formats_buckets_permissions(
api, api_client, minimal_deposit, deposit_url, db, es, users,
locations, json_extra_auth_headers, extra_auth_headers, license_record,
user_email, status
):
"""Test Files-REST permissions for the extra formats bucket and files."""
# Create deposit
response = api_client.post(
deposit_url, json=minimal_deposit, headers=json_extra_auth_headers)
data = response.json
# Get identifier and links
depid = data['record_id']
links = data['links']
# Upload 1 files
response = api_client.put(
links['bucket'] + '/test1.txt',
data='ctx',
headers=extra_auth_headers,
)
# Add extra_formats bucket with a file
response = api_client.put(
'/deposit/depositions/{0}/formats'.format(depid),
data='foo file',
headers=[('Content-Type', 'application/foo+xml')] +
extra_auth_headers
)
dep_uuid, deposit = deposit_resolver.resolve(depid)
if user_email:
# Login as user
login_user_via_session(api_client, email=user_email)
response = api_client.get(
'/files/' + str(deposit.extra_formats.bucket.id)
)
assert response.status_code == status
response = api_client.put(
'/files/' + str(deposit.extra_formats.bucket.id) +
'/application/foo+xml',
data='ctx'
)
assert response.status_code == status
# Publish deposition
response = api_client.post(links['publish'], headers=extra_auth_headers)
if user_email:
# Login as user
login_user_via_session(api_client, email=user_email)
response = api_client.get(
'/files/' + str(deposit.extra_formats.bucket.id)
)
assert response.status_code == status
response = api_client.put(
'/files/' + str(deposit.extra_formats.bucket.id) +
'/application/foo+xml',
data='ctx'
)
assert response.status_code == status
def versioning_link_records(recids):
"""Link several non-versioned records into one versioning scheme.
The records are linked in the order as they appear in the list, with
the first record being base for minting of the conceptdoi.
In case one of the records is already upgraded, its taken as the base
for conceptdoi instead, with preserving the requested order.
:param recids: list of recid values (strings) to link,
e.g.: ['1234','55125','51269']
:type recids: list of str
"""
recids_records = [
record_resolver.resolve(recid_val) for recid_val in recids
]
depids_deposits = [
deposit_resolver.resolve(record['_deposit']['id'])
for _, record in recids_records
]
rec_comms = sorted(
set(sum([rec.get('communities', []) for _, rec in recids_records],
[])))
dep_comms = sorted(
set(sum([dep.get('communities', []) for _, dep in depids_deposits],
[])))
upgraded = [(recid, rec) for recid, rec in recids_records
if 'conceptdoi' in rec]
# Determine the base record for versioning
if len(upgraded) == 0:
recid_v, record_v = recids_records[0]
elif len(upgraded) == 1:
recid_v, record_v = upgraded[0]
elif len(upgraded) > 1:
recid_v, record_v = upgraded[0]
child_recids = [
int(recid.pid_value)
for recid in PIDVersioning(child=recid_v).children.all()
]
i_upgraded = [int(recid.pid_value) for recid, rec in upgraded]
if set(child_recids) != set(i_upgraded):
raise Exception('Multiple upgraded records, which belong'
'to different versioning schemes.')
# Get the first record and mint the concept DOI for it
conceptdoi = zenodo_concept_doi_minter(record_v.id, record_v)
conceptrecid_v = PersistentIdentifier.get('recid',
record_v['conceptrecid'])
conceptrecid_v_val = conceptrecid_v.pid_value
pv_r1 = PIDVersioning(parent=conceptrecid_v)
children_recids = [c.pid_value for c in pv_r1.children.all()]
if not all(cr in recids for cr in children_recids):
raise Exception('Children of the already upgraded record: {0} are '
'not specified in the ordering: {1}'
''.format(children_recids, recids))
for (recid, record), (depid, deposit) in \
zip(recids_records, depids_deposits):
# Remove old versioning schemes for non-base recids
# Note: This will remove the child of the base-conceptrecid as well
# but that's OK, since it will be added again afterwards in the
# correct order.
conceptrecid = PersistentIdentifier.get('recid',
record['conceptrecid'])
pv = PIDVersioning(parent=conceptrecid)
pv.remove_child(recid)
if conceptrecid.pid_value != conceptrecid_v_val:
conceptrecid.delete()
# Update the 'conceptrecid' and 'conceptdoi' in records and deposits
record['conceptdoi'] = conceptdoi.pid_value
record['conceptrecid'] = conceptrecid_v.pid_value
record['communities'] = rec_comms
record.commit()
deposit['conceptdoi'] = conceptdoi.pid_value
deposit['conceptrecid'] = conceptrecid_v.pid_value
deposit['communities'] = dep_comms
deposit.commit()
# Add the child to the new versioning scheme
pv_r1.insert_child(recid)
pv_r1.update_redirect()
db.session.commit()
conceptrecid_v = PersistentIdentifier.get('recid', conceptrecid_v_val)
pv = PIDVersioning(parent=conceptrecid_v)
if current_app.config['DEPOSIT_DATACITE_MINTING_ENABLED']:
datacite_register.delay(pv.last_child.pid_value,
str(pv.last_child.object_uuid))
index_siblings(pv.last_child, with_deposits=True, eager=True)
def test_autoadd(app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Test basic workflow using Deposit and Communities API."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c3' (owned by user) through deposit v2
deposit_v2['communities'] = [
'c1',
'c2',
'c3',
]
deposit_v2['grants'] = [
{
'title': 'SomeGrant'
},
]
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
assert record_v2['grants'] == [
{
'title': 'SomeGrant'
},
]
recid_v2_value = recid_v2.pid_value
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == ['c3', 'grants_comm']
assert record_v2.get('communities', []) == ['c3', 'grants_comm']
assert deposit_v1.get('communities', []) == [
'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'
]
assert deposit_v2.get('communities', []) == [
'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'
]
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
c3_api = ZenodoCommunity('c3')
grants_comm_api = ZenodoCommunity('grants_comm')
ecfunded_api = ZenodoCommunity('ecfunded')
zenodo_api = ZenodoCommunity('zenodo')
# Inclusion requests should be visible for both records
assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c3_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0
assert c3_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0
assert grants_comm_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0
assert grants_comm_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0
assert ecfunded_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert ecfunded_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert zenodo_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert zenodo_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
# Accept to 'c1' through record_v2 (as originally requested),
# and 'c2' through record_v1 (resolved through version)
c1_api.accept_record(record_v2, pid=recid_v2)
c2_api.accept_record(record_v1, pid=recid_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
# Accepting individual record to a community should propagate the changes
# to all versions
assert record_v1.get('communities',
[]) == ['c1', 'c2', 'c3', 'grants_comm']
assert record_v2.get('communities',
[]) == ['c1', 'c2', 'c3', 'grants_comm']
assert deposit_v1.get('communities', []) == [
'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'
]
assert deposit_v2.get('communities', []) == [
'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'
]
# Removing 'c1'-'c3' from deposit_v1 should remove it from two published
# records and other deposits as well
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1 = deposit_v1.edit()
deposit_v1['communities'] = []
deposit_v1 = publish_and_expunge(db, deposit_v1)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
assert record_v1.get('communities', []) == [
'grants_comm',
]
assert record_v2.get('communities', []) == [
'grants_comm',
]
assert deposit_v1.get('communities',
[]) == ['ecfunded', 'grants_comm', 'zenodo']
assert deposit_v2.get('communities',
[]) == ['ecfunded', 'grants_comm', 'zenodo']
def test_autoadd(app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Test basic workflow using Deposit and Communities API."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c3' (owned by user) through deposit v2
deposit_v2['communities'] = ['c1', 'c2', 'c3', ]
deposit_v2['grants'] = [{'title': 'SomeGrant'}, ]
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
assert record_v2['grants'] == [{'title': 'SomeGrant'}, ]
recid_v2_value = recid_v2.pid_value
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == ['c3', 'grants_comm']
assert record_v2.get('communities', []) == ['c3', 'grants_comm']
assert deposit_v1.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded',
'grants_comm', 'zenodo']
assert deposit_v2.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded',
'grants_comm', 'zenodo']
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
c3_api = ZenodoCommunity('c3')
grants_comm_api = ZenodoCommunity('grants_comm')
ecfunded_api = ZenodoCommunity('ecfunded')
zenodo_api = ZenodoCommunity('zenodo')
# Inclusion requests should be visible for both records
assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c3_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0
assert c3_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0
assert grants_comm_api.get_comm_irs(
record_v1, pid=recid_v1).count() == 0
assert grants_comm_api.get_comm_irs(
record_v2, pid=recid_v2).count() == 0
assert ecfunded_api.get_comm_irs(
record_v1, pid=recid_v1).count() == 1
assert ecfunded_api.get_comm_irs(
record_v2, pid=recid_v2).count() == 1
assert zenodo_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert zenodo_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
# Accept to 'c1' through record_v2 (as originally requested),
# and 'c2' through record_v1 (resolved through version)
c1_api.accept_record(record_v2, pid=recid_v2)
c2_api.accept_record(record_v1, pid=recid_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
# Accepting individual record to a community should propagate the changes
# to all versions
assert record_v1.get('communities', []) == ['c1', 'c2', 'c3',
'grants_comm']
assert record_v2.get('communities', []) == ['c1', 'c2', 'c3',
'grants_comm']
assert deposit_v1.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded',
'grants_comm', 'zenodo']
assert deposit_v2.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded',
'grants_comm', 'zenodo']
# Removing 'c1'-'c3' from deposit_v1 should remove it from two published
# records and other deposits as well
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1 = deposit_v1.edit()
deposit_v1['communities'] = []
deposit_v1 = publish_and_expunge(db, deposit_v1)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
assert record_v1.get('communities', []) == ['grants_comm', ]
assert record_v2.get('communities', []) == ['grants_comm', ]
assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
def test_relations_serialization(app, db, deposit, deposit_file):
"""Serialize PID relations."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
expected = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 1
}]
}
assert serialize_relations(recid_v1) == expected
deposit_v1.newversion()
# Should contain "draft_child_deposit" information
expected = {
"version": [{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 1,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert serialize_relations(recid_v1) == expected
# Publish the new version
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
# Should no longer contain "draft_child_deposit" info after publishing
# and no longer be the last child
expected = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}]
}
assert serialize_relations(recid_v1) == expected
# New version should be the last child now
expected = {
"version": [{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert serialize_relations(recid_v2) == expected
def test_versioning_indexing(db, es, deposit, deposit_file):
"""Test the indexing of 'version' relations."""
deposit_index_name = 'deposits-records-record-v1.0.0'
records_index_name = 'records-record-v1.0.0'
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
RecordIndexer().index_by_id(str(record_v1.id))
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 1
assert len(s_rec) == 1
assert 'relations' in s_dep[0]['_source']
assert 'relations' in s_rec[0]['_source']
expected = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 1,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert s_dep[0]['_source']['relations'] == expected
assert s_rec[0]['_source']['relations'] == expected
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid)
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 2 # Two deposits should be indexed
assert len(s_rec) == 1 # One, since record does not exist yet
s_dep1 = current_search.client.get(index=deposit_index_name,
id=deposit_v1.id)
s_dep2 = current_search.client.get(index=deposit_index_name,
id=deposit_v2.id)
expected_d1 = {
"version": [{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2 # For deposit, draft children are also counted
}]
}
expected_d2 = {
"version": [{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 2, # For deposit, draft children are also counted
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert s_dep1['_source']['relations'] == expected_d1
assert s_dep2['_source']['relations'] == expected_d2
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
RecordIndexer().index_by_id(str(record_v2.id))
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 2
assert len(s_rec) == 2
s_dep1 = current_search.client.get(index=deposit_index_name,
id=deposit_v1.id)
s_dep2 = current_search.client.get(index=deposit_index_name,
id=deposit_v2.id)
s_rec1 = current_search.client.get(index=records_index_name,
id=record_v1.id)
s_rec2 = current_search.client.get(index=records_index_name,
id=record_v2.id)
expected_d1 = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}]
}
expected_d2 = {
"version": [{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert s_dep1['_source']['relations'] == expected_d1
assert s_dep2['_source']['relations'] == expected_d2
expected_r1 = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}]
}
expected_r2 = {
"version": [{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert s_rec1['_source']['relations'] == expected_r1
assert s_rec2['_source']['relations'] == expected_r2
def delete_record(record_uuid, reason, user):
"""Delete the record and it's PIDs.
:param record_uuid: UUID of the record to be removed.
:param reason: Reason for removal. Either one of: 'spam', 'uploader',
'takedown' (see 'ZENODO_REMOVAL_REASONS' variable in config),
otherwise using it as a verbatim "Reason" string.
:param user: ID or email of the Zenodo user (admin)
responsible for the removal.
"""
from invenio_github.models import ReleaseStatus
if isinstance(user, text_type):
user_id = User.query.filter_by(email=user).one().id
elif isinstance(user, int):
user_id = User.query.get(user).id
else:
raise TypeError("User cannot be determined from argument: {0}".format(
user))
record = ZenodoRecord.get_record(record_uuid)
# Remove the record from versioning and delete the recid
recid = PersistentIdentifier.get('recid', record['recid'])
pv = PIDVersioning(child=recid)
pv.remove_child(recid)
pv.update_redirect()
recid.delete()
# Remove the record from index
try:
RecordIndexer().delete(record)
except NotFoundError:
pass
# Remove buckets
record_bucket = record.files.bucket
RecordsBuckets.query.filter_by(record_id=record.id).delete()
record_bucket.locked = False
record_bucket.remove()
removal_reasons = dict(current_app.config['ZENODO_REMOVAL_REASONS'])
if reason in removal_reasons:
reason = removal_reasons[reason]
depid, deposit = deposit_resolver.resolve(record['_deposit']['id'])
try:
doi = PersistentIdentifier.get('doi', record['doi'])
except PIDDoesNotExistError:
doi = None
# Record OpenAIRE info
try:
original_id = openaire_original_id(record, openaire_type(record))[1]
datasource_id = openaire_datasource_id(record)
except PIDDoesNotExistError:
original_id = None
datasource_id = None
if pv.children.count() == 0:
conceptrecid = PersistentIdentifier.get('recid',
record['conceptrecid'])
conceptrecid.delete()
new_last_child = None
else:
new_last_child = (pv.last_child.pid_value,
str(pv.last_child.object_uuid))
if 'conceptdoi' in record:
conceptdoi_value = record['conceptdoi']
else:
conceptdoi_value = None
# Completely delete the deposit
# Deposit will be removed from index
deposit.delete(delete_published=True)
# Clear the record and put the deletion information
record.clear()
record.update({
'removal_reason': reason,
'removed_by': user_id,
})
record.commit()
# Mark the relevant GitHub Release as deleted
for ghr in record.model.github_releases:
ghr.status = ReleaseStatus.DELETED
db.session.commit()
# After successful DB commit, sync the DOIs with DataCite
datacite_inactivate.delay(doi.pid_value)
if conceptdoi_value:
if new_last_child:
# Update last child (updates also conceptdoi)
pid_value, rec_uuid = new_last_child
datacite_register.delay(pid_value, rec_uuid)
else:
datacite_inactivate.delay(conceptdoi_value)
# Also delete from OpenAIRE index
if current_app.config['OPENAIRE_DIRECT_INDEXING_ENABLED'] and original_id \
and datasource_id:
openaire_delete.delay(original_id=original_id,
datasource_id=datasource_id)
def test_extra_formats_buckets(
api, api_client, db, es, locations, json_extra_auth_headers,
deposit_url, get_json, extra_auth_headers, json_headers,
license_record, communities, resolver, minimal_deposit):
"""Test simple flow using REST API."""
headers = json_extra_auth_headers
client = api_client
test_data = minimal_deposit
# Create deposit
response = client.post(
deposit_url, json=test_data, headers=headers)
data = get_json(response, code=201)
# Get identifier and links
depid = data['record_id']
links = data['links']
# Upload 1 files
response = client.put(
links['bucket'] + '/test1.txt',
data='ctx',
headers=extra_auth_headers,
)
assert response.status_code == 200
# Check for extra_formats bucket
response = api_client.options(
extra_formats_urls['deposit'].format(depid), headers=headers)
data = get_json(response, code=200)
# Check that no extra_formats bucket is present
buckets = Bucket.query.all()
assert len(buckets) == 1
# There are no extra_formats files
assert data == []
use_extra_formats_functions(
extra_auth_headers, api_client, get_json, depid=depid)
buckets = Bucket.query.all()
assert len(buckets) == 2
deposit = deposit_resolver.resolve(depid)[1]
assert deposit['_buckets']['extra_formats'] == \
str(deposit.extra_formats.bucket.id)
# Publish deposition
response = client.post(links['publish'], headers=extra_auth_headers)
data = get_json(response, code=202)
first_version_recid = data['record_id']
# Get the list of the extra_formats files attached to this deposit
response = api_client.options(
extra_formats_urls['record'].format(first_version_recid))
data = get_json(response, code=200)
assert data[0]['key'] == 'application/foo+xml'
assert len(data) == 1
# Test actions and clear extra_formats bucket
use_extra_formats_functions(extra_auth_headers, api_client, get_json,
depid=depid, recid=first_version_recid)
# Get newversion url
data = get_json(
client.get(links['self'], headers=extra_auth_headers), code=200
)
new_version_url = data['links']['newversion']
# New Version
data = get_json(
client.post(new_version_url, headers=extra_auth_headers), code=201)
links = data['links']
# Get the list of the extra_formats files attached to the new deposit
# Should be the same with the previous version
response = api_client.options(
extra_formats_urls['deposit'].format(depid),
headers=extra_auth_headers
)
data = get_json(response, code=200)
assert data[0]['key'] == 'application/foo+xml'
assert len(data) == 1
# Get latest version
data = get_json(
client.get(links['latest_draft'], headers=extra_auth_headers),
code=200)
links = data['links']
depid = data['record_id']
# Add a file to the new deposit
get_json(client.put(
links['bucket'] + '/newfile.txt',
data='newfile',
headers=extra_auth_headers,
), code=200)
# Publish the new record
response = client.post(links['publish'], headers=extra_auth_headers)
data = get_json(response, code=202)
links = data['links']
recid = data['record_id']
# Get the list of the extra_formats files attached to the new record
response = api_client.options(extra_formats_urls['record'].format(recid))
data = get_json(response, code=200)
assert data[0]['key'] == 'application/foo+xml'
assert len(data) == 1
# Add file to extra_formats bucket
response = api_client.put(
extra_formats_urls['deposit'].format(recid),
data='bar file content',
headers=extra_formats_headers['bar'] + extra_auth_headers
)
data = get_json(response, code=200)
assert data['message'] == 'Extra format "application/bar+xml" updated.'
# Get the list of the extra_formats files attached to the new record
response = api_client.options(extra_formats_urls['record'].format(recid))
data = get_json(response, code=200)
assert {f['key'] for f in data} == \
{'application/foo+xml', 'application/bar+xml'}
# Get the list of the extra_formats files attached to the previous record
# Make sure that the snapshots are independent
response = api_client.options(
extra_formats_urls['record'].format(first_version_recid))
data = get_json(response, code=200)
first_record = record_resolver.resolve(first_version_recid)[1]
new_record = record_resolver.resolve(recid)[1]
assert first_record.extra_formats.bucket.id != \
new_record.extra_formats.bucket.id
assert data[0]['key'] == 'application/foo+xml'
assert len(data) == 1
# Test actions and clear extra_formats bucket of deposit
use_extra_formats_functions(
extra_auth_headers, api_client, get_json, depid=depid, recid=recid)
def test_extra_formats_buckets(api, api_client, db, es, locations,
json_extra_auth_headers, deposit_url, get_json,
extra_auth_headers, json_headers,
license_record, communities, resolver,
minimal_deposit):
"""Test simple flow using REST API."""
headers = json_extra_auth_headers
client = api_client
test_data = minimal_deposit
# Create deposit
response = client.post(deposit_url, json=test_data, headers=headers)
data = get_json(response, code=201)
# Get identifier and links
depid = data['record_id']
links = data['links']
# Upload 1 files
response = client.put(
links['bucket'] + '/test1.txt',
data='ctx',
headers=extra_auth_headers,
)
assert response.status_code == 200
# Check for extra_formats bucket
response = api_client.options(extra_formats_urls['deposit'].format(depid),
headers=headers)
data = get_json(response, code=200)
# Check that no extra_formats bucket is present
buckets = Bucket.query.all()
assert len(buckets) == 1
# There are no extra_formats files
assert data == []
use_extra_formats_functions(extra_auth_headers,
api_client,
get_json,
depid=depid)
buckets = Bucket.query.all()
assert len(buckets) == 2
deposit = deposit_resolver.resolve(depid)[1]
assert deposit['_buckets']['extra_formats'] == \
str(deposit.extra_formats.bucket.id)
# Publish deposition
response = client.post(links['publish'], headers=extra_auth_headers)
data = get_json(response, code=202)
first_version_recid = data['record_id']
# Get the list of the extra_formats files attached to this deposit
response = api_client.options(
extra_formats_urls['record'].format(first_version_recid))
data = get_json(response, code=200)
assert data[0]['key'] == 'application/foo+xml'
assert len(data) == 1
# Test actions and clear extra_formats bucket
use_extra_formats_functions(extra_auth_headers,
api_client,
get_json,
depid=depid,
recid=first_version_recid)
# Get newversion url
data = get_json(client.get(links['self'], headers=extra_auth_headers),
code=200)
new_version_url = data['links']['newversion']
# New Version
data = get_json(client.post(new_version_url, headers=extra_auth_headers),
code=201)
links = data['links']
# Get the list of the extra_formats files attached to the new deposit
# Should be the same with the previous version
response = api_client.options(extra_formats_urls['deposit'].format(depid),
headers=extra_auth_headers)
data = get_json(response, code=200)
assert data[0]['key'] == 'application/foo+xml'
assert len(data) == 1
# Get latest version
data = get_json(client.get(links['latest_draft'],
headers=extra_auth_headers),
code=200)
links = data['links']
depid = data['record_id']
# Add a file to the new deposit
get_json(client.put(
links['bucket'] + '/newfile.txt',
data='newfile',
headers=extra_auth_headers,
),
code=200)
# Publish the new record
response = client.post(links['publish'], headers=extra_auth_headers)
data = get_json(response, code=202)
links = data['links']
recid = data['record_id']
# Get the list of the extra_formats files attached to the new record
response = api_client.options(extra_formats_urls['record'].format(recid))
data = get_json(response, code=200)
assert data[0]['key'] == 'application/foo+xml'
assert len(data) == 1
# Add file to extra_formats bucket
response = api_client.put(extra_formats_urls['deposit'].format(recid),
data='bar file content',
headers=extra_formats_headers['bar'] +
extra_auth_headers)
data = get_json(response, code=200)
assert data['message'] == 'Extra format "application/bar+xml" updated.'
# Get the list of the extra_formats files attached to the new record
response = api_client.options(extra_formats_urls['record'].format(recid))
data = get_json(response, code=200)
assert {f['key'] for f in data} == \
{'application/foo+xml', 'application/bar+xml'}
# Get the list of the extra_formats files attached to the previous record
# Make sure that the snapshots are independent
response = api_client.options(
extra_formats_urls['record'].format(first_version_recid))
data = get_json(response, code=200)
first_record = record_resolver.resolve(first_version_recid)[1]
new_record = record_resolver.resolve(recid)[1]
assert first_record.extra_formats.bucket.id != \
new_record.extra_formats.bucket.id
assert data[0]['key'] == 'application/foo+xml'
assert len(data) == 1
# Test actions and clear extra_formats bucket of deposit
use_extra_formats_functions(extra_auth_headers,
api_client,
get_json,
depid=depid,
recid=recid)
def test_versioning_indexing(db, es, deposit, deposit_file):
"""Test the indexing of 'version' relations."""
deposit_index_name = 'deposits-records-record-v1.0.0'
records_index_name = 'records-record-v1.0.0'
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
RecordIndexer().index_by_id(str(record_v1.id))
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 1
assert len(s_rec) == 1
assert 'relations' in s_dep[0]['_source']
assert 'relations' in s_rec[0]['_source']
expected = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 1,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert s_dep[0]['_source']['relations'] == expected
assert s_rec[0]['_source']['relations'] == expected
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid)
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 2 # Two deposits should be indexed
assert len(s_rec) == 1 # One, since record does not exist yet
s_dep1 = current_search.client.get(
index=deposit_index_name, id=deposit_v1.id)
s_dep2 = current_search.client.get(
index=deposit_index_name, id=deposit_v2.id)
expected_d1 = {
"version": [
{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2 # For deposit, draft children are also counted
}
]
}
expected_d2 = {
"version": [
{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 2, # For deposit, draft children are also counted
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert s_dep1['_source']['relations'] == expected_d1
assert s_dep2['_source']['relations'] == expected_d2
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
RecordIndexer().index_by_id(str(record_v2.id))
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 2
assert len(s_rec) == 2
s_dep1 = current_search.client.get(
index=deposit_index_name, id=deposit_v1.id)
s_dep2 = current_search.client.get(
index=deposit_index_name, id=deposit_v2.id)
s_rec1 = current_search.client.get(
index=records_index_name, id=record_v1.id)
s_rec2 = current_search.client.get(
index=records_index_name, id=record_v2.id)
expected_d1 = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}
]
}
expected_d2 = {
"version": [
{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert s_dep1['_source']['relations'] == expected_d1
assert s_dep2['_source']['relations'] == expected_d2
expected_r1 = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}
]
}
expected_r2 = {
"version": [
{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert s_rec1['_source']['relations'] == expected_r1
assert s_rec2['_source']['relations'] == expected_r2
def test_related_identifiers_serialization(app, db, deposit, deposit_file):
"""Serialize PID Relations to related identifiers."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c2' through deposit v2
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
rids = serialize_related_identifiers(recid_v1)
expected_v1 = [
{
'scheme': 'doi',
'identifier': '10.5072/zenodo.1',
'relation': 'isPartOf'
}
# TODO: serialization of new version realtions is disabled
# {
# 'scheme': 'doi',
# 'identifier': '10.5072/zenodo.3',
# 'relation': 'isPreviousVersionOf'
# }
]
assert rids == expected_v1
rids = serialize_related_identifiers(recid_v2)
expected_v2 = [
{
'scheme': 'doi',
'identifier': '10.5072/zenodo.1',
'relation': 'isPartOf'
}
# TODO: serialization of new version realtions is disabled
# {
# 'scheme': 'doi',
# 'identifier': '10.5072/zenodo.2',
# 'relation': 'isNewVersionOf'
# }
]
assert rids == expected_v2
parent_pid = PersistentIdentifier.get('recid', '1')
rids = serialize_related_identifiers(parent_pid)
expected_parent = [{
'relation': 'hasPart',
'scheme': 'doi',
'identifier': '10.5072/zenodo.2'
}, {
'relation': 'hasPart',
'scheme': 'doi',
'identifier': '10.5072/zenodo.3'
}]
assert rids == expected_parent
