def test_autoadd_explicit_newversion(
app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Explicitly the autoadded communities in a new version."""
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2['communities'] = ['ecfunded', 'grants_comm', 'zenodo']
deposit_v2['grants'] = [{'title': 'SomeGrant'}, ]
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
assert record_v1.get('communities', []) == ['grants_comm', ]
assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
assert record_v2.get('communities', []) == ['grants_comm', ]
assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
def test_fixed_autoadd_edit(app, db, users, communities, deposit,
deposit_file, communities_autoadd_enabled):
"""Test automatic adding and requesting to fixed communities.
Add to grants_comm also after later addition of grant information.
"""
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['zenodo', ]
assert 'communities' not in record
ir = InclusionRequest.query.one()
assert ir.id_community == 'zenodo'
assert ir.id_record == record.id
deposit = deposit.edit()
deposit['grants'] = [{'title': 'SomeGrant'}, ]
# Requesting for 'grants_comm' and 'ecfunded' manually even though it will
# be added due to specifying 'grants' shouldn't cause problems
deposit['communities'] = ['ecfunded', 'grants_comm', 'zenodo']
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
InclusionRequest.query.count() == 2
ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one()
assert ir1.id_record == record.id
ir2 = InclusionRequest.query.filter_by(id_community='ecfunded').one()
assert ir2.id_record == record.id
assert deposit['communities'] == ['ecfunded', 'grants_comm', 'zenodo', ]
assert record['communities'] == ['grants_comm', ]
def test_autoaccept_owned_communities(app, db, users, communities, deposit,
deposit_file):
"""Automatically accept records requested by community owners."""
# 'c3' is owned by the user, but not 'c1'
deposit['communities'] = ['c1', 'c3', ]
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', 'c3', ]
assert record['communities'] == ['c3', ]
assert record['_oai']['sets'] == ['user-c3']
assert InclusionRequest.query.count() == 1
ir = InclusionRequest.query.one()
assert ir.id_community == 'c1'
assert ir.id_record == record.id
# Edit the deposit, and add more communities
# 'c4' should be added automatically, but not 'c2'
deposit = deposit.edit()
deposit['communities'] = ['c1', 'c2', 'c3', 'c4', ]
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', 'c2', 'c3', 'c4', ]
assert record['communities'] == ['c3', 'c4', ]
assert set(record['_oai']['sets']) == set(['user-c3', 'user-c4'])
assert InclusionRequest.query.count() == 2
ir1 = InclusionRequest.query.filter_by(id_community='c1').one()
ir2 = InclusionRequest.query.filter_by(id_community='c2').one()
assert ir1.id_record == record.id
assert ir2.id_record == record.id
def test_autoadd_explicit_newversion(
app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Explicitly the autoadded communities in a new version."""
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2['communities'] = ['ecfunded', 'grants_comm', 'zenodo']
deposit_v2['grants'] = [{'title': 'SomeGrant'}, ]
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
assert record_v1.get('communities', []) == ['grants_comm', ]
assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
assert record_v2.get('communities', []) == ['grants_comm', ]
assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
def test_deposit_versioning_draft_child_unlinking_bug(
app, db, communities, deposit, deposit_file):
"""
Bug with draft_child_deposit unlinking.
Bug where a draft_child_deposit was unlinked from a new version draft,
when another version of a record was edited and published.
"""
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit.fetch_published()
recid_v1_value = recid_v1.pid_value
# Initiate a new version draft
deposit_v1.newversion()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
pv = PIDVersioning(child=recid_v1)
assert pv.draft_child_deposit
assert pv.draft_child
deposit_v1.edit()
deposit_v1 = deposit_v1.edit()
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
pv = PIDVersioning(child=recid_v1)
# Make sure the draft child deposit was not unliked due to publishing of
# the edited draft
assert pv.draft_child_deposit
assert pv.draft_child
def test_deposit_versioning_draft_child_unlinking_bug(app, db, communities,
deposit, deposit_file):
"""
Bug with draft_child_deposit unlinking.
Bug where a draft_child_deposit was unlinked from a new version draft,
when another version of a record was edited and published.
"""
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit.fetch_published()
recid_v1_value = recid_v1.pid_value
# Initiate a new version draft
deposit_v1.newversion()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
pv = PIDVersioning(child=recid_v1)
assert pv.draft_child_deposit
assert pv.draft_child
deposit_v1.edit()
deposit_v1 = deposit_v1.edit()
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
pv = PIDVersioning(child=recid_v1)
# Make sure the draft child deposit was not unliked due to publishing of
# the edited draft
assert pv.draft_child_deposit
assert pv.draft_child
def test_propagation_with_newversion_open(
app, db, users, communities, deposit, deposit_file):
"""Adding old versions to a community should propagate to all drafts."""
# deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
deposit_v1 = deposit_v1.edit()
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
# New version in 'deposit_v2' has not been published yet
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
# depid_v1_value = deposit_v1['_deposit']['id']
# depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1['communities'] = ['c1', 'c2', ]
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
c1_api = ZenodoCommunity('c1')
c1_api.accept_record(record_v1, pid=recid_v1)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
assert deposit_v2['communities'] == ['c1', 'c2']
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
assert record_v2['communities'] == ['c1', ]
def test_record_modified_while_edit(app, db, communities, deposit,
deposit_file):
"""Test deposit publishing with concurrent events.
Modify a record, while deposit in open edit and then published.
"""
deposit['communities'] = ['c1', ]
deposit = publish_and_expunge(db, deposit)
assert InclusionRequest.query.count() == 1
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', ]
assert not record.get('communities', [])
# Open for edit
deposit = deposit.edit()
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', ]
assert not record.get('communities', [])
assert InclusionRequest.query.count() == 1
# Meanwhile, a record is modified
record['title'] = 'Other title'
record.commit()
db.session.commit()
# Publish and make sure nothing is missing
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', ]
assert not record.get('communities', [])
assert InclusionRequest.query.count() == 1
ir = InclusionRequest.query.one()
assert ir.id_community == 'c1'
assert ir.id_record == record.id
def test_propagation_with_newversion_open(
app, db, users, communities, deposit, deposit_file):
"""Adding old versions to a community should propagate to all drafts."""
# deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
deposit_v1 = deposit_v1.edit()
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
# New version in 'deposit_v2' has not been published yet
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
# depid_v1_value = deposit_v1['_deposit']['id']
# depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1['communities'] = ['c1', 'c2', ]
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
c1_api = ZenodoCommunity('c1')
c1_api.accept_record(record_v1, pid=recid_v1)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
assert deposit_v2['communities'] == ['c1', 'c2']
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
assert record_v2['communities'] == ['c1', ]
def test_fixed_communities_edit(app, db, users, communities, deposit,
deposit_file, communities_autoadd_enabled):
"""Test automatic adding and requesting to fixed communities."""
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['zenodo', ]
assert 'communities' not in record
ir = InclusionRequest.query.one()
assert ir.id_community == 'zenodo'
assert ir.id_record == record.id
# Open for edit
deposit = deposit.edit()
# Make sure 'zenodo' community is requested
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['zenodo', ]
assert not record.get('communities', [])
assert InclusionRequest.query.count() == 1
comm = Community.get('zenodo')
comm.accept_record(record)
record.commit()
db.session.commit()
# Publish and make sure nothing is missing
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['zenodo']
assert record['communities'] == ['zenodo', ]
assert record['_oai']['sets'] == ['user-zenodo', ]
assert InclusionRequest.query.count() == 0
def test_basic_api(app, db, communities, deposit, deposit_file):
"""Test basic workflow using Deposit and Communities API."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c2' through deposit v2
deposit_v2['communities'] = [
'c1',
'c2',
]
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
recid_v2_value = recid_v2.pid_value
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
# Inclusion requests should be visible for both records
assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
# Accept to 'c1' through record_v2 (as originally requested),
# and 'c2' through record_v1 (version)
c1_api.accept_record(record_v2, pid=recid_v2)
c2_api.accept_record(record_v1, pid=recid_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
# Accepting individual record to a community should propagate the changes
# to all versions
assert record_v1['communities'] == record_v2['communities'] == \
['c1', 'c2', ]
# Removing 'c1' from deposit_v1 should remove it from two published records
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1 = deposit_v1.edit()
deposit_v1['communities'] = []
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
def test_record_delete_v2(mocker, app, db, users, deposit, deposit_file):
"""Delete a record (only last version) with multiple versions."""
dc_mock = mocker.patch(
'invenio_pidstore.providers.datacite.DataCiteMDSClient')
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1.newversion()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
# Stash a copy of v1 for later
rec1 = deepcopy(record_v1)
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
# Stash a copy of v2 for later
rec2 = deepcopy(record_v2)
rec2_id = str(record_v2.id)
assert dc_mock().metadata_delete.call_count == 0
# Remove the first version
delete_record(rec2_id, 'spam', users[0]['id'])
# Make sure all PIDs are deleted
assert PID.get('doi', rec2['doi']).status == PIDStatus.DELETED
assert PID.get('recid', rec2['recid']).status == PIDStatus.DELETED
assert PID.get('depid', rec2['_deposit']['id']).status == PIDStatus.DELETED
# Concept DOI should be left registered
assert PID.get('doi', rec2['conceptdoi']).status == PIDStatus.REGISTERED
# Make sure conceptrecid is redirecting to v1
crecid = PID.get('recid', rec2['conceptrecid'])
assert crecid.status == PIDStatus.REDIRECTED
assert crecid.get_redirect() == PID.get('recid', rec1['recid'])
# Make sure the v1 PIDs are kept intact
assert PID.get('oai', rec1['_oai']['id']).status == PIDStatus.REGISTERED
assert PID.get('doi', rec1['doi']).status == PIDStatus.REGISTERED
assert PID.get('recid', rec1['recid']).status == PIDStatus.REGISTERED
assert PID.get('depid', rec1['_deposit']['id']).status == \
PIDStatus.REGISTERED
# Only the v1 DOI should be deleted
assert dc_mock().doi_post.call_count == 2
assert dc_mock().doi_post.has_any_call('10.5072/zenodo.2')
assert dc_mock().doi_post.has_any_call('10.5072/zenodo.1')
assert dc_mock().metadata_delete.call_count == 1
dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.3')
record = Record.get_record(rec2_id)
assert record['removed_by'] == users[0]['id']
assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def test_nonexisting_communities(app, db, users, communities, deposit,
deposit_file):
"""Test adding nonexisting community."""
deposit['communities'] = ['nonexisting', ]
with pytest.raises(MissingCommunityError) as exc_info:
publish_and_expunge(db, deposit)
assert exc_info.value.errors[0].res == {
'message': 'Provided community does not exist: nonexisting',
'field': 'metadata.communities',
}
def test_basic_api(app, db, communities, deposit, deposit_file):
"""Test basic workflow using Deposit and Communities API."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c2' through deposit v2
deposit_v2['communities'] = ['c1', 'c2', ]
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
recid_v2_value = recid_v2.pid_value
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
# Inclusion requests should be visible for both records
assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
# Accept to 'c1' through record_v2 (as originally requested),
# and 'c2' through record_v1 (version)
c1_api.accept_record(record_v2, pid=recid_v2)
c2_api.accept_record(record_v1, pid=recid_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
# Accepting individual record to a community should propagate the changes
# to all versions
assert record_v1['communities'] == record_v2['communities'] == \
['c1', 'c2', ]
# Removing 'c1' from deposit_v1 should remove it from two published records
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1 = deposit_v1.edit()
deposit_v1['communities'] = []
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
def test_remove_community_by_key_del(app, db, communities, deposit,
deposit_file):
"""Test removal of communities by key deletion.
Communities can be removed by not providing or deleting the communities
from the key deposit. Moreover, the redundant 'empty' keys should not be
automatically added to deposit nor record.
"""
# If 'communities' key was not in deposit metadata,
# it shouldn't be automatically added
assert 'communities' not in deposit
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert 'communities' not in deposit
assert 'communities' not in record
assert not record['_oai'].get('sets', [])
# Request for 'c1' and 'c2'
deposit = deposit.edit()
deposit['communities'] = [
'c1',
'c2',
]
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
# No reason to have 'communities' in record since nothing was accepted
assert 'communities' not in record
assert not record['_oai'].get('sets', [])
# Accept 'c1'
c1 = Community.get('c1')
c1.accept_record(record)
record.commit()
pid, record = deposit.fetch_published()
assert deposit['communities'] == [
'c1',
'c2',
]
assert InclusionRequest.query.count() == 1
assert record['communities'] == [
'c1',
]
assert set(record['_oai']['sets']) == set(['user-c1'])
# Remove the key from deposit and publish
deposit = deposit.edit()
del deposit['communities']
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert 'communities' not in deposit
assert 'communities' not in record
assert InclusionRequest.query.count() == 0
assert not record['_oai'].get('sets', [])
def test_fixed_communities_grants(app, db, users, communities, deposit,
deposit_file, communities_autoadd_enabled):
"""Test automatic adding and requesting to fixed communities.
Add to grants_comm also after later addition of grant information.
"""
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['zenodo', ]
assert 'communities' not in record
ir = InclusionRequest.query.one()
assert ir.id_community == 'zenodo'
assert ir.id_record == record.id
deposit = deposit.edit()
deposit['grants'] = [{'title': 'SomeGrant'}, ]
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['ecfunded', 'grants_comm', 'zenodo', ]
assert record['communities'] == ['grants_comm', ]
InclusionRequest.query.count() == 2
ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one()
assert ir1.id_record == record.id
ir2 = InclusionRequest.query.filter_by(id_community='ecfunded').one()
assert ir2.id_record == record.id
# Remove 'grants' without auto requested community being accepted.
# We should not remove the inclusion request as we don't know if user
# requested it manually or whether it was an automatic request
deposit = deposit.edit()
deposit['grants'] = []
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['ecfunded', 'grants_comm', 'zenodo', ]
assert record['communities'] == ['grants_comm', ]
InclusionRequest.query.count() == 2
ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one()
assert ir1.id_record == record.id
ir2 = InclusionRequest.query.filter_by(id_community='ecfunded').one()
assert ir2.id_record == record.id
# However, if user explicitly removed auto-requested community, and grants
# have been removed too, the IR should be removed.
deposit = deposit.edit()
deposit['grants'] = []
# Removed 'ecfunded' and 'grants_comm' from deposit
deposit['communities'] = ['zenodo', ]
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['zenodo', ]
assert 'communities' not in record
InclusionRequest.query.count() == 1
ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one()
assert ir1.id_record == record.id
def test_nonexisting_communities(app, db, users, communities, deposit,
deposit_file):
"""Test adding nonexisting community."""
deposit['communities'] = [
'nonexisting',
]
with pytest.raises(MissingCommunityError) as exc_info:
publish_and_expunge(db, deposit)
assert exc_info.value.errors[0].res == {
'message': 'Provided community does not exist: nonexisting',
'field': 'metadata.communities',
}
def test_basic_deposit_edit(app, db, communities, deposit, deposit_file):
"""Test simple deposit publishing."""
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
initial_oai = deepcopy(record['_oai'])
# Create some potential corruptions to protected fields
deposit = deposit.edit()
deposit['_files'][0]['bucket'] = record['_buckets']['deposit']
deposit['_oai'] = {}
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert record['_oai'] == initial_oai
assert record['_files'][0]['bucket'] == record['_buckets']['record']
def test_basic_deposit_edit(app, db, communities, deposit, deposit_file):
"""Test simple deposit publishing."""
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
initial_oai = deepcopy(record['_oai'])
# Create some potential corruptions to protected fields
deposit = deposit.edit()
deposit['_files'][0]['bucket'] = record['_buckets']['deposit']
deposit['_oai'] = {}
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert record['_oai'] == initial_oai
assert record['_files'][0]['bucket'] == record['_buckets']['record']
def test_communities_newversion_while_ir_pending_bug(app, db, users,
communities, deposit,
deposit_file):
"""Make sure that pending IRs remain after a new version (bug)."""
deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
# Two inclusion requests are pending
assert InclusionRequest.query.count() == 2
# Accept one community
c1_api = ZenodoCommunity('c1')
c1_api.accept_record(record_v1, pid=recid_v1)
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
# Make sure there is still IR to community 'c2' after newversion
assert InclusionRequest.query.count() == 1
assert InclusionRequest.query.one().id_community == 'c2'
assert record_v1.get('communities', []) == [
'c1',
]
assert deposit_v1.get('communities', []) == [
'c1',
'c2',
]
assert record_v2.get('communities', []) == [
'c1',
]
assert deposit_v2.get('communities', []) == [
'c1',
'c2',
]
def test_record_delete(mocker, app, db, users, deposit, deposit_file):
"""Delete the record with a single version."""
dc_mock = mocker.patch(
'invenio_pidstore.providers.datacite.DataCiteMDSClient')
deposit = publish_and_expunge(db, deposit)
recid, record = deposit.fetch_published()
# Stash a copy of record metadata for later
rec = deepcopy(record)
record_uuid = str(record.id)
assert dc_mock().metadata_delete.call_count == 0
# users[0] is not an Admin but it doesn't matter in this case.
delete_record(record.id, 'spam', users[0]['id'])
# Make sure all PIDs are deleted
# TODO: oai PID is left registered
# assert PID.get('oai', rec['_oai']['id']) == PIDStatus.DELETED
assert PID.get('doi', rec['doi']).status == PIDStatus.DELETED
assert PID.get('doi', rec['conceptdoi']).status == PIDStatus.DELETED
assert PID.get('recid', rec['recid']).status == PIDStatus.DELETED
assert PID.get('recid', rec['conceptrecid']).status == PIDStatus.DELETED
assert PID.get('depid', rec['_deposit']['id']).status == PIDStatus.DELETED
assert dc_mock().metadata_delete.call_count == 2
dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.1')
dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.2')
record = Record.get_record(record_uuid)
assert record['removed_by'] == users[0]['id']
assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def test_record_delete_legacy(dc_mock, app, db, users, deposit, deposit_file):
"""Delete the non-versioned record."""
deposit = publish_and_expunge(db, deposit)
recid, record = deposit.fetch_published()
# 'Simulate' a non-versioned record by removing 'conceptdoi' key
del deposit['conceptdoi']
del record['conceptdoi']
deposit.commit()
record.commit()
db.session.commit()
# Stash a copy of record metadata for later
rec = deepcopy(record)
record_uuid = str(record.id)
assert dc_mock().metadata_delete.call_count == 0
# users[0] is not an Admin but it doesn't matter in this case.
delete_record(record.id, 'spam', users[0]['id'])
# Make sure all PIDs are deleted
# TODO: oai PID is left registered
# assert PID.get('oai', rec['_oai']['id']) == PIDStatus.DELETED
assert PID.get('doi', rec['doi']).status == PIDStatus.DELETED
assert PID.get('recid', rec['recid']).status == PIDStatus.DELETED
assert PID.get('recid', rec['conceptrecid']).status == PIDStatus.DELETED
assert PID.get('depid', rec['_deposit']['id']).status == PIDStatus.DELETED
assert dc_mock().metadata_delete.call_count == 1
dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.2')
record = Record.get_record(record_uuid)
assert record['removed_by'] == users[0]['id']
assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def test_communities_newversion_addition(app, db, users, communities, deposit,
deposit_file):
"""Make sure that new version of record synchronizes the communities."""
deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
c1_api.accept_record(record_v1, pid=recid_v1)
c2_api.accept_record(record_v1, pid=recid_v1)
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
# Remove 'c2' and request for 'c5'. Make sure that communities from
# previous record version are preserved/removed properly
deposit_v2['communities'] = ['c1', 'c5']
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
assert record_v1.get('communities', []) == [
'c1',
]
assert deposit_v1.get('communities', []) == [
'c1',
'c5',
]
assert record_v2.get('communities', []) == [
'c1',
]
assert deposit_v2.get('communities', []) == [
'c1',
'c5',
]
def test_remove_community_by_key_del(app, db, communities, deposit,
deposit_file):
"""Test removal of communities by key deletion.
Communities can be removed by not providing or deleting the communities
from the key deposit. Moreover, the redundant 'empty' keys should not be
automatically added to deposit nor record.
"""
# If 'communities' key was not in deposit metadata,
# it shouldn't be automatically added
assert 'communities' not in deposit
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert 'communities' not in deposit
assert 'communities' not in record
assert not record['_oai'].get('sets', [])
# Request for 'c1' and 'c2'
deposit = deposit.edit()
deposit['communities'] = ['c1', 'c2', ]
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
# No reason to have 'communities' in record since nothing was accepted
assert 'communities' not in record
assert not record['_oai'].get('sets', [])
# Accept 'c1'
c1 = Community.get('c1')
c1.accept_record(record)
record.commit()
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', 'c2', ]
assert InclusionRequest.query.count() == 1
assert record['communities'] == ['c1', ]
assert set(record['_oai']['sets']) == set(['user-c1'])
# Remove the key from deposit and publish
deposit = deposit.edit()
del deposit['communities']
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert 'communities' not in deposit
assert 'communities' not in record
assert InclusionRequest.query.count() == 0
assert not record['_oai'].get('sets', [])
def test_remove_obsolete_irs(app, db, communities, deposit, deposit_file):
"""Test removal of obsolete IRs in-between deposit edits."""
# Request for 'c1'
deposit['communities'] = ['c1', ]
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert InclusionRequest.query.count() == 1
assert deposit['communities'] == ['c1', ]
assert not record.get('communities', [])
# Open for edit and remove the request to community 'c1'
deposit = deposit.edit()
deposit['communities'] = []
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert InclusionRequest.query.count() == 0
assert not deposit.get('communities', [])
assert not record.get('communities', [])
def test_accept_while_edit(app, db, communities, deposit, deposit_file):
"""Test deposit publishing with concurrent events.
Accept a record, while deposit in open edit and then published.
"""
deposit['communities'] = ['c1', 'c2']
deposit = publish_and_expunge(db, deposit)
assert InclusionRequest.query.count() == 2
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', 'c2']
assert not record.get('communities', [])
assert not record['_oai'].get('sets', [])
# Open for edit
deposit = deposit.edit()
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', 'c2']
assert not record.get('communities', [])
assert not record['_oai'].get('sets', [])
assert InclusionRequest.query.count() == 2
# Accept a record meanwhile
c1 = Community.get('c1')
c1.accept_record(record)
record.commit()
db.session.commit()
# Publish and make sure nothing is missing
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', 'c2']
assert record['communities'] == [
'c1',
]
assert record['_oai']['sets'] == [
'user-c1',
]
assert InclusionRequest.query.count() == 1
ir = InclusionRequest.query.one()
assert ir.id_community == 'c2'
assert ir.id_record == record.id
def test_autoadd_explicit(
app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Explicitly the autoadded communities."""
deposit['communities'] = ['ecfunded', 'grants_comm', 'zenodo']
deposit['grants'] = [{'title': 'SomeGrant'}, ]
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == ['grants_comm', ]
assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
def test_autoadd_explicit(
app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Explicitly the autoadded communities."""
deposit['communities'] = ['ecfunded', 'grants_comm', 'zenodo']
deposit['grants'] = [{'title': 'SomeGrant'}, ]
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == ['grants_comm', ]
assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
def test_reject_while_edit(app, db, communities, deposit, deposit_file):
"""Test deposit publishing with concurrent events.
Reject a record, while deposit in open edit and published.
"""
# Request for community 'c1'
deposit['communities'] = [
'c1',
]
deposit = publish_and_expunge(db, deposit)
assert deposit['communities'] == [
'c1',
]
pid, record = deposit.fetch_published()
assert not record.get('communities', [])
assert InclusionRequest.query.count() == 1
ir = InclusionRequest.query.one()
assert ir.id_community == 'c1'
assert ir.id_record == record.id
# Open deposit in edit mode and request another community 'c2'
deposit = deposit.edit()
deposit['communities'] = ['c1', 'c2']
# Reject the request for community 'c1'
c1 = Community.get('c1')
c1.reject_record(record)
db.session.commit()
# Publish the deposit
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
# NOTE: 'c1' is requested again!
assert InclusionRequest.query.count() == 2
ir1 = InclusionRequest.query.filter_by(id_community='c1').one()
ir2 = InclusionRequest.query.filter_by(id_community='c2').one()
assert ir1.id_record == record.id
assert ir2.id_record == record.id
assert deposit['communities'] == ['c1', 'c2']
assert not record.get('communities', [])
def test_accept_while_edit(app, db, communities, deposit, deposit_file):
"""Test deposit publishing with concurrent events.
Accept a record, while deposit in open edit and then published.
"""
deposit['communities'] = ['c1', 'c2']
deposit = publish_and_expunge(db, deposit)
assert InclusionRequest.query.count() == 2
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', 'c2']
assert not record.get('communities', [])
assert not record['_oai'].get('sets', [])
# Open for edit
deposit = deposit.edit()
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', 'c2']
assert not record.get('communities', [])
assert not record['_oai'].get('sets', [])
assert InclusionRequest.query.count() == 2
# Accept a record meanwhile
c1 = Community.get('c1')
c1.accept_record(record)
record.commit()
db.session.commit()
# Publish and make sure nothing is missing
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert deposit['communities'] == ['c1', 'c2']
assert record['communities'] == ['c1', ]
assert record['_oai']['sets'] == ['user-c1', ]
assert InclusionRequest.query.count() == 1
ir = InclusionRequest.query.one()
assert ir.id_community == 'c2'
assert ir.id_record == record.id
def test_communities_newversion_while_ir_pending_bug(
app, db, users, communities, deposit, deposit_file):
"""Make sure that pending IRs remain after a new version (bug)."""
deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
# Two inclusion requests are pending
assert InclusionRequest.query.count() == 2
# Accept one community
c1_api = ZenodoCommunity('c1')
c1_api.accept_record(record_v1, pid=recid_v1)
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
# Make sure there is still IR to community 'c2' after newversion
assert InclusionRequest.query.count() == 1
assert InclusionRequest.query.one().id_community == 'c2'
assert record_v1.get('communities', []) == ['c1', ]
assert deposit_v1.get('communities', []) == ['c1', 'c2', ]
assert record_v2.get('communities', []) == ['c1', ]
assert deposit_v2.get('communities', []) == ['c1', 'c2', ]
def test_communities_newversion_addition(
app, db, users, communities, deposit, deposit_file):
"""Make sure that new version of record synchronizes the communities."""
deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
c1_api.accept_record(record_v1, pid=recid_v1)
c2_api.accept_record(record_v1, pid=recid_v1)
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
# Remove 'c2' and request for 'c5'. Make sure that communities from
# previous record version are preserved/removed properly
deposit_v2['communities'] = ['c1', 'c5']
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
assert record_v1.get('communities', []) == ['c1', ]
assert deposit_v1.get('communities', []) == ['c1', 'c5', ]
assert record_v2.get('communities', []) == ['c1', ]
assert deposit_v2.get('communities', []) == ['c1', 'c5', ]
def test_reject_while_edit(app, db, communities, deposit, deposit_file):
"""Test deposit publishing with concurrent events.
Reject a record, while deposit in open edit and published.
"""
# Request for community 'c1'
deposit['communities'] = ['c1', ]
deposit = publish_and_expunge(db, deposit)
assert deposit['communities'] == ['c1', ]
pid, record = deposit.fetch_published()
assert not record.get('communities', [])
assert InclusionRequest.query.count() == 1
ir = InclusionRequest.query.one()
assert ir.id_community == 'c1'
assert ir.id_record == record.id
# Open deposit in edit mode and request another community 'c2'
deposit = deposit.edit()
deposit['communities'] = ['c1', 'c2']
# Reject the request for community 'c1'
c1 = Community.get('c1')
c1.reject_record(record)
db.session.commit()
# Publish the deposit
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
# NOTE: 'c1' is requested again!
assert InclusionRequest.query.count() == 2
ir1 = InclusionRequest.query.filter_by(id_community='c1').one()
ir2 = InclusionRequest.query.filter_by(id_community='c2').one()
assert ir1.id_record == record.id
assert ir2.id_record == record.id
assert deposit['communities'] == ['c1', 'c2']
assert not record.get('communities', [])
def test_fixed_communities(app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Test automatic adding and requesting to fixed communities."""
deposit['grants'] = [{'title': 'SomeGrant'}, ]
# 'c3' is owned by one of the deposit owner
assert Community.get('c3').id_user in deposit['_deposit']['owners']
deposit['communities'] = ['c3', ]
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert record['communities'] == ['c3', 'grants_comm']
assert deposit['communities'] == ['c3', 'ecfunded', 'grants_comm',
'zenodo']
InclusionRequest.query.count() == 2
ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one()
assert ir1.id_record == record.id
ir2 = InclusionRequest.query.filter_by(id_community='ecfunded').one()
assert ir2.id_record == record.id
def test_basic_community_workflow(app, db, communities, deposit, deposit_file):
"""Test simple (without concurrent events) deposit publishing workflow."""
deposit = publish_and_expunge(db, deposit)
assert InclusionRequest.query.count() == 0
pid, record = deposit.fetch_published()
assert not record.get('communities', [])
# Open record for edit, request a community and publish
deposit = deposit.edit()
deposit['communities'] = ['c1', ]
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
# Should contain just an InclusionRequest
assert not record.get('communities', [])
assert not record['_oai'].get('sets', [])
assert InclusionRequest.query.count() == 1
ir = InclusionRequest.query.one()
assert ir.id_community == 'c1'
assert ir.id_record == record.id
# Accept a record to the community 'c1'
c1 = Community.get('c1')
c1.accept_record(record)
record.commit()
db.session.commit()
assert InclusionRequest.query.count() == 0
assert record['communities'] == ['c1', ]
assert record['_oai']['sets'] == ['user-c1', ]
# Open for edit and request another community
deposit = deposit.edit()
assert deposit['communities'] == ['c1', ]
deposit['communities'] = ['c1', 'c2', ] # New request for community 'c2'
deposit = publish_and_expunge(db, deposit)
deposit['communities'] = ['c1', 'c2', ]
pid, record = deposit.fetch_published()
assert record['communities'] == ['c1', ]
assert record['_oai']['sets'] == ['user-c1', ]
assert InclusionRequest.query.count() == 1
ir = InclusionRequest.query.one()
assert ir.id_community == 'c2'
assert ir.id_record == record.id
# Reject the request for community 'c2'
c2 = Community.get('c2')
c2.reject_record(record)
db.session.commit()
deposit = deposit.edit()
# The deposit should not contain obsolete inclusion requests
assert deposit['communities'] == ['c1', ]
assert InclusionRequest.query.count() == 0
pid, record = deposit.fetch_published()
assert record['communities'] == ['c1', ]
assert record['_oai']['sets'] == ['user-c1', ]
# Request for removal from a previously accepted community 'c1'
deposit['communities'] = []
deposit = publish_and_expunge(db, deposit)
pid, record = deposit.fetch_published()
assert not deposit.get('communities', [])
assert not record.get('communities', [])
assert not record['_oai'].get('sets', [])
assert InclusionRequest.query.count() == 0
def test_autoadd(app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Test basic workflow using Deposit and Communities API."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c3' (owned by user) through deposit v2
deposit_v2['communities'] = ['c1', 'c2', 'c3', ]
deposit_v2['grants'] = [{'title': 'SomeGrant'}, ]
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
assert record_v2['grants'] == [{'title': 'SomeGrant'}, ]
recid_v2_value = recid_v2.pid_value
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == ['c3', 'grants_comm']
assert record_v2.get('communities', []) == ['c3', 'grants_comm']
assert deposit_v1.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded',
'grants_comm', 'zenodo']
assert deposit_v2.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded',
'grants_comm', 'zenodo']
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
c3_api = ZenodoCommunity('c3')
grants_comm_api = ZenodoCommunity('grants_comm')
ecfunded_api = ZenodoCommunity('ecfunded')
zenodo_api = ZenodoCommunity('zenodo')
# Inclusion requests should be visible for both records
assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c3_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0
assert c3_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0
assert grants_comm_api.get_comm_irs(
record_v1, pid=recid_v1).count() == 0
assert grants_comm_api.get_comm_irs(
record_v2, pid=recid_v2).count() == 0
assert ecfunded_api.get_comm_irs(
record_v1, pid=recid_v1).count() == 1
assert ecfunded_api.get_comm_irs(
record_v2, pid=recid_v2).count() == 1
assert zenodo_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert zenodo_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
# Accept to 'c1' through record_v2 (as originally requested),
# and 'c2' through record_v1 (resolved through version)
c1_api.accept_record(record_v2, pid=recid_v2)
c2_api.accept_record(record_v1, pid=recid_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
# Accepting individual record to a community should propagate the changes
# to all versions
assert record_v1.get('communities', []) == ['c1', 'c2', 'c3',
'grants_comm']
assert record_v2.get('communities', []) == ['c1', 'c2', 'c3',
'grants_comm']
assert deposit_v1.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded',
'grants_comm', 'zenodo']
assert deposit_v2.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded',
'grants_comm', 'zenodo']
# Removing 'c1'-'c3' from deposit_v1 should remove it from two published
# records and other deposits as well
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1 = deposit_v1.edit()
deposit_v1['communities'] = []
deposit_v1 = publish_and_expunge(db, deposit_v1)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
assert record_v1.get('communities', []) == ['grants_comm', ]
assert record_v2.get('communities', []) == ['grants_comm', ]
assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
def test_relations_serialization(app, db, deposit, deposit_file):
"""Serialize PID relations."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
expected = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 1
}]
}
assert serialize_relations(recid_v1) == expected
deposit_v1.newversion()
# Should contain "draft_child_deposit" information
expected = {
"version": [{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 1,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert serialize_relations(recid_v1) == expected
# Publish the new version
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
# Should no longer contain "draft_child_deposit" info after publishing
# and no longer be the last child
expected = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}]
}
assert serialize_relations(recid_v1) == expected
# New version should be the last child now
expected = {
"version": [{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert serialize_relations(recid_v2) == expected
def test_archiving(app, db, deposit, deposit_file, locations, archive_fs):
"""Test ZenodoSIP archiving."""
# Stash the configuration and enable writing
orig = app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED']
app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] = True
deposit.files['test2.txt'] = BytesIO(b'test-two')
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_id = recid_v1.id
# Record files after publishing: 'test.txt', 'test2.txt'
sip1 = SIP(SIPModel.query.one())
sip1_id = sip1.id
# Edit the metadata
deposit_v1 = deposit_v1.edit()
deposit_v1['title'] = "New title"
deposit_v1 = publish_and_expunge(db, deposit_v1)
# Record files after publishing: 'test.txt', 'test2.txt'
sip2_id = SIPModel.query.order_by(SIPModel.created.desc()).first().id
# Create a new version
deposit_v1.newversion()
recid_v1 = PersistentIdentifier.query.get(recid_v1_id)
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid)
del deposit_v2.files['test.txt']
deposit_v2.files['test3.txt'] = BytesIO(b('test-three'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
# Record files after publishing: 'test2.txt', 'test3.txt'
sip1 = SIP(SIPModel.query.get(sip1_id))
sip2 = SIP(SIPModel.query.get(sip2_id))
sip3 = SIP(SIPModel.query.order_by(SIPModel.created.desc()).first())
# Becase we are using secure_filename when writing SIPFiles we need to
# genenarate the correct names: <SIPFile.id>-<secure_filename>
s1_file1_fn = '{0}-test.txt'.format(fetch_suff(sip1, 'test.txt').file_id)
s1_file1_fp = 'data/files/{0}'.format(s1_file1_fn)
s1_file2_fn = '{0}-test2.txt'.format(fetch_suff(sip1, 'test2.txt').file_id)
s1_file2_fp = 'data/files/{0}'.format(s1_file2_fn)
s3_file2_fn = '{0}-test2.txt'.format(fetch_suff(sip3, 'test2.txt').file_id)
s3_file2_fp = 'data/files/{0}'.format(s3_file2_fn)
s3_file3_fn = '{0}-test3.txt'.format(fetch_suff(sip3, 'test3.txt').file_id)
s3_file3_fp = 'data/files/{0}'.format(s3_file3_fn)
sip1_bagmeta = json.loads(next(
m.content for m in sip1.metadata if m.type.name == 'bagit'))['files']
sip2_bagmeta = json.loads(next(
m.content for m in sip2.metadata if m.type.name == 'bagit'))['files']
sip3_bagmeta = json.loads(next(
m.content for m in sip3.metadata if m.type.name == 'bagit'))['files']
# Check if Bagit metadata contains the correct file-fetching information
assert set([f['filepath'] for f in sip1_bagmeta]) == \
set([s1_file1_fp,
s1_file2_fp,
'data/filenames.txt',
'data/metadata/record-json.json', 'bag-info.txt',
'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt'])
assert not BagItArchiver._is_fetched(
get_m_item(sip1_bagmeta, s1_file1_fp))
assert not BagItArchiver._is_fetched(
get_m_item(sip1_bagmeta, s1_file2_fp))
assert set([f['filepath'] for f in sip2_bagmeta]) == \
set([s1_file1_fp,
s1_file2_fp,
'data/filenames.txt',
'data/metadata/record-json.json', 'bag-info.txt',
'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt',
'fetch.txt'])
# Both files should be fetched since it's only metadata-edit submission
assert BagItArchiver._is_fetched(
get_m_item(sip2_bagmeta, s1_file1_fp))
assert BagItArchiver._is_fetched(
get_m_item(sip2_bagmeta, s1_file2_fp))
assert set([f['filepath'] for f in sip3_bagmeta]) == \
set([s3_file2_fp,
s3_file3_fp,
'data/filenames.txt',
'data/metadata/record-json.json', 'bag-info.txt',
'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt',
'fetch.txt'])
# First file should be fetched from previous version and new file should
# be archived in this bag.
assert BagItArchiver._is_fetched(
get_m_item(sip3_bagmeta, s3_file2_fp))
assert not BagItArchiver._is_fetched(
get_m_item(sip3_bagmeta, s3_file3_fp))
archiver1 = BagItArchiver(sip1)
archiver2 = BagItArchiver(sip2)
archiver3 = BagItArchiver(sip3)
# Each archiver subpath follows: '<recid>/r/<ISO-8601-SIP-timestamp>'
sip1_ts = arrow.get(sip1.model.created).isoformat()
sip2_ts = arrow.get(sip2.model.created).isoformat()
sip3_ts = arrow.get(sip3.model.created).isoformat()
assert archiver1.get_archive_subpath() == '2/r/{0}'.format(sip1_ts)
assert archiver2.get_archive_subpath() == '2/r/{0}'.format(sip2_ts)
assert archiver3.get_archive_subpath() == '3/r/{0}'.format(sip3_ts)
# As a test, write the SIPs in reverse chronological order
assert not sip1.archived
assert not sip2.archived
assert not sip3.archived
archive_sip.delay(sip3.id)
archive_sip.delay(sip2.id)
archive_sip.delay(sip1.id)
assert sip1.archived
assert sip2.archived
assert sip3.archived
fs1 = archive_fs.opendir(archiver1.get_archive_subpath())
assert set(fs1.listdir()) == set(['tagmanifest-md5.txt', 'bagit.txt',
'manifest-md5.txt', 'bag-info.txt',
'data'])
assert set(fs1.listdir('data')) == set(['metadata', 'files',
'filenames.txt'])
assert fs1.listdir('data/metadata') == ['record-json.json', ]
assert set(fs1.listdir('data/files')) == set([s1_file1_fn, s1_file2_fn])
fs2 = archive_fs.opendir(archiver2.get_archive_subpath())
assert set(fs2.listdir()) == set(['tagmanifest-md5.txt', 'bagit.txt',
'manifest-md5.txt', 'bag-info.txt',
'data', 'fetch.txt'])
# Second SIP has written only the metadata,
# because of that There should be no 'files/', but 'filesnames.txt' should
# still be there becasue of the fetch.txt
assert set(fs2.listdir('data')) == set(['metadata', 'filenames.txt'])
assert fs2.listdir('data/metadata') == ['record-json.json', ]
with fs2.open('fetch.txt') as fp:
cnt = fp.read().splitlines()
# Fetched files should correctly fetch the files from the first archive
base_uri = archiver1.get_archive_base_uri()
assert set(cnt) == set([
'{base}/2/r/{s1ts}/{fn} 4 {fn}'.format(fn=s1_file1_fp, base=base_uri,
s1ts=sip1_ts),
'{base}/2/r/{s1ts}/{fn} 8 {fn}'.format(fn=s1_file2_fp, base=base_uri,
s1ts=sip1_ts),
])
fs3 = archive_fs.opendir(archiver3.get_archive_subpath())
assert set(fs3.listdir()) == set(['tagmanifest-md5.txt', 'bagit.txt',
'manifest-md5.txt', 'bag-info.txt',
'data', 'fetch.txt'])
# Third SIP should write only the extra 'test3.txt' file
assert set(fs3.listdir('data')) == set(['metadata', 'files',
'filenames.txt'])
assert fs3.listdir('data/metadata') == ['record-json.json', ]
assert fs3.listdir('data/files') == [s3_file3_fn, ]
with fs3.open('fetch.txt') as fp:
cnt = fp.read().splitlines()
# Since 'file.txt' was removed in third SIP, we should only fetch the
# 'test2.txt', also from the first archive, since that's where this
# file resides physically.
base_uri = archiver1.get_archive_base_uri()
assert set(cnt) == set([
'{base}/2/r/{s1ts}/{fn} 8 {fn}'.format(fn=s3_file2_fp, base=base_uri,
s1ts=sip1_ts),
])
app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] = orig
def test_related_identifiers_serialization(app, db, deposit, deposit_file):
"""Serialize PID Relations to related identifiers."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c2' through deposit v2
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
rids = serialize_related_identifiers(recid_v1)
expected_v1 = [
{
'scheme': 'doi',
'identifier': '10.5072/zenodo.1',
'relation': 'isPartOf'
}
# TODO: serialization of new version realtions is disabled
# {
# 'scheme': 'doi',
# 'identifier': '10.5072/zenodo.3',
# 'relation': 'isPreviousVersionOf'
# }
]
assert rids == expected_v1
rids = serialize_related_identifiers(recid_v2)
expected_v2 = [
{
'scheme': 'doi',
'identifier': '10.5072/zenodo.1',
'relation': 'isPartOf'
}
# TODO: serialization of new version realtions is disabled
# {
# 'scheme': 'doi',
# 'identifier': '10.5072/zenodo.2',
# 'relation': 'isNewVersionOf'
# }
]
assert rids == expected_v2
parent_pid = PersistentIdentifier.get('recid', '1')
rids = serialize_related_identifiers(parent_pid)
expected_parent = [
{
'relation': 'hasPart',
'scheme': 'doi',
'identifier': '10.5072/zenodo.2'
},
{
'relation': 'hasPart',
'scheme': 'doi',
'identifier': '10.5072/zenodo.3'
}
]
assert rids == expected_parent
def test_versioning_indexing(db, es, deposit, deposit_file):
"""Test the indexing of 'version' relations."""
deposit_index_name = 'deposits-records-record-v1.0.0'
records_index_name = 'records-record-v1.0.0'
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
RecordIndexer().index_by_id(str(record_v1.id))
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 1
assert len(s_rec) == 1
assert 'relations' in s_dep[0]['_source']
assert 'relations' in s_rec[0]['_source']
expected = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 1,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert s_dep[0]['_source']['relations'] == expected
assert s_rec[0]['_source']['relations'] == expected
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid)
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 2 # Two deposits should be indexed
assert len(s_rec) == 1 # One, since record does not exist yet
s_dep1 = current_search.client.get(index=deposit_index_name,
id=deposit_v1.id)
s_dep2 = current_search.client.get(index=deposit_index_name,
id=deposit_v2.id)
expected_d1 = {
"version": [{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2 # For deposit, draft children are also counted
}]
}
expected_d2 = {
"version": [{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 2, # For deposit, draft children are also counted
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert s_dep1['_source']['relations'] == expected_d1
assert s_dep2['_source']['relations'] == expected_d2
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
RecordIndexer().index_by_id(str(record_v2.id))
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 2
assert len(s_rec) == 2
s_dep1 = current_search.client.get(index=deposit_index_name,
id=deposit_v1.id)
s_dep2 = current_search.client.get(index=deposit_index_name,
id=deposit_v2.id)
s_rec1 = current_search.client.get(index=records_index_name,
id=record_v1.id)
s_rec2 = current_search.client.get(index=records_index_name,
id=record_v2.id)
expected_d1 = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}]
}
expected_d2 = {
"version": [{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert s_dep1['_source']['relations'] == expected_d1
assert s_dep2['_source']['relations'] == expected_d2
expected_r1 = {
"version": [{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}]
}
expected_r2 = {
"version": [{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}]
}
assert s_rec1['_source']['relations'] == expected_r1
assert s_rec2['_source']['relations'] == expected_r2
def test_archiving(app, db, deposit, deposit_file, locations, archive_fs):
"""Test ZenodoSIP archiving."""
# Stash the configuration and enable writing
orig = app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED']
app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] = True
deposit.files['test2.txt'] = BytesIO(b'test-two')
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_id = recid_v1.id
# Record files after publishing: 'test.txt', 'test2.txt'
sip1 = SIP(SIPModel.query.one())
sip1_id = sip1.id
# Edit the metadata
deposit_v1 = deposit_v1.edit()
deposit_v1['title'] = "New title"
deposit_v1 = publish_and_expunge(db, deposit_v1)
# Record files after publishing: 'test.txt', 'test2.txt'
sip2_id = SIPModel.query.order_by(SIPModel.created.desc()).first().id
# Create a new version
deposit_v1.newversion()
recid_v1 = PersistentIdentifier.query.get(recid_v1_id)
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid)
del deposit_v2.files['test.txt']
deposit_v2.files['test3.txt'] = BytesIO(b('test-three'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
# Record files after publishing: 'test2.txt', 'test3.txt'
sip1 = SIP(SIPModel.query.get(sip1_id))
sip2 = SIP(SIPModel.query.get(sip2_id))
sip3 = SIP(SIPModel.query.order_by(SIPModel.created.desc()).first())
# Becase we are using secure_filename when writing SIPFiles we need to
# genenarate the correct names: <SIPFile.id>-<secure_filename>
s1_file1_fn = '{0}-test.txt'.format(fetch_suff(sip1, 'test.txt').file_id)
s1_file1_fp = 'data/files/{0}'.format(s1_file1_fn)
s1_file2_fn = '{0}-test2.txt'.format(fetch_suff(sip1, 'test2.txt').file_id)
s1_file2_fp = 'data/files/{0}'.format(s1_file2_fn)
s3_file2_fn = '{0}-test2.txt'.format(fetch_suff(sip3, 'test2.txt').file_id)
s3_file2_fp = 'data/files/{0}'.format(s3_file2_fn)
s3_file3_fn = '{0}-test3.txt'.format(fetch_suff(sip3, 'test3.txt').file_id)
s3_file3_fp = 'data/files/{0}'.format(s3_file3_fn)
sip1_bagmeta = json.loads(
next(m.content for m in sip1.metadata
if m.type.name == 'bagit'))['files']
sip2_bagmeta = json.loads(
next(m.content for m in sip2.metadata
if m.type.name == 'bagit'))['files']
sip3_bagmeta = json.loads(
next(m.content for m in sip3.metadata
if m.type.name == 'bagit'))['files']
# Check if Bagit metadata contains the correct file-fetching information
assert set([f['filepath'] for f in sip1_bagmeta]) == \
set([s1_file1_fp,
s1_file2_fp,
'data/filenames.txt',
'data/metadata/record-json.json', 'bag-info.txt',
'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt'])
assert not BagItArchiver._is_fetched(get_m_item(sip1_bagmeta, s1_file1_fp))
assert not BagItArchiver._is_fetched(get_m_item(sip1_bagmeta, s1_file2_fp))
assert set([f['filepath'] for f in sip2_bagmeta]) == \
set([s1_file1_fp,
s1_file2_fp,
'data/filenames.txt',
'data/metadata/record-json.json', 'bag-info.txt',
'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt',
'fetch.txt'])
# Both files should be fetched since it's only metadata-edit submission
assert BagItArchiver._is_fetched(get_m_item(sip2_bagmeta, s1_file1_fp))
assert BagItArchiver._is_fetched(get_m_item(sip2_bagmeta, s1_file2_fp))
assert set([f['filepath'] for f in sip3_bagmeta]) == \
set([s3_file2_fp,
s3_file3_fp,
'data/filenames.txt',
'data/metadata/record-json.json', 'bag-info.txt',
'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt',
'fetch.txt'])
# First file should be fetched from previous version and new file should
# be archived in this bag.
assert BagItArchiver._is_fetched(get_m_item(sip3_bagmeta, s3_file2_fp))
assert not BagItArchiver._is_fetched(get_m_item(sip3_bagmeta, s3_file3_fp))
archiver1 = BagItArchiver(sip1)
archiver2 = BagItArchiver(sip2)
archiver3 = BagItArchiver(sip3)
# Each archiver subpath follows: '<recid>/r/<ISO-8601-SIP-timestamp>'
sip1_ts = arrow.get(sip1.model.created).isoformat()
sip2_ts = arrow.get(sip2.model.created).isoformat()
sip3_ts = arrow.get(sip3.model.created).isoformat()
assert archiver1.get_archive_subpath() == '2/r/{0}'.format(sip1_ts)
assert archiver2.get_archive_subpath() == '2/r/{0}'.format(sip2_ts)
assert archiver3.get_archive_subpath() == '3/r/{0}'.format(sip3_ts)
# As a test, write the SIPs in reverse chronological order
assert not sip1.archived
assert not sip2.archived
assert not sip3.archived
archive_sip.delay(sip3.id)
archive_sip.delay(sip2.id)
archive_sip.delay(sip1.id)
assert sip1.archived
assert sip2.archived
assert sip3.archived
fs1 = archive_fs.opendir(archiver1.get_archive_subpath())
assert set(fs1.listdir()) == set([
'tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt', 'bag-info.txt',
'data'
])
assert set(fs1.listdir('data')) == set(
['metadata', 'files', 'filenames.txt'])
assert fs1.listdir('data/metadata') == [
'record-json.json',
]
assert set(fs1.listdir('data/files')) == set([s1_file1_fn, s1_file2_fn])
fs2 = archive_fs.opendir(archiver2.get_archive_subpath())
assert set(fs2.listdir()) == set([
'tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt', 'bag-info.txt',
'data', 'fetch.txt'
])
# Second SIP has written only the metadata,
# because of that There should be no 'files/', but 'filesnames.txt' should
# still be there becasue of the fetch.txt
assert set(fs2.listdir('data')) == set(['metadata', 'filenames.txt'])
assert fs2.listdir('data/metadata') == [
'record-json.json',
]
with fs2.open('fetch.txt') as fp:
cnt = fp.read().splitlines()
# Fetched files should correctly fetch the files from the first archive
base_uri = archiver1.get_archive_base_uri()
assert set(cnt) == set([
'{base}/2/r/{s1ts}/{fn} 4 {fn}'.format(fn=s1_file1_fp,
base=base_uri,
s1ts=sip1_ts),
'{base}/2/r/{s1ts}/{fn} 8 {fn}'.format(fn=s1_file2_fp,
base=base_uri,
s1ts=sip1_ts),
])
fs3 = archive_fs.opendir(archiver3.get_archive_subpath())
assert set(fs3.listdir()) == set([
'tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt', 'bag-info.txt',
'data', 'fetch.txt'
])
# Third SIP should write only the extra 'test3.txt' file
assert set(fs3.listdir('data')) == set(
['metadata', 'files', 'filenames.txt'])
assert fs3.listdir('data/metadata') == [
'record-json.json',
]
assert fs3.listdir('data/files') == [
s3_file3_fn,
]
with fs3.open('fetch.txt') as fp:
cnt = fp.read().splitlines()
# Since 'file.txt' was removed in third SIP, we should only fetch the
# 'test2.txt', also from the first archive, since that's where this
# file resides physically.
base_uri = archiver1.get_archive_base_uri()
assert set(cnt) == set([
'{base}/2/r/{s1ts}/{fn} 8 {fn}'.format(fn=s3_file2_fp,
base=base_uri,
s1ts=sip1_ts),
])
app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] = orig
def test_relations_serialization(app, db, deposit, deposit_file):
"""Serialize PID relations."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
expected = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 1
}
]
}
assert serialize_relations(recid_v1) == expected
deposit_v1.newversion()
# Should contain "draft_child_deposit" information
expected = {
"version": [
{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 1,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert serialize_relations(recid_v1) == expected
# Publish the new version
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
# Should no longer contain "draft_child_deposit" info after publishing
# and no longer be the last child
expected = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}
]
}
assert serialize_relations(recid_v1) == expected
# New version should be the last child now
expected = {
"version": [
{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert serialize_relations(recid_v2) == expected
def test_autoadd(app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Test basic workflow using Deposit and Communities API."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c3' (owned by user) through deposit v2
deposit_v2['communities'] = [
'c1',
'c2',
'c3',
]
deposit_v2['grants'] = [
{
'title': 'SomeGrant'
},
]
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
assert record_v2['grants'] == [
{
'title': 'SomeGrant'
},
]
recid_v2_value = recid_v2.pid_value
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == ['c3', 'grants_comm']
assert record_v2.get('communities', []) == ['c3', 'grants_comm']
assert deposit_v1.get('communities', []) == [
'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'
]
assert deposit_v2.get('communities', []) == [
'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'
]
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
c3_api = ZenodoCommunity('c3')
grants_comm_api = ZenodoCommunity('grants_comm')
ecfunded_api = ZenodoCommunity('ecfunded')
zenodo_api = ZenodoCommunity('zenodo')
# Inclusion requests should be visible for both records
assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c3_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0
assert c3_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0
assert grants_comm_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0
assert grants_comm_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0
assert ecfunded_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert ecfunded_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert zenodo_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert zenodo_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
# Accept to 'c1' through record_v2 (as originally requested),
# and 'c2' through record_v1 (resolved through version)
c1_api.accept_record(record_v2, pid=recid_v2)
c2_api.accept_record(record_v1, pid=recid_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
# Accepting individual record to a community should propagate the changes
# to all versions
assert record_v1.get('communities',
[]) == ['c1', 'c2', 'c3', 'grants_comm']
assert record_v2.get('communities',
[]) == ['c1', 'c2', 'c3', 'grants_comm']
assert deposit_v1.get('communities', []) == [
'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'
]
assert deposit_v2.get('communities', []) == [
'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'
]
# Removing 'c1'-'c3' from deposit_v1 should remove it from two published
# records and other deposits as well
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1 = deposit_v1.edit()
deposit_v1['communities'] = []
deposit_v1 = publish_and_expunge(db, deposit_v1)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
assert record_v1.get('communities', []) == [
'grants_comm',
]
assert record_v2.get('communities', []) == [
'grants_comm',
]
assert deposit_v1.get('communities',
[]) == ['ecfunded', 'grants_comm', 'zenodo']
assert deposit_v2.get('communities',
[]) == ['ecfunded', 'grants_comm', 'zenodo']
def test_versioning_indexing(db, es, deposit, deposit_file):
"""Test the indexing of 'version' relations."""
deposit_index_name = 'deposits-records-record-v1.0.0'
records_index_name = 'records-record-v1.0.0'
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
RecordIndexer().index_by_id(str(record_v1.id))
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 1
assert len(s_rec) == 1
assert 'relations' in s_dep[0]['_source']
assert 'relations' in s_rec[0]['_source']
expected = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 1,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert s_dep[0]['_source']['relations'] == expected
assert s_rec[0]['_source']['relations'] == expected
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid)
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 2 # Two deposits should be indexed
assert len(s_rec) == 1 # One, since record does not exist yet
s_dep1 = current_search.client.get(
index=deposit_index_name, id=deposit_v1.id)
s_dep2 = current_search.client.get(
index=deposit_index_name, id=deposit_v2.id)
expected_d1 = {
"version": [
{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2 # For deposit, draft children are also counted
}
]
}
expected_d2 = {
"version": [
{
"draft_child_deposit": {
"pid_type": "depid",
"pid_value": "3"
},
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "2"
},
"count": 2, # For deposit, draft children are also counted
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert s_dep1['_source']['relations'] == expected_d1
assert s_dep2['_source']['relations'] == expected_d2
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
RecordIndexer().index_by_id(str(record_v2.id))
RecordIndexer().process_bulk_queue()
current_search.flush_and_refresh(index=deposit_index_name)
current_search.flush_and_refresh(index=records_index_name)
s_dep = current_search.client.search(
index=deposit_index_name)['hits']['hits']
s_rec = current_search.client.search(
index=records_index_name)['hits']['hits']
assert len(s_dep) == 2
assert len(s_rec) == 2
s_dep1 = current_search.client.get(
index=deposit_index_name, id=deposit_v1.id)
s_dep2 = current_search.client.get(
index=deposit_index_name, id=deposit_v2.id)
s_rec1 = current_search.client.get(
index=records_index_name, id=record_v1.id)
s_rec2 = current_search.client.get(
index=records_index_name, id=record_v2.id)
expected_d1 = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}
]
}
expected_d2 = {
"version": [
{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert s_dep1['_source']['relations'] == expected_d1
assert s_dep2['_source']['relations'] == expected_d2
expected_r1 = {
"version": [
{
"draft_child_deposit": None,
"index": 0,
"is_last": False,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
"count": 2
}
]
}
expected_r2 = {
"version": [
{
"draft_child_deposit": None,
"index": 1,
"is_last": True,
"last_child": {
"pid_type": "recid",
"pid_value": "3"
},
"count": 2,
"parent": {
"pid_type": "recid",
"pid_value": "1"
},
}
]
}
assert s_rec1['_source']['relations'] == expected_r1
assert s_rec2['_source']['relations'] == expected_r2
def test_related_identifiers_serialization(app, db, deposit, deposit_file):
"""Serialize PID Relations to related identifiers."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c2' through deposit v2
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
rids = serialize_related_identifiers(recid_v1)
expected_v1 = [
{
'scheme': 'doi',
'identifier': '10.5072/zenodo.1',
'relation': 'isPartOf'
}
# TODO: serialization of new version realtions is disabled
# {
# 'scheme': 'doi',
# 'identifier': '10.5072/zenodo.3',
# 'relation': 'isPreviousVersionOf'
# }
]
assert rids == expected_v1
rids = serialize_related_identifiers(recid_v2)
expected_v2 = [
{
'scheme': 'doi',
'identifier': '10.5072/zenodo.1',
'relation': 'isPartOf'
}
# TODO: serialization of new version realtions is disabled
# {
# 'scheme': 'doi',
# 'identifier': '10.5072/zenodo.2',
# 'relation': 'isNewVersionOf'
# }
]
assert rids == expected_v2
parent_pid = PersistentIdentifier.get('recid', '1')
rids = serialize_related_identifiers(parent_pid)
expected_parent = [{
'relation': 'hasPart',
'scheme': 'doi',
'identifier': '10.5072/zenodo.2'
}, {
'relation': 'hasPart',
'scheme': 'doi',
'identifier': '10.5072/zenodo.3'
}]
assert rids == expected_parent
