def test_autoadd_explicit_newversion( app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Explicitly the autoadded communities in a new version.""" deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2['communities'] = ['ecfunded', 'grants_comm', 'zenodo'] deposit_v2['grants'] = [{'title': 'SomeGrant'}, ] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) assert record_v1.get('communities', []) == ['grants_comm', ] assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo'] assert record_v2.get('communities', []) == ['grants_comm', ] assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo']
def test_fixed_autoadd_edit(app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Test automatic adding and requesting to fixed communities. Add to grants_comm also after later addition of grant information. """ deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['zenodo', ] assert 'communities' not in record ir = InclusionRequest.query.one() assert ir.id_community == 'zenodo' assert ir.id_record == record.id deposit = deposit.edit() deposit['grants'] = [{'title': 'SomeGrant'}, ] # Requesting for 'grants_comm' and 'ecfunded' manually even though it will # be added due to specifying 'grants' shouldn't cause problems deposit['communities'] = ['ecfunded', 'grants_comm', 'zenodo'] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() InclusionRequest.query.count() == 2 ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one() assert ir1.id_record == record.id ir2 = InclusionRequest.query.filter_by(id_community='ecfunded').one() assert ir2.id_record == record.id assert deposit['communities'] == ['ecfunded', 'grants_comm', 'zenodo', ] assert record['communities'] == ['grants_comm', ]
def test_autoaccept_owned_communities(app, db, users, communities, deposit, deposit_file): """Automatically accept records requested by community owners.""" # 'c3' is owned by the user, but not 'c1' deposit['communities'] = ['c1', 'c3', ] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', 'c3', ] assert record['communities'] == ['c3', ] assert record['_oai']['sets'] == ['user-c3'] assert InclusionRequest.query.count() == 1 ir = InclusionRequest.query.one() assert ir.id_community == 'c1' assert ir.id_record == record.id # Edit the deposit, and add more communities # 'c4' should be added automatically, but not 'c2' deposit = deposit.edit() deposit['communities'] = ['c1', 'c2', 'c3', 'c4', ] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', 'c2', 'c3', 'c4', ] assert record['communities'] == ['c3', 'c4', ] assert set(record['_oai']['sets']) == set(['user-c3', 'user-c4']) assert InclusionRequest.query.count() == 2 ir1 = InclusionRequest.query.filter_by(id_community='c1').one() ir2 = InclusionRequest.query.filter_by(id_community='c2').one() assert ir1.id_record == record.id assert ir2.id_record == record.id
def test_deposit_versioning_draft_child_unlinking_bug( app, db, communities, deposit, deposit_file): """ Bug with draft_child_deposit unlinking. Bug where a draft_child_deposit was unlinked from a new version draft, when another version of a record was edited and published. """ deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit.fetch_published() recid_v1_value = recid_v1.pid_value # Initiate a new version draft deposit_v1.newversion() recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) pv = PIDVersioning(child=recid_v1) assert pv.draft_child_deposit assert pv.draft_child deposit_v1.edit() deposit_v1 = deposit_v1.edit() deposit_v1 = publish_and_expunge(db, deposit_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) pv = PIDVersioning(child=recid_v1) # Make sure the draft child deposit was not unliked due to publishing of # the edited draft assert pv.draft_child_deposit assert pv.draft_child
def test_deposit_versioning_draft_child_unlinking_bug(app, db, communities, deposit, deposit_file): """ Bug with draft_child_deposit unlinking. Bug where a draft_child_deposit was unlinked from a new version draft, when another version of a record was edited and published. """ deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit.fetch_published() recid_v1_value = recid_v1.pid_value # Initiate a new version draft deposit_v1.newversion() recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) pv = PIDVersioning(child=recid_v1) assert pv.draft_child_deposit assert pv.draft_child deposit_v1.edit() deposit_v1 = deposit_v1.edit() deposit_v1 = publish_and_expunge(db, deposit_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) pv = PIDVersioning(child=recid_v1) # Make sure the draft child deposit was not unliked due to publishing of # the edited draft assert pv.draft_child_deposit assert pv.draft_child
def test_propagation_with_newversion_open( app, db, users, communities, deposit, deposit_file): """Adding old versions to a community should propagate to all drafts.""" # deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) deposit_v1 = deposit_v1.edit() recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value # New version in 'deposit_v2' has not been published yet deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) # depid_v1_value = deposit_v1['_deposit']['id'] # depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1['communities'] = ['c1', 'c2', ] deposit_v1 = publish_and_expunge(db, deposit_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) c1_api = ZenodoCommunity('c1') c1_api.accept_record(record_v1, pid=recid_v1) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) assert deposit_v2['communities'] == ['c1', 'c2'] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() assert record_v2['communities'] == ['c1', ]
def test_record_modified_while_edit(app, db, communities, deposit, deposit_file): """Test deposit publishing with concurrent events. Modify a record, while deposit in open edit and then published. """ deposit['communities'] = ['c1', ] deposit = publish_and_expunge(db, deposit) assert InclusionRequest.query.count() == 1 pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', ] assert not record.get('communities', []) # Open for edit deposit = deposit.edit() pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', ] assert not record.get('communities', []) assert InclusionRequest.query.count() == 1 # Meanwhile, a record is modified record['title'] = 'Other title' record.commit() db.session.commit() # Publish and make sure nothing is missing deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', ] assert not record.get('communities', []) assert InclusionRequest.query.count() == 1 ir = InclusionRequest.query.one() assert ir.id_community == 'c1' assert ir.id_record == record.id
def test_fixed_communities_edit(app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Test automatic adding and requesting to fixed communities.""" deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['zenodo', ] assert 'communities' not in record ir = InclusionRequest.query.one() assert ir.id_community == 'zenodo' assert ir.id_record == record.id # Open for edit deposit = deposit.edit() # Make sure 'zenodo' community is requested pid, record = deposit.fetch_published() assert deposit['communities'] == ['zenodo', ] assert not record.get('communities', []) assert InclusionRequest.query.count() == 1 comm = Community.get('zenodo') comm.accept_record(record) record.commit() db.session.commit() # Publish and make sure nothing is missing deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['zenodo'] assert record['communities'] == ['zenodo', ] assert record['_oai']['sets'] == ['user-zenodo', ] assert InclusionRequest.query.count() == 0
def test_basic_api(app, db, communities, deposit, deposit_file): """Test basic workflow using Deposit and Communities API.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c2' through deposit v2 deposit_v2['communities'] = [ 'c1', 'c2', ] deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() recid_v2_value = recid_v2.pid_value depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() assert record_v1.get('communities', []) == [] assert record_v2.get('communities', []) == [] c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') # Inclusion requests should be visible for both records assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 # Accept to 'c1' through record_v2 (as originally requested), # and 'c2' through record_v1 (version) c1_api.accept_record(record_v2, pid=recid_v2) c2_api.accept_record(record_v1, pid=recid_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) # Accepting individual record to a community should propagate the changes # to all versions assert record_v1['communities'] == record_v2['communities'] == \ ['c1', 'c2', ] # Removing 'c1' from deposit_v1 should remove it from two published records depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1 = deposit_v1.edit() deposit_v1['communities'] = [] deposit_v1 = publish_and_expunge(db, deposit_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) assert record_v1.get('communities', []) == [] assert record_v2.get('communities', []) == []
def test_record_delete_v2(mocker, app, db, users, deposit, deposit_file): """Delete a record (only last version) with multiple versions.""" dc_mock = mocker.patch( 'invenio_pidstore.providers.datacite.DataCiteMDSClient') deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1.newversion() recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) # Stash a copy of v1 for later rec1 = deepcopy(record_v1) pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() # Stash a copy of v2 for later rec2 = deepcopy(record_v2) rec2_id = str(record_v2.id) assert dc_mock().metadata_delete.call_count == 0 # Remove the first version delete_record(rec2_id, 'spam', users[0]['id']) # Make sure all PIDs are deleted assert PID.get('doi', rec2['doi']).status == PIDStatus.DELETED assert PID.get('recid', rec2['recid']).status == PIDStatus.DELETED assert PID.get('depid', rec2['_deposit']['id']).status == PIDStatus.DELETED # Concept DOI should be left registered assert PID.get('doi', rec2['conceptdoi']).status == PIDStatus.REGISTERED # Make sure conceptrecid is redirecting to v1 crecid = PID.get('recid', rec2['conceptrecid']) assert crecid.status == PIDStatus.REDIRECTED assert crecid.get_redirect() == PID.get('recid', rec1['recid']) # Make sure the v1 PIDs are kept intact assert PID.get('oai', rec1['_oai']['id']).status == PIDStatus.REGISTERED assert PID.get('doi', rec1['doi']).status == PIDStatus.REGISTERED assert PID.get('recid', rec1['recid']).status == PIDStatus.REGISTERED assert PID.get('depid', rec1['_deposit']['id']).status == \ PIDStatus.REGISTERED # Only the v1 DOI should be deleted assert dc_mock().doi_post.call_count == 2 assert dc_mock().doi_post.has_any_call('10.5072/zenodo.2') assert dc_mock().doi_post.has_any_call('10.5072/zenodo.1') assert dc_mock().metadata_delete.call_count == 1 dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.3') record = Record.get_record(rec2_id) assert record['removed_by'] == users[0]['id'] assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def test_nonexisting_communities(app, db, users, communities, deposit, deposit_file): """Test adding nonexisting community.""" deposit['communities'] = ['nonexisting', ] with pytest.raises(MissingCommunityError) as exc_info: publish_and_expunge(db, deposit) assert exc_info.value.errors[0].res == { 'message': 'Provided community does not exist: nonexisting', 'field': 'metadata.communities', }
def test_basic_api(app, db, communities, deposit, deposit_file): """Test basic workflow using Deposit and Communities API.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c2' through deposit v2 deposit_v2['communities'] = ['c1', 'c2', ] deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() recid_v2_value = recid_v2.pid_value depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() assert record_v1.get('communities', []) == [] assert record_v2.get('communities', []) == [] c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') # Inclusion requests should be visible for both records assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 # Accept to 'c1' through record_v2 (as originally requested), # and 'c2' through record_v1 (version) c1_api.accept_record(record_v2, pid=recid_v2) c2_api.accept_record(record_v1, pid=recid_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) # Accepting individual record to a community should propagate the changes # to all versions assert record_v1['communities'] == record_v2['communities'] == \ ['c1', 'c2', ] # Removing 'c1' from deposit_v1 should remove it from two published records depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1 = deposit_v1.edit() deposit_v1['communities'] = [] deposit_v1 = publish_and_expunge(db, deposit_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) assert record_v1.get('communities', []) == [] assert record_v2.get('communities', []) == []
def test_remove_community_by_key_del(app, db, communities, deposit, deposit_file): """Test removal of communities by key deletion. Communities can be removed by not providing or deleting the communities from the key deposit. Moreover, the redundant 'empty' keys should not be automatically added to deposit nor record. """ # If 'communities' key was not in deposit metadata, # it shouldn't be automatically added assert 'communities' not in deposit deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert 'communities' not in deposit assert 'communities' not in record assert not record['_oai'].get('sets', []) # Request for 'c1' and 'c2' deposit = deposit.edit() deposit['communities'] = [ 'c1', 'c2', ] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() # No reason to have 'communities' in record since nothing was accepted assert 'communities' not in record assert not record['_oai'].get('sets', []) # Accept 'c1' c1 = Community.get('c1') c1.accept_record(record) record.commit() pid, record = deposit.fetch_published() assert deposit['communities'] == [ 'c1', 'c2', ] assert InclusionRequest.query.count() == 1 assert record['communities'] == [ 'c1', ] assert set(record['_oai']['sets']) == set(['user-c1']) # Remove the key from deposit and publish deposit = deposit.edit() del deposit['communities'] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert 'communities' not in deposit assert 'communities' not in record assert InclusionRequest.query.count() == 0 assert not record['_oai'].get('sets', [])
def test_fixed_communities_grants(app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Test automatic adding and requesting to fixed communities. Add to grants_comm also after later addition of grant information. """ deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['zenodo', ] assert 'communities' not in record ir = InclusionRequest.query.one() assert ir.id_community == 'zenodo' assert ir.id_record == record.id deposit = deposit.edit() deposit['grants'] = [{'title': 'SomeGrant'}, ] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['ecfunded', 'grants_comm', 'zenodo', ] assert record['communities'] == ['grants_comm', ] InclusionRequest.query.count() == 2 ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one() assert ir1.id_record == record.id ir2 = InclusionRequest.query.filter_by(id_community='ecfunded').one() assert ir2.id_record == record.id # Remove 'grants' without auto requested community being accepted. # We should not remove the inclusion request as we don't know if user # requested it manually or whether it was an automatic request deposit = deposit.edit() deposit['grants'] = [] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['ecfunded', 'grants_comm', 'zenodo', ] assert record['communities'] == ['grants_comm', ] InclusionRequest.query.count() == 2 ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one() assert ir1.id_record == record.id ir2 = InclusionRequest.query.filter_by(id_community='ecfunded').one() assert ir2.id_record == record.id # However, if user explicitly removed auto-requested community, and grants # have been removed too, the IR should be removed. deposit = deposit.edit() deposit['grants'] = [] # Removed 'ecfunded' and 'grants_comm' from deposit deposit['communities'] = ['zenodo', ] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['zenodo', ] assert 'communities' not in record InclusionRequest.query.count() == 1 ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one() assert ir1.id_record == record.id
def test_nonexisting_communities(app, db, users, communities, deposit, deposit_file): """Test adding nonexisting community.""" deposit['communities'] = [ 'nonexisting', ] with pytest.raises(MissingCommunityError) as exc_info: publish_and_expunge(db, deposit) assert exc_info.value.errors[0].res == { 'message': 'Provided community does not exist: nonexisting', 'field': 'metadata.communities', }
def test_basic_deposit_edit(app, db, communities, deposit, deposit_file): """Test simple deposit publishing.""" deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() initial_oai = deepcopy(record['_oai']) # Create some potential corruptions to protected fields deposit = deposit.edit() deposit['_files'][0]['bucket'] = record['_buckets']['deposit'] deposit['_oai'] = {} deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert record['_oai'] == initial_oai assert record['_files'][0]['bucket'] == record['_buckets']['record']
def test_communities_newversion_while_ir_pending_bug(app, db, users, communities, deposit, deposit_file): """Make sure that pending IRs remain after a new version (bug).""" deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value # Two inclusion requests are pending assert InclusionRequest.query.count() == 2 # Accept one community c1_api = ZenodoCommunity('c1') c1_api.accept_record(record_v1, pid=recid_v1) deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) # Make sure there is still IR to community 'c2' after newversion assert InclusionRequest.query.count() == 1 assert InclusionRequest.query.one().id_community == 'c2' assert record_v1.get('communities', []) == [ 'c1', ] assert deposit_v1.get('communities', []) == [ 'c1', 'c2', ] assert record_v2.get('communities', []) == [ 'c1', ] assert deposit_v2.get('communities', []) == [ 'c1', 'c2', ]
def test_record_delete(mocker, app, db, users, deposit, deposit_file): """Delete the record with a single version.""" dc_mock = mocker.patch( 'invenio_pidstore.providers.datacite.DataCiteMDSClient') deposit = publish_and_expunge(db, deposit) recid, record = deposit.fetch_published() # Stash a copy of record metadata for later rec = deepcopy(record) record_uuid = str(record.id) assert dc_mock().metadata_delete.call_count == 0 # users[0] is not an Admin but it doesn't matter in this case. delete_record(record.id, 'spam', users[0]['id']) # Make sure all PIDs are deleted # TODO: oai PID is left registered # assert PID.get('oai', rec['_oai']['id']) == PIDStatus.DELETED assert PID.get('doi', rec['doi']).status == PIDStatus.DELETED assert PID.get('doi', rec['conceptdoi']).status == PIDStatus.DELETED assert PID.get('recid', rec['recid']).status == PIDStatus.DELETED assert PID.get('recid', rec['conceptrecid']).status == PIDStatus.DELETED assert PID.get('depid', rec['_deposit']['id']).status == PIDStatus.DELETED assert dc_mock().metadata_delete.call_count == 2 dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.1') dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.2') record = Record.get_record(record_uuid) assert record['removed_by'] == users[0]['id'] assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def test_record_delete_legacy(dc_mock, app, db, users, deposit, deposit_file): """Delete the non-versioned record.""" deposit = publish_and_expunge(db, deposit) recid, record = deposit.fetch_published() # 'Simulate' a non-versioned record by removing 'conceptdoi' key del deposit['conceptdoi'] del record['conceptdoi'] deposit.commit() record.commit() db.session.commit() # Stash a copy of record metadata for later rec = deepcopy(record) record_uuid = str(record.id) assert dc_mock().metadata_delete.call_count == 0 # users[0] is not an Admin but it doesn't matter in this case. delete_record(record.id, 'spam', users[0]['id']) # Make sure all PIDs are deleted # TODO: oai PID is left registered # assert PID.get('oai', rec['_oai']['id']) == PIDStatus.DELETED assert PID.get('doi', rec['doi']).status == PIDStatus.DELETED assert PID.get('recid', rec['recid']).status == PIDStatus.DELETED assert PID.get('recid', rec['conceptrecid']).status == PIDStatus.DELETED assert PID.get('depid', rec['_deposit']['id']).status == PIDStatus.DELETED assert dc_mock().metadata_delete.call_count == 1 dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.2') record = Record.get_record(record_uuid) assert record['removed_by'] == users[0]['id'] assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def test_communities_newversion_addition(app, db, users, communities, deposit, deposit_file): """Make sure that new version of record synchronizes the communities.""" deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') c1_api.accept_record(record_v1, pid=recid_v1) c2_api.accept_record(record_v1, pid=recid_v1) deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) # Remove 'c2' and request for 'c5'. Make sure that communities from # previous record version are preserved/removed properly deposit_v2['communities'] = ['c1', 'c5'] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) assert record_v1.get('communities', []) == [ 'c1', ] assert deposit_v1.get('communities', []) == [ 'c1', 'c5', ] assert record_v2.get('communities', []) == [ 'c1', ] assert deposit_v2.get('communities', []) == [ 'c1', 'c5', ]
def test_remove_community_by_key_del(app, db, communities, deposit, deposit_file): """Test removal of communities by key deletion. Communities can be removed by not providing or deleting the communities from the key deposit. Moreover, the redundant 'empty' keys should not be automatically added to deposit nor record. """ # If 'communities' key was not in deposit metadata, # it shouldn't be automatically added assert 'communities' not in deposit deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert 'communities' not in deposit assert 'communities' not in record assert not record['_oai'].get('sets', []) # Request for 'c1' and 'c2' deposit = deposit.edit() deposit['communities'] = ['c1', 'c2', ] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() # No reason to have 'communities' in record since nothing was accepted assert 'communities' not in record assert not record['_oai'].get('sets', []) # Accept 'c1' c1 = Community.get('c1') c1.accept_record(record) record.commit() pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', 'c2', ] assert InclusionRequest.query.count() == 1 assert record['communities'] == ['c1', ] assert set(record['_oai']['sets']) == set(['user-c1']) # Remove the key from deposit and publish deposit = deposit.edit() del deposit['communities'] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert 'communities' not in deposit assert 'communities' not in record assert InclusionRequest.query.count() == 0 assert not record['_oai'].get('sets', [])
def test_remove_obsolete_irs(app, db, communities, deposit, deposit_file): """Test removal of obsolete IRs in-between deposit edits.""" # Request for 'c1' deposit['communities'] = ['c1', ] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert InclusionRequest.query.count() == 1 assert deposit['communities'] == ['c1', ] assert not record.get('communities', []) # Open for edit and remove the request to community 'c1' deposit = deposit.edit() deposit['communities'] = [] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert InclusionRequest.query.count() == 0 assert not deposit.get('communities', []) assert not record.get('communities', [])
def test_accept_while_edit(app, db, communities, deposit, deposit_file): """Test deposit publishing with concurrent events. Accept a record, while deposit in open edit and then published. """ deposit['communities'] = ['c1', 'c2'] deposit = publish_and_expunge(db, deposit) assert InclusionRequest.query.count() == 2 pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', 'c2'] assert not record.get('communities', []) assert not record['_oai'].get('sets', []) # Open for edit deposit = deposit.edit() pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', 'c2'] assert not record.get('communities', []) assert not record['_oai'].get('sets', []) assert InclusionRequest.query.count() == 2 # Accept a record meanwhile c1 = Community.get('c1') c1.accept_record(record) record.commit() db.session.commit() # Publish and make sure nothing is missing deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', 'c2'] assert record['communities'] == [ 'c1', ] assert record['_oai']['sets'] == [ 'user-c1', ] assert InclusionRequest.query.count() == 1 ir = InclusionRequest.query.one() assert ir.id_community == 'c2' assert ir.id_record == record.id
def test_autoadd_explicit( app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Explicitly the autoadded communities.""" deposit['communities'] = ['ecfunded', 'grants_comm', 'zenodo'] deposit['grants'] = [{'title': 'SomeGrant'}, ] deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() assert record_v1.get('communities', []) == ['grants_comm', ] assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo']
def test_reject_while_edit(app, db, communities, deposit, deposit_file): """Test deposit publishing with concurrent events. Reject a record, while deposit in open edit and published. """ # Request for community 'c1' deposit['communities'] = [ 'c1', ] deposit = publish_and_expunge(db, deposit) assert deposit['communities'] == [ 'c1', ] pid, record = deposit.fetch_published() assert not record.get('communities', []) assert InclusionRequest.query.count() == 1 ir = InclusionRequest.query.one() assert ir.id_community == 'c1' assert ir.id_record == record.id # Open deposit in edit mode and request another community 'c2' deposit = deposit.edit() deposit['communities'] = ['c1', 'c2'] # Reject the request for community 'c1' c1 = Community.get('c1') c1.reject_record(record) db.session.commit() # Publish the deposit deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() # NOTE: 'c1' is requested again! assert InclusionRequest.query.count() == 2 ir1 = InclusionRequest.query.filter_by(id_community='c1').one() ir2 = InclusionRequest.query.filter_by(id_community='c2').one() assert ir1.id_record == record.id assert ir2.id_record == record.id assert deposit['communities'] == ['c1', 'c2'] assert not record.get('communities', [])
def test_accept_while_edit(app, db, communities, deposit, deposit_file): """Test deposit publishing with concurrent events. Accept a record, while deposit in open edit and then published. """ deposit['communities'] = ['c1', 'c2'] deposit = publish_and_expunge(db, deposit) assert InclusionRequest.query.count() == 2 pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', 'c2'] assert not record.get('communities', []) assert not record['_oai'].get('sets', []) # Open for edit deposit = deposit.edit() pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', 'c2'] assert not record.get('communities', []) assert not record['_oai'].get('sets', []) assert InclusionRequest.query.count() == 2 # Accept a record meanwhile c1 = Community.get('c1') c1.accept_record(record) record.commit() db.session.commit() # Publish and make sure nothing is missing deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert deposit['communities'] == ['c1', 'c2'] assert record['communities'] == ['c1', ] assert record['_oai']['sets'] == ['user-c1', ] assert InclusionRequest.query.count() == 1 ir = InclusionRequest.query.one() assert ir.id_community == 'c2' assert ir.id_record == record.id
def test_communities_newversion_while_ir_pending_bug( app, db, users, communities, deposit, deposit_file): """Make sure that pending IRs remain after a new version (bug).""" deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value # Two inclusion requests are pending assert InclusionRequest.query.count() == 2 # Accept one community c1_api = ZenodoCommunity('c1') c1_api.accept_record(record_v1, pid=recid_v1) deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) # Make sure there is still IR to community 'c2' after newversion assert InclusionRequest.query.count() == 1 assert InclusionRequest.query.one().id_community == 'c2' assert record_v1.get('communities', []) == ['c1', ] assert deposit_v1.get('communities', []) == ['c1', 'c2', ] assert record_v2.get('communities', []) == ['c1', ] assert deposit_v2.get('communities', []) == ['c1', 'c2', ]
def test_communities_newversion_addition( app, db, users, communities, deposit, deposit_file): """Make sure that new version of record synchronizes the communities.""" deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') c1_api.accept_record(record_v1, pid=recid_v1) c2_api.accept_record(record_v1, pid=recid_v1) deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) # Remove 'c2' and request for 'c5'. Make sure that communities from # previous record version are preserved/removed properly deposit_v2['communities'] = ['c1', 'c5'] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) assert record_v1.get('communities', []) == ['c1', ] assert deposit_v1.get('communities', []) == ['c1', 'c5', ] assert record_v2.get('communities', []) == ['c1', ] assert deposit_v2.get('communities', []) == ['c1', 'c5', ]
def test_reject_while_edit(app, db, communities, deposit, deposit_file): """Test deposit publishing with concurrent events. Reject a record, while deposit in open edit and published. """ # Request for community 'c1' deposit['communities'] = ['c1', ] deposit = publish_and_expunge(db, deposit) assert deposit['communities'] == ['c1', ] pid, record = deposit.fetch_published() assert not record.get('communities', []) assert InclusionRequest.query.count() == 1 ir = InclusionRequest.query.one() assert ir.id_community == 'c1' assert ir.id_record == record.id # Open deposit in edit mode and request another community 'c2' deposit = deposit.edit() deposit['communities'] = ['c1', 'c2'] # Reject the request for community 'c1' c1 = Community.get('c1') c1.reject_record(record) db.session.commit() # Publish the deposit deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() # NOTE: 'c1' is requested again! assert InclusionRequest.query.count() == 2 ir1 = InclusionRequest.query.filter_by(id_community='c1').one() ir2 = InclusionRequest.query.filter_by(id_community='c2').one() assert ir1.id_record == record.id assert ir2.id_record == record.id assert deposit['communities'] == ['c1', 'c2'] assert not record.get('communities', [])
def test_fixed_communities(app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Test automatic adding and requesting to fixed communities.""" deposit['grants'] = [{'title': 'SomeGrant'}, ] # 'c3' is owned by one of the deposit owner assert Community.get('c3').id_user in deposit['_deposit']['owners'] deposit['communities'] = ['c3', ] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert record['communities'] == ['c3', 'grants_comm'] assert deposit['communities'] == ['c3', 'ecfunded', 'grants_comm', 'zenodo'] InclusionRequest.query.count() == 2 ir1 = InclusionRequest.query.filter_by(id_community='zenodo').one() assert ir1.id_record == record.id ir2 = InclusionRequest.query.filter_by(id_community='ecfunded').one() assert ir2.id_record == record.id
def test_basic_community_workflow(app, db, communities, deposit, deposit_file): """Test simple (without concurrent events) deposit publishing workflow.""" deposit = publish_and_expunge(db, deposit) assert InclusionRequest.query.count() == 0 pid, record = deposit.fetch_published() assert not record.get('communities', []) # Open record for edit, request a community and publish deposit = deposit.edit() deposit['communities'] = ['c1', ] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() # Should contain just an InclusionRequest assert not record.get('communities', []) assert not record['_oai'].get('sets', []) assert InclusionRequest.query.count() == 1 ir = InclusionRequest.query.one() assert ir.id_community == 'c1' assert ir.id_record == record.id # Accept a record to the community 'c1' c1 = Community.get('c1') c1.accept_record(record) record.commit() db.session.commit() assert InclusionRequest.query.count() == 0 assert record['communities'] == ['c1', ] assert record['_oai']['sets'] == ['user-c1', ] # Open for edit and request another community deposit = deposit.edit() assert deposit['communities'] == ['c1', ] deposit['communities'] = ['c1', 'c2', ] # New request for community 'c2' deposit = publish_and_expunge(db, deposit) deposit['communities'] = ['c1', 'c2', ] pid, record = deposit.fetch_published() assert record['communities'] == ['c1', ] assert record['_oai']['sets'] == ['user-c1', ] assert InclusionRequest.query.count() == 1 ir = InclusionRequest.query.one() assert ir.id_community == 'c2' assert ir.id_record == record.id # Reject the request for community 'c2' c2 = Community.get('c2') c2.reject_record(record) db.session.commit() deposit = deposit.edit() # The deposit should not contain obsolete inclusion requests assert deposit['communities'] == ['c1', ] assert InclusionRequest.query.count() == 0 pid, record = deposit.fetch_published() assert record['communities'] == ['c1', ] assert record['_oai']['sets'] == ['user-c1', ] # Request for removal from a previously accepted community 'c1' deposit['communities'] = [] deposit = publish_and_expunge(db, deposit) pid, record = deposit.fetch_published() assert not deposit.get('communities', []) assert not record.get('communities', []) assert not record['_oai'].get('sets', []) assert InclusionRequest.query.count() == 0
def test_autoadd(app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Test basic workflow using Deposit and Communities API.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c3' (owned by user) through deposit v2 deposit_v2['communities'] = ['c1', 'c2', 'c3', ] deposit_v2['grants'] = [{'title': 'SomeGrant'}, ] deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() assert record_v2['grants'] == [{'title': 'SomeGrant'}, ] recid_v2_value = recid_v2.pid_value depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() assert record_v1.get('communities', []) == ['c3', 'grants_comm'] assert record_v2.get('communities', []) == ['c3', 'grants_comm'] assert deposit_v1.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'] assert deposit_v2.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'] c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') c3_api = ZenodoCommunity('c3') grants_comm_api = ZenodoCommunity('grants_comm') ecfunded_api = ZenodoCommunity('ecfunded') zenodo_api = ZenodoCommunity('zenodo') # Inclusion requests should be visible for both records assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c3_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0 assert c3_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0 assert grants_comm_api.get_comm_irs( record_v1, pid=recid_v1).count() == 0 assert grants_comm_api.get_comm_irs( record_v2, pid=recid_v2).count() == 0 assert ecfunded_api.get_comm_irs( record_v1, pid=recid_v1).count() == 1 assert ecfunded_api.get_comm_irs( record_v2, pid=recid_v2).count() == 1 assert zenodo_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert zenodo_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 # Accept to 'c1' through record_v2 (as originally requested), # and 'c2' through record_v1 (resolved through version) c1_api.accept_record(record_v2, pid=recid_v2) c2_api.accept_record(record_v1, pid=recid_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) # Accepting individual record to a community should propagate the changes # to all versions assert record_v1.get('communities', []) == ['c1', 'c2', 'c3', 'grants_comm'] assert record_v2.get('communities', []) == ['c1', 'c2', 'c3', 'grants_comm'] assert deposit_v1.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'] assert deposit_v2.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'] # Removing 'c1'-'c3' from deposit_v1 should remove it from two published # records and other deposits as well depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1 = deposit_v1.edit() deposit_v1['communities'] = [] deposit_v1 = publish_and_expunge(db, deposit_v1) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) assert record_v1.get('communities', []) == ['grants_comm', ] assert record_v2.get('communities', []) == ['grants_comm', ] assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo'] assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo']
def test_relations_serialization(app, db, deposit, deposit_file): """Serialize PID relations.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() expected = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 1 }] } assert serialize_relations(recid_v1) == expected deposit_v1.newversion() # Should contain "draft_child_deposit" information expected = { "version": [{ "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 1, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert serialize_relations(recid_v1) == expected # Publish the new version pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() # Should no longer contain "draft_child_deposit" info after publishing # and no longer be the last child expected = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 }] } assert serialize_relations(recid_v1) == expected # New version should be the last child now expected = { "version": [{ "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert serialize_relations(recid_v2) == expected
def test_archiving(app, db, deposit, deposit_file, locations, archive_fs): """Test ZenodoSIP archiving.""" # Stash the configuration and enable writing orig = app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] = True deposit.files['test2.txt'] = BytesIO(b'test-two') deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_id = recid_v1.id # Record files after publishing: 'test.txt', 'test2.txt' sip1 = SIP(SIPModel.query.one()) sip1_id = sip1.id # Edit the metadata deposit_v1 = deposit_v1.edit() deposit_v1['title'] = "New title" deposit_v1 = publish_and_expunge(db, deposit_v1) # Record files after publishing: 'test.txt', 'test2.txt' sip2_id = SIPModel.query.order_by(SIPModel.created.desc()).first().id # Create a new version deposit_v1.newversion() recid_v1 = PersistentIdentifier.query.get(recid_v1_id) pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid) del deposit_v2.files['test.txt'] deposit_v2.files['test3.txt'] = BytesIO(b('test-three')) deposit_v2 = publish_and_expunge(db, deposit_v2) # Record files after publishing: 'test2.txt', 'test3.txt' sip1 = SIP(SIPModel.query.get(sip1_id)) sip2 = SIP(SIPModel.query.get(sip2_id)) sip3 = SIP(SIPModel.query.order_by(SIPModel.created.desc()).first()) # Becase we are using secure_filename when writing SIPFiles we need to # genenarate the correct names: <SIPFile.id>-<secure_filename> s1_file1_fn = '{0}-test.txt'.format(fetch_suff(sip1, 'test.txt').file_id) s1_file1_fp = 'data/files/{0}'.format(s1_file1_fn) s1_file2_fn = '{0}-test2.txt'.format(fetch_suff(sip1, 'test2.txt').file_id) s1_file2_fp = 'data/files/{0}'.format(s1_file2_fn) s3_file2_fn = '{0}-test2.txt'.format(fetch_suff(sip3, 'test2.txt').file_id) s3_file2_fp = 'data/files/{0}'.format(s3_file2_fn) s3_file3_fn = '{0}-test3.txt'.format(fetch_suff(sip3, 'test3.txt').file_id) s3_file3_fp = 'data/files/{0}'.format(s3_file3_fn) sip1_bagmeta = json.loads(next( m.content for m in sip1.metadata if m.type.name == 'bagit'))['files'] sip2_bagmeta = json.loads(next( m.content for m in sip2.metadata if m.type.name == 'bagit'))['files'] sip3_bagmeta = json.loads(next( m.content for m in sip3.metadata if m.type.name == 'bagit'))['files'] # Check if Bagit metadata contains the correct file-fetching information assert set([f['filepath'] for f in sip1_bagmeta]) == \ set([s1_file1_fp, s1_file2_fp, 'data/filenames.txt', 'data/metadata/record-json.json', 'bag-info.txt', 'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt']) assert not BagItArchiver._is_fetched( get_m_item(sip1_bagmeta, s1_file1_fp)) assert not BagItArchiver._is_fetched( get_m_item(sip1_bagmeta, s1_file2_fp)) assert set([f['filepath'] for f in sip2_bagmeta]) == \ set([s1_file1_fp, s1_file2_fp, 'data/filenames.txt', 'data/metadata/record-json.json', 'bag-info.txt', 'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt', 'fetch.txt']) # Both files should be fetched since it's only metadata-edit submission assert BagItArchiver._is_fetched( get_m_item(sip2_bagmeta, s1_file1_fp)) assert BagItArchiver._is_fetched( get_m_item(sip2_bagmeta, s1_file2_fp)) assert set([f['filepath'] for f in sip3_bagmeta]) == \ set([s3_file2_fp, s3_file3_fp, 'data/filenames.txt', 'data/metadata/record-json.json', 'bag-info.txt', 'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt', 'fetch.txt']) # First file should be fetched from previous version and new file should # be archived in this bag. assert BagItArchiver._is_fetched( get_m_item(sip3_bagmeta, s3_file2_fp)) assert not BagItArchiver._is_fetched( get_m_item(sip3_bagmeta, s3_file3_fp)) archiver1 = BagItArchiver(sip1) archiver2 = BagItArchiver(sip2) archiver3 = BagItArchiver(sip3) # Each archiver subpath follows: '<recid>/r/<ISO-8601-SIP-timestamp>' sip1_ts = arrow.get(sip1.model.created).isoformat() sip2_ts = arrow.get(sip2.model.created).isoformat() sip3_ts = arrow.get(sip3.model.created).isoformat() assert archiver1.get_archive_subpath() == '2/r/{0}'.format(sip1_ts) assert archiver2.get_archive_subpath() == '2/r/{0}'.format(sip2_ts) assert archiver3.get_archive_subpath() == '3/r/{0}'.format(sip3_ts) # As a test, write the SIPs in reverse chronological order assert not sip1.archived assert not sip2.archived assert not sip3.archived archive_sip.delay(sip3.id) archive_sip.delay(sip2.id) archive_sip.delay(sip1.id) assert sip1.archived assert sip2.archived assert sip3.archived fs1 = archive_fs.opendir(archiver1.get_archive_subpath()) assert set(fs1.listdir()) == set(['tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt', 'bag-info.txt', 'data']) assert set(fs1.listdir('data')) == set(['metadata', 'files', 'filenames.txt']) assert fs1.listdir('data/metadata') == ['record-json.json', ] assert set(fs1.listdir('data/files')) == set([s1_file1_fn, s1_file2_fn]) fs2 = archive_fs.opendir(archiver2.get_archive_subpath()) assert set(fs2.listdir()) == set(['tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt', 'bag-info.txt', 'data', 'fetch.txt']) # Second SIP has written only the metadata, # because of that There should be no 'files/', but 'filesnames.txt' should # still be there becasue of the fetch.txt assert set(fs2.listdir('data')) == set(['metadata', 'filenames.txt']) assert fs2.listdir('data/metadata') == ['record-json.json', ] with fs2.open('fetch.txt') as fp: cnt = fp.read().splitlines() # Fetched files should correctly fetch the files from the first archive base_uri = archiver1.get_archive_base_uri() assert set(cnt) == set([ '{base}/2/r/{s1ts}/{fn} 4 {fn}'.format(fn=s1_file1_fp, base=base_uri, s1ts=sip1_ts), '{base}/2/r/{s1ts}/{fn} 8 {fn}'.format(fn=s1_file2_fp, base=base_uri, s1ts=sip1_ts), ]) fs3 = archive_fs.opendir(archiver3.get_archive_subpath()) assert set(fs3.listdir()) == set(['tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt', 'bag-info.txt', 'data', 'fetch.txt']) # Third SIP should write only the extra 'test3.txt' file assert set(fs3.listdir('data')) == set(['metadata', 'files', 'filenames.txt']) assert fs3.listdir('data/metadata') == ['record-json.json', ] assert fs3.listdir('data/files') == [s3_file3_fn, ] with fs3.open('fetch.txt') as fp: cnt = fp.read().splitlines() # Since 'file.txt' was removed in third SIP, we should only fetch the # 'test2.txt', also from the first archive, since that's where this # file resides physically. base_uri = archiver1.get_archive_base_uri() assert set(cnt) == set([ '{base}/2/r/{s1ts}/{fn} 8 {fn}'.format(fn=s3_file2_fp, base=base_uri, s1ts=sip1_ts), ]) app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] = orig
def test_related_identifiers_serialization(app, db, deposit, deposit_file): """Serialize PID Relations to related identifiers.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c2' through deposit v2 deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) rids = serialize_related_identifiers(recid_v1) expected_v1 = [ { 'scheme': 'doi', 'identifier': '10.5072/zenodo.1', 'relation': 'isPartOf' } # TODO: serialization of new version realtions is disabled # { # 'scheme': 'doi', # 'identifier': '10.5072/zenodo.3', # 'relation': 'isPreviousVersionOf' # } ] assert rids == expected_v1 rids = serialize_related_identifiers(recid_v2) expected_v2 = [ { 'scheme': 'doi', 'identifier': '10.5072/zenodo.1', 'relation': 'isPartOf' } # TODO: serialization of new version realtions is disabled # { # 'scheme': 'doi', # 'identifier': '10.5072/zenodo.2', # 'relation': 'isNewVersionOf' # } ] assert rids == expected_v2 parent_pid = PersistentIdentifier.get('recid', '1') rids = serialize_related_identifiers(parent_pid) expected_parent = [ { 'relation': 'hasPart', 'scheme': 'doi', 'identifier': '10.5072/zenodo.2' }, { 'relation': 'hasPart', 'scheme': 'doi', 'identifier': '10.5072/zenodo.3' } ] assert rids == expected_parent
def test_versioning_indexing(db, es, deposit, deposit_file): """Test the indexing of 'version' relations.""" deposit_index_name = 'deposits-records-record-v1.0.0' records_index_name = 'records-record-v1.0.0' deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value RecordIndexer().index_by_id(str(record_v1.id)) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 1 assert len(s_rec) == 1 assert 'relations' in s_dep[0]['_source'] assert 'relations' in s_rec[0]['_source'] expected = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 1, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert s_dep[0]['_source']['relations'] == expected assert s_rec[0]['_source']['relations'] == expected deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid) deposit_v2.files['file.txt'] = BytesIO(b('file1')) depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 2 # Two deposits should be indexed assert len(s_rec) == 1 # One, since record does not exist yet s_dep1 = current_search.client.get(index=deposit_index_name, id=deposit_v1.id) s_dep2 = current_search.client.get(index=deposit_index_name, id=deposit_v2.id) expected_d1 = { "version": [{ "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "2" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 # For deposit, draft children are also counted }] } expected_d2 = { "version": [{ "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 2, # For deposit, draft children are also counted "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert s_dep1['_source']['relations'] == expected_d1 assert s_dep2['_source']['relations'] == expected_d2 deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) RecordIndexer().index_by_id(str(record_v2.id)) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 2 assert len(s_rec) == 2 s_dep1 = current_search.client.get(index=deposit_index_name, id=deposit_v1.id) s_dep2 = current_search.client.get(index=deposit_index_name, id=deposit_v2.id) s_rec1 = current_search.client.get(index=records_index_name, id=record_v1.id) s_rec2 = current_search.client.get(index=records_index_name, id=record_v2.id) expected_d1 = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 }] } expected_d2 = { "version": [{ "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert s_dep1['_source']['relations'] == expected_d1 assert s_dep2['_source']['relations'] == expected_d2 expected_r1 = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 }] } expected_r2 = { "version": [{ "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert s_rec1['_source']['relations'] == expected_r1 assert s_rec2['_source']['relations'] == expected_r2
def test_archiving(app, db, deposit, deposit_file, locations, archive_fs): """Test ZenodoSIP archiving.""" # Stash the configuration and enable writing orig = app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] = True deposit.files['test2.txt'] = BytesIO(b'test-two') deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_id = recid_v1.id # Record files after publishing: 'test.txt', 'test2.txt' sip1 = SIP(SIPModel.query.one()) sip1_id = sip1.id # Edit the metadata deposit_v1 = deposit_v1.edit() deposit_v1['title'] = "New title" deposit_v1 = publish_and_expunge(db, deposit_v1) # Record files after publishing: 'test.txt', 'test2.txt' sip2_id = SIPModel.query.order_by(SIPModel.created.desc()).first().id # Create a new version deposit_v1.newversion() recid_v1 = PersistentIdentifier.query.get(recid_v1_id) pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid) del deposit_v2.files['test.txt'] deposit_v2.files['test3.txt'] = BytesIO(b('test-three')) deposit_v2 = publish_and_expunge(db, deposit_v2) # Record files after publishing: 'test2.txt', 'test3.txt' sip1 = SIP(SIPModel.query.get(sip1_id)) sip2 = SIP(SIPModel.query.get(sip2_id)) sip3 = SIP(SIPModel.query.order_by(SIPModel.created.desc()).first()) # Becase we are using secure_filename when writing SIPFiles we need to # genenarate the correct names: <SIPFile.id>-<secure_filename> s1_file1_fn = '{0}-test.txt'.format(fetch_suff(sip1, 'test.txt').file_id) s1_file1_fp = 'data/files/{0}'.format(s1_file1_fn) s1_file2_fn = '{0}-test2.txt'.format(fetch_suff(sip1, 'test2.txt').file_id) s1_file2_fp = 'data/files/{0}'.format(s1_file2_fn) s3_file2_fn = '{0}-test2.txt'.format(fetch_suff(sip3, 'test2.txt').file_id) s3_file2_fp = 'data/files/{0}'.format(s3_file2_fn) s3_file3_fn = '{0}-test3.txt'.format(fetch_suff(sip3, 'test3.txt').file_id) s3_file3_fp = 'data/files/{0}'.format(s3_file3_fn) sip1_bagmeta = json.loads( next(m.content for m in sip1.metadata if m.type.name == 'bagit'))['files'] sip2_bagmeta = json.loads( next(m.content for m in sip2.metadata if m.type.name == 'bagit'))['files'] sip3_bagmeta = json.loads( next(m.content for m in sip3.metadata if m.type.name == 'bagit'))['files'] # Check if Bagit metadata contains the correct file-fetching information assert set([f['filepath'] for f in sip1_bagmeta]) == \ set([s1_file1_fp, s1_file2_fp, 'data/filenames.txt', 'data/metadata/record-json.json', 'bag-info.txt', 'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt']) assert not BagItArchiver._is_fetched(get_m_item(sip1_bagmeta, s1_file1_fp)) assert not BagItArchiver._is_fetched(get_m_item(sip1_bagmeta, s1_file2_fp)) assert set([f['filepath'] for f in sip2_bagmeta]) == \ set([s1_file1_fp, s1_file2_fp, 'data/filenames.txt', 'data/metadata/record-json.json', 'bag-info.txt', 'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt', 'fetch.txt']) # Both files should be fetched since it's only metadata-edit submission assert BagItArchiver._is_fetched(get_m_item(sip2_bagmeta, s1_file1_fp)) assert BagItArchiver._is_fetched(get_m_item(sip2_bagmeta, s1_file2_fp)) assert set([f['filepath'] for f in sip3_bagmeta]) == \ set([s3_file2_fp, s3_file3_fp, 'data/filenames.txt', 'data/metadata/record-json.json', 'bag-info.txt', 'manifest-md5.txt', 'bagit.txt', 'tagmanifest-md5.txt', 'fetch.txt']) # First file should be fetched from previous version and new file should # be archived in this bag. assert BagItArchiver._is_fetched(get_m_item(sip3_bagmeta, s3_file2_fp)) assert not BagItArchiver._is_fetched(get_m_item(sip3_bagmeta, s3_file3_fp)) archiver1 = BagItArchiver(sip1) archiver2 = BagItArchiver(sip2) archiver3 = BagItArchiver(sip3) # Each archiver subpath follows: '<recid>/r/<ISO-8601-SIP-timestamp>' sip1_ts = arrow.get(sip1.model.created).isoformat() sip2_ts = arrow.get(sip2.model.created).isoformat() sip3_ts = arrow.get(sip3.model.created).isoformat() assert archiver1.get_archive_subpath() == '2/r/{0}'.format(sip1_ts) assert archiver2.get_archive_subpath() == '2/r/{0}'.format(sip2_ts) assert archiver3.get_archive_subpath() == '3/r/{0}'.format(sip3_ts) # As a test, write the SIPs in reverse chronological order assert not sip1.archived assert not sip2.archived assert not sip3.archived archive_sip.delay(sip3.id) archive_sip.delay(sip2.id) archive_sip.delay(sip1.id) assert sip1.archived assert sip2.archived assert sip3.archived fs1 = archive_fs.opendir(archiver1.get_archive_subpath()) assert set(fs1.listdir()) == set([ 'tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt', 'bag-info.txt', 'data' ]) assert set(fs1.listdir('data')) == set( ['metadata', 'files', 'filenames.txt']) assert fs1.listdir('data/metadata') == [ 'record-json.json', ] assert set(fs1.listdir('data/files')) == set([s1_file1_fn, s1_file2_fn]) fs2 = archive_fs.opendir(archiver2.get_archive_subpath()) assert set(fs2.listdir()) == set([ 'tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt', 'bag-info.txt', 'data', 'fetch.txt' ]) # Second SIP has written only the metadata, # because of that There should be no 'files/', but 'filesnames.txt' should # still be there becasue of the fetch.txt assert set(fs2.listdir('data')) == set(['metadata', 'filenames.txt']) assert fs2.listdir('data/metadata') == [ 'record-json.json', ] with fs2.open('fetch.txt') as fp: cnt = fp.read().splitlines() # Fetched files should correctly fetch the files from the first archive base_uri = archiver1.get_archive_base_uri() assert set(cnt) == set([ '{base}/2/r/{s1ts}/{fn} 4 {fn}'.format(fn=s1_file1_fp, base=base_uri, s1ts=sip1_ts), '{base}/2/r/{s1ts}/{fn} 8 {fn}'.format(fn=s1_file2_fp, base=base_uri, s1ts=sip1_ts), ]) fs3 = archive_fs.opendir(archiver3.get_archive_subpath()) assert set(fs3.listdir()) == set([ 'tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt', 'bag-info.txt', 'data', 'fetch.txt' ]) # Third SIP should write only the extra 'test3.txt' file assert set(fs3.listdir('data')) == set( ['metadata', 'files', 'filenames.txt']) assert fs3.listdir('data/metadata') == [ 'record-json.json', ] assert fs3.listdir('data/files') == [ s3_file3_fn, ] with fs3.open('fetch.txt') as fp: cnt = fp.read().splitlines() # Since 'file.txt' was removed in third SIP, we should only fetch the # 'test2.txt', also from the first archive, since that's where this # file resides physically. base_uri = archiver1.get_archive_base_uri() assert set(cnt) == set([ '{base}/2/r/{s1ts}/{fn} 8 {fn}'.format(fn=s3_file2_fp, base=base_uri, s1ts=sip1_ts), ]) app.config['SIPSTORE_ARCHIVER_WRITING_ENABLED'] = orig
def test_relations_serialization(app, db, deposit, deposit_file): """Serialize PID relations.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() expected = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 1 } ] } assert serialize_relations(recid_v1) == expected deposit_v1.newversion() # Should contain "draft_child_deposit" information expected = { "version": [ { "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 1, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert serialize_relations(recid_v1) == expected # Publish the new version pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() # Should no longer contain "draft_child_deposit" info after publishing # and no longer be the last child expected = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 } ] } assert serialize_relations(recid_v1) == expected # New version should be the last child now expected = { "version": [ { "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert serialize_relations(recid_v2) == expected
def test_autoadd(app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Test basic workflow using Deposit and Communities API.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c3' (owned by user) through deposit v2 deposit_v2['communities'] = [ 'c1', 'c2', 'c3', ] deposit_v2['grants'] = [ { 'title': 'SomeGrant' }, ] deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() assert record_v2['grants'] == [ { 'title': 'SomeGrant' }, ] recid_v2_value = recid_v2.pid_value depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() assert record_v1.get('communities', []) == ['c3', 'grants_comm'] assert record_v2.get('communities', []) == ['c3', 'grants_comm'] assert deposit_v1.get('communities', []) == [ 'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo' ] assert deposit_v2.get('communities', []) == [ 'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo' ] c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') c3_api = ZenodoCommunity('c3') grants_comm_api = ZenodoCommunity('grants_comm') ecfunded_api = ZenodoCommunity('ecfunded') zenodo_api = ZenodoCommunity('zenodo') # Inclusion requests should be visible for both records assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c3_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0 assert c3_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0 assert grants_comm_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0 assert grants_comm_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0 assert ecfunded_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert ecfunded_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert zenodo_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert zenodo_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 # Accept to 'c1' through record_v2 (as originally requested), # and 'c2' through record_v1 (resolved through version) c1_api.accept_record(record_v2, pid=recid_v2) c2_api.accept_record(record_v1, pid=recid_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) # Accepting individual record to a community should propagate the changes # to all versions assert record_v1.get('communities', []) == ['c1', 'c2', 'c3', 'grants_comm'] assert record_v2.get('communities', []) == ['c1', 'c2', 'c3', 'grants_comm'] assert deposit_v1.get('communities', []) == [ 'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo' ] assert deposit_v2.get('communities', []) == [ 'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo' ] # Removing 'c1'-'c3' from deposit_v1 should remove it from two published # records and other deposits as well depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1 = deposit_v1.edit() deposit_v1['communities'] = [] deposit_v1 = publish_and_expunge(db, deposit_v1) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) assert record_v1.get('communities', []) == [ 'grants_comm', ] assert record_v2.get('communities', []) == [ 'grants_comm', ] assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo'] assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo']
def test_versioning_indexing(db, es, deposit, deposit_file): """Test the indexing of 'version' relations.""" deposit_index_name = 'deposits-records-record-v1.0.0' records_index_name = 'records-record-v1.0.0' deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value RecordIndexer().index_by_id(str(record_v1.id)) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 1 assert len(s_rec) == 1 assert 'relations' in s_dep[0]['_source'] assert 'relations' in s_rec[0]['_source'] expected = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 1, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert s_dep[0]['_source']['relations'] == expected assert s_rec[0]['_source']['relations'] == expected deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid) depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 2 # Two deposits should be indexed assert len(s_rec) == 1 # One, since record does not exist yet s_dep1 = current_search.client.get( index=deposit_index_name, id=deposit_v1.id) s_dep2 = current_search.client.get( index=deposit_index_name, id=deposit_v2.id) expected_d1 = { "version": [ { "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "2" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 # For deposit, draft children are also counted } ] } expected_d2 = { "version": [ { "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 2, # For deposit, draft children are also counted "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert s_dep1['_source']['relations'] == expected_d1 assert s_dep2['_source']['relations'] == expected_d2 deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) RecordIndexer().index_by_id(str(record_v2.id)) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 2 assert len(s_rec) == 2 s_dep1 = current_search.client.get( index=deposit_index_name, id=deposit_v1.id) s_dep2 = current_search.client.get( index=deposit_index_name, id=deposit_v2.id) s_rec1 = current_search.client.get( index=records_index_name, id=record_v1.id) s_rec2 = current_search.client.get( index=records_index_name, id=record_v2.id) expected_d1 = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 } ] } expected_d2 = { "version": [ { "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert s_dep1['_source']['relations'] == expected_d1 assert s_dep2['_source']['relations'] == expected_d2 expected_r1 = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 } ] } expected_r2 = { "version": [ { "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert s_rec1['_source']['relations'] == expected_r1 assert s_rec2['_source']['relations'] == expected_r2
def test_related_identifiers_serialization(app, db, deposit, deposit_file): """Serialize PID Relations to related identifiers.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c2' through deposit v2 deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) rids = serialize_related_identifiers(recid_v1) expected_v1 = [ { 'scheme': 'doi', 'identifier': '10.5072/zenodo.1', 'relation': 'isPartOf' } # TODO: serialization of new version realtions is disabled # { # 'scheme': 'doi', # 'identifier': '10.5072/zenodo.3', # 'relation': 'isPreviousVersionOf' # } ] assert rids == expected_v1 rids = serialize_related_identifiers(recid_v2) expected_v2 = [ { 'scheme': 'doi', 'identifier': '10.5072/zenodo.1', 'relation': 'isPartOf' } # TODO: serialization of new version realtions is disabled # { # 'scheme': 'doi', # 'identifier': '10.5072/zenodo.2', # 'relation': 'isNewVersionOf' # } ] assert rids == expected_v2 parent_pid = PersistentIdentifier.get('recid', '1') rids = serialize_related_identifiers(parent_pid) expected_parent = [{ 'relation': 'hasPart', 'scheme': 'doi', 'identifier': '10.5072/zenodo.2' }, { 'relation': 'hasPart', 'scheme': 'doi', 'identifier': '10.5072/zenodo.3' }] assert rids == expected_parent