def test_autoadd_explicit_newversion( app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Explicitly the autoadded communities in a new version.""" deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2['communities'] = ['ecfunded', 'grants_comm', 'zenodo'] deposit_v2['grants'] = [{'title': 'SomeGrant'}, ] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) assert record_v1.get('communities', []) == ['grants_comm', ] assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo'] assert record_v2.get('communities', []) == ['grants_comm', ] assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo']
def test_autoadd_explicit_newversion( app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Explicitly the autoadded communities in a new version.""" deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2['communities'] = ['ecfunded', 'grants_comm', 'zenodo'] deposit_v2['grants'] = [{'title': 'SomeGrant'}, ] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) assert record_v1.get('communities', []) == ['grants_comm', ] assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo'] assert record_v2.get('communities', []) == ['grants_comm', ] assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo']
def test_basic_api(app, db, communities, deposit, deposit_file): """Test basic workflow using Deposit and Communities API.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c2' through deposit v2 deposit_v2['communities'] = [ 'c1', 'c2', ] deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() recid_v2_value = recid_v2.pid_value depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() assert record_v1.get('communities', []) == [] assert record_v2.get('communities', []) == [] c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') # Inclusion requests should be visible for both records assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 # Accept to 'c1' through record_v2 (as originally requested), # and 'c2' through record_v1 (version) c1_api.accept_record(record_v2, pid=recid_v2) c2_api.accept_record(record_v1, pid=recid_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) # Accepting individual record to a community should propagate the changes # to all versions assert record_v1['communities'] == record_v2['communities'] == \ ['c1', 'c2', ] # Removing 'c1' from deposit_v1 should remove it from two published records depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1 = deposit_v1.edit() deposit_v1['communities'] = [] deposit_v1 = publish_and_expunge(db, deposit_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) assert record_v1.get('communities', []) == [] assert record_v2.get('communities', []) == []
def test_basic_api(app, db, communities, deposit, deposit_file): """Test basic workflow using Deposit and Communities API.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c2' through deposit v2 deposit_v2['communities'] = ['c1', 'c2', ] deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() recid_v2_value = recid_v2.pid_value depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() assert record_v1.get('communities', []) == [] assert record_v2.get('communities', []) == [] c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') # Inclusion requests should be visible for both records assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 # Accept to 'c1' through record_v2 (as originally requested), # and 'c2' through record_v1 (version) c1_api.accept_record(record_v2, pid=recid_v2) c2_api.accept_record(record_v1, pid=recid_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) # Accepting individual record to a community should propagate the changes # to all versions assert record_v1['communities'] == record_v2['communities'] == \ ['c1', 'c2', ] # Removing 'c1' from deposit_v1 should remove it from two published records depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1 = deposit_v1.edit() deposit_v1['communities'] = [] deposit_v1 = publish_and_expunge(db, deposit_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) assert record_v1.get('communities', []) == [] assert record_v2.get('communities', []) == []
def test_communities_newversion_while_ir_pending_bug(app, db, users, communities, deposit, deposit_file): """Make sure that pending IRs remain after a new version (bug).""" deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value # Two inclusion requests are pending assert InclusionRequest.query.count() == 2 # Accept one community c1_api = ZenodoCommunity('c1') c1_api.accept_record(record_v1, pid=recid_v1) deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) # Make sure there is still IR to community 'c2' after newversion assert InclusionRequest.query.count() == 1 assert InclusionRequest.query.one().id_community == 'c2' assert record_v1.get('communities', []) == [ 'c1', ] assert deposit_v1.get('communities', []) == [ 'c1', 'c2', ] assert record_v2.get('communities', []) == [ 'c1', ] assert deposit_v2.get('communities', []) == [ 'c1', 'c2', ]
def test_propagation_with_newversion_open( app, db, users, communities, deposit, deposit_file): """Adding old versions to a community should propagate to all drafts.""" # deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) deposit_v1 = deposit_v1.edit() recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value # New version in 'deposit_v2' has not been published yet deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) # depid_v1_value = deposit_v1['_deposit']['id'] # depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1['communities'] = ['c1', 'c2', ] deposit_v1 = publish_and_expunge(db, deposit_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) c1_api = ZenodoCommunity('c1') c1_api.accept_record(record_v1, pid=recid_v1) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) assert deposit_v2['communities'] == ['c1', 'c2'] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() assert record_v2['communities'] == ['c1', ]
def test_propagation_with_newversion_open( app, db, users, communities, deposit, deposit_file): """Adding old versions to a community should propagate to all drafts.""" # deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) deposit_v1 = deposit_v1.edit() recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value # New version in 'deposit_v2' has not been published yet deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) # depid_v1_value = deposit_v1['_deposit']['id'] # depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1['communities'] = ['c1', 'c2', ] deposit_v1 = publish_and_expunge(db, deposit_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) c1_api = ZenodoCommunity('c1') c1_api.accept_record(record_v1, pid=recid_v1) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) assert deposit_v2['communities'] == ['c1', 'c2'] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() assert record_v2['communities'] == ['c1', ]
def test_rat_deposit_files_access(app, db, api_client, deposit, deposit_file, deposit_url, json_auth_headers, license_record, rat_generate_token): """Test deposit files access via RATs.""" client = api_client depid = deposit['_deposit']['id'] deposit['owners'] = [rat_generate_token.user_id] deposit['_deposit']['owners'] = [rat_generate_token.user_id] deposit.commit() db.session.commit() rat_token = jwt.encode( payload={ 'iat': datetime.utcnow(), 'sub': { 'deposit_id': depid, 'access': 'read', }, }, key=rat_generate_token.access_token, algorithm='HS256', headers={'kid': str(rat_generate_token.id)}, ) deposit_url += '/' + str(depid) file_url = '/files/{}/test.txt'.format(deposit['_buckets']['deposit']) publish_url = deposit_url + '/actions/publish' res = client.get(file_url) assert res.status_code == 404 res = client.get(file_url, query_string={'token': rat_token}) assert res.status_code == 200 # Try other forbidden operations using the RAT res = client.get(deposit_url, query_string={'token': rat_token}) assert res.status_code == 401 data = json.dumps(get_data()) res = client.put(deposit_url, data=data, query_string={'token': rat_token}) assert res.status_code == 401 res = client.put(file_url, data=data, query_string={'token': rat_token}) assert res.status_code == 404 res = client.post(publish_url, query_string={'token': rat_token}) assert res.status_code == 401 # Change record owner depid, deposit = deposit_resolver.resolve(depid) deposit['owners'] = [123] deposit['_deposit']['owners'] = [123] deposit.commit() db.session.commit() res = client.get(file_url) assert res.status_code == 404 res = client.get(file_url, query_string={'token': rat_token}) assert res.status_code == 404
def test_communities_newversion_addition(app, db, users, communities, deposit, deposit_file): """Make sure that new version of record synchronizes the communities.""" deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') c1_api.accept_record(record_v1, pid=recid_v1) c2_api.accept_record(record_v1, pid=recid_v1) deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) # Remove 'c2' and request for 'c5'. Make sure that communities from # previous record version are preserved/removed properly deposit_v2['communities'] = ['c1', 'c5'] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) assert record_v1.get('communities', []) == [ 'c1', ] assert deposit_v1.get('communities', []) == [ 'c1', 'c5', ] assert record_v2.get('communities', []) == [ 'c1', ] assert deposit_v2.get('communities', []) == [ 'c1', 'c5', ]
def test_extra_formats_buckets_permissions(api, api_client, minimal_deposit, deposit_url, db, es, users, locations, json_extra_auth_headers, extra_auth_headers, license_record, user_email, status): """Test Files-REST permissions for the extra formats bucket and files.""" # Create deposit response = api_client.post(deposit_url, json=minimal_deposit, headers=json_extra_auth_headers) data = response.json # Get identifier and links depid = data['record_id'] links = data['links'] # Upload 1 files response = api_client.put( links['bucket'] + '/test1.txt', data='ctx', headers=extra_auth_headers, ) # Add extra_formats bucket with a file response = api_client.put( '/deposit/depositions/{0}/formats'.format(depid), data='foo file', headers=[('Content-Type', 'application/foo+xml')] + extra_auth_headers) dep_uuid, deposit = deposit_resolver.resolve(depid) if user_email: # Login as user login_user_via_session(api_client, email=user_email) response = api_client.get('/files/' + str(deposit.extra_formats.bucket.id)) assert response.status_code == status response = api_client.put('/files/' + str(deposit.extra_formats.bucket.id) + '/application/foo+xml', data='ctx') assert response.status_code == status # Publish deposition response = api_client.post(links['publish'], headers=extra_auth_headers) if user_email: # Login as user login_user_via_session(api_client, email=user_email) response = api_client.get('/files/' + str(deposit.extra_formats.bucket.id)) assert response.status_code == status response = api_client.put('/files/' + str(deposit.extra_formats.bucket.id) + '/application/foo+xml', data='ctx') assert response.status_code == status
def test_communities_newversion_while_ir_pending_bug( app, db, users, communities, deposit, deposit_file): """Make sure that pending IRs remain after a new version (bug).""" deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value # Two inclusion requests are pending assert InclusionRequest.query.count() == 2 # Accept one community c1_api = ZenodoCommunity('c1') c1_api.accept_record(record_v1, pid=recid_v1) deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) # Make sure there is still IR to community 'c2' after newversion assert InclusionRequest.query.count() == 1 assert InclusionRequest.query.one().id_community == 'c2' assert record_v1.get('communities', []) == ['c1', ] assert deposit_v1.get('communities', []) == ['c1', 'c2', ] assert record_v2.get('communities', []) == ['c1', ] assert deposit_v2.get('communities', []) == ['c1', 'c2', ]
def test_communities_newversion_addition( app, db, users, communities, deposit, deposit_file): """Make sure that new version of record synchronizes the communities.""" deposit['communities'] = ['c1', 'c2'] deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1_value = deposit_v1['_deposit']['id'] recid_v1_value = recid_v1.pid_value c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') c1_api.accept_record(record_v1, pid=recid_v1) c2_api.accept_record(record_v1, pid=recid_v1) deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) # Remove 'c2' and request for 'c5'. Make sure that communities from # previous record version are preserved/removed properly deposit_v2['communities'] = ['c1', 'c5'] deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) assert record_v1.get('communities', []) == ['c1', ] assert deposit_v1.get('communities', []) == ['c1', 'c5', ] assert record_v2.get('communities', []) == ['c1', ] assert deposit_v2.get('communities', []) == ['c1', 'c5', ]
def test_relations_serialization(app, db, deposit, deposit_file): """Serialize PID relations.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() expected = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 1 } ] } assert serialize_relations(recid_v1) == expected deposit_v1.newversion() # Should contain "draft_child_deposit" information expected = { "version": [ { "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 1, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert serialize_relations(recid_v1) == expected # Publish the new version pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() # Should no longer contain "draft_child_deposit" info after publishing # and no longer be the last child expected = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 } ] } assert serialize_relations(recid_v1) == expected # New version should be the last child now expected = { "version": [ { "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert serialize_relations(recid_v2) == expected
def test_related_identifiers_serialization(app, db, deposit, deposit_file): """Serialize PID Relations to related identifiers.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c2' through deposit v2 deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) rids = serialize_related_identifiers(recid_v1) expected_v1 = [ { 'scheme': 'doi', 'identifier': '10.5072/zenodo.1', 'relation': 'isPartOf' } # TODO: serialization of new version realtions is disabled # { # 'scheme': 'doi', # 'identifier': '10.5072/zenodo.3', # 'relation': 'isPreviousVersionOf' # } ] assert rids == expected_v1 rids = serialize_related_identifiers(recid_v2) expected_v2 = [ { 'scheme': 'doi', 'identifier': '10.5072/zenodo.1', 'relation': 'isPartOf' } # TODO: serialization of new version realtions is disabled # { # 'scheme': 'doi', # 'identifier': '10.5072/zenodo.2', # 'relation': 'isNewVersionOf' # } ] assert rids == expected_v2 parent_pid = PersistentIdentifier.get('recid', '1') rids = serialize_related_identifiers(parent_pid) expected_parent = [ { 'relation': 'hasPart', 'scheme': 'doi', 'identifier': '10.5072/zenodo.2' }, { 'relation': 'hasPart', 'scheme': 'doi', 'identifier': '10.5072/zenodo.3' } ] assert rids == expected_parent
def delete_record(record_uuid, reason, user): """Delete the record and it's PIDs. :param record_uuid: UUID of the record to be removed. :param reason: Reason for removal. Either one of: 'spam', 'uploader', 'takedown' (see 'ZENODO_REMOVAL_REASONS' variable in config), otherwise using it as a verbatim "Reason" string. :param user: ID or email of the Zenodo user (admin) responsible for the removal. """ from invenio_github.models import ReleaseStatus if isinstance(user, text_type): user_id = User.query.filter_by(email=user).one().id elif isinstance(user, int): user_id = User.query.get(user).id else: raise TypeError( "User cannot be determined from argument: {0}".format(user)) record = ZenodoRecord.get_record(record_uuid) # Remove the record from versioning and delete the recid recid = PersistentIdentifier.get('recid', record['recid']) pv = PIDVersioning(child=recid) pv.remove_child(recid) pv.update_redirect() recid.delete() # Remove the record from index try: RecordIndexer().delete(record) except NotFoundError: pass # Remove buckets record_bucket = record.files.bucket RecordsBuckets.query.filter_by(record_id=record.id).delete() record_bucket.locked = False record_bucket.remove() removal_reasons = dict(current_app.config['ZENODO_REMOVAL_REASONS']) if reason in removal_reasons: reason = removal_reasons[reason] depid, deposit = deposit_resolver.resolve(record['_deposit']['id']) try: doi = PersistentIdentifier.get('doi', record['doi']) except PIDDoesNotExistError: doi = None # Record OpenAIRE info try: original_id = openaire_original_id(record, openaire_type(record))[1] datasource_id = openaire_datasource_id(record) except PIDDoesNotExistError: original_id = None datasource_id = None if pv.children.count() == 0: conceptrecid = PersistentIdentifier.get('recid', record['conceptrecid']) conceptrecid.delete() new_last_child = None else: new_last_child = (pv.last_child.pid_value, str(pv.last_child.object_uuid)) if 'conceptdoi' in record: conceptdoi_value = record['conceptdoi'] else: conceptdoi_value = None # Completely delete the deposit # Deposit will be removed from index deposit.delete(delete_published=True) # Clear the record and put the deletion information record.clear() record.update({ 'removal_reason': reason, 'removed_by': user_id, }) record.commit() # Mark the relevant GitHub Release as deleted for ghr in record.model.github_releases: ghr.status = ReleaseStatus.DELETED if not is_local_doi(doi.pid_value): db.session.delete(doi) db.session.commit() # After successful DB commit, sync the DOIs with DataCite if is_local_doi(doi.pid_value): datacite_inactivate.delay(doi.pid_value) if conceptdoi_value: if new_last_child: # Update last child (updates also conceptdoi) pid_value, rec_uuid = new_last_child datacite_register.delay(pid_value, rec_uuid) else: datacite_inactivate.delay(conceptdoi_value) # Also delete from OpenAIRE index if current_app.config['OPENAIRE_DIRECT_INDEXING_ENABLED'] and original_id \ and datasource_id: openaire_delete.delay(original_id=original_id, datasource_id=datasource_id)
def test_extra_formats_buckets_permissions( api, api_client, minimal_deposit, deposit_url, db, es, users, locations, json_extra_auth_headers, extra_auth_headers, license_record, user_email, status ): """Test Files-REST permissions for the extra formats bucket and files.""" # Create deposit response = api_client.post( deposit_url, json=minimal_deposit, headers=json_extra_auth_headers) data = response.json # Get identifier and links depid = data['record_id'] links = data['links'] # Upload 1 files response = api_client.put( links['bucket'] + '/test1.txt', data='ctx', headers=extra_auth_headers, ) # Add extra_formats bucket with a file response = api_client.put( '/deposit/depositions/{0}/formats'.format(depid), data='foo file', headers=[('Content-Type', 'application/foo+xml')] + extra_auth_headers ) dep_uuid, deposit = deposit_resolver.resolve(depid) if user_email: # Login as user login_user_via_session(api_client, email=user_email) response = api_client.get( '/files/' + str(deposit.extra_formats.bucket.id) ) assert response.status_code == status response = api_client.put( '/files/' + str(deposit.extra_formats.bucket.id) + '/application/foo+xml', data='ctx' ) assert response.status_code == status # Publish deposition response = api_client.post(links['publish'], headers=extra_auth_headers) if user_email: # Login as user login_user_via_session(api_client, email=user_email) response = api_client.get( '/files/' + str(deposit.extra_formats.bucket.id) ) assert response.status_code == status response = api_client.put( '/files/' + str(deposit.extra_formats.bucket.id) + '/application/foo+xml', data='ctx' ) assert response.status_code == status
def versioning_link_records(recids): """Link several non-versioned records into one versioning scheme. The records are linked in the order as they appear in the list, with the first record being base for minting of the conceptdoi. In case one of the records is already upgraded, its taken as the base for conceptdoi instead, with preserving the requested order. :param recids: list of recid values (strings) to link, e.g.: ['1234','55125','51269'] :type recids: list of str """ recids_records = [ record_resolver.resolve(recid_val) for recid_val in recids ] depids_deposits = [ deposit_resolver.resolve(record['_deposit']['id']) for _, record in recids_records ] rec_comms = sorted( set(sum([rec.get('communities', []) for _, rec in recids_records], []))) dep_comms = sorted( set(sum([dep.get('communities', []) for _, dep in depids_deposits], []))) upgraded = [(recid, rec) for recid, rec in recids_records if 'conceptdoi' in rec] # Determine the base record for versioning if len(upgraded) == 0: recid_v, record_v = recids_records[0] elif len(upgraded) == 1: recid_v, record_v = upgraded[0] elif len(upgraded) > 1: recid_v, record_v = upgraded[0] child_recids = [ int(recid.pid_value) for recid in PIDVersioning(child=recid_v).children.all() ] i_upgraded = [int(recid.pid_value) for recid, rec in upgraded] if set(child_recids) != set(i_upgraded): raise Exception('Multiple upgraded records, which belong' 'to different versioning schemes.') # Get the first record and mint the concept DOI for it conceptdoi = zenodo_concept_doi_minter(record_v.id, record_v) conceptrecid_v = PersistentIdentifier.get('recid', record_v['conceptrecid']) conceptrecid_v_val = conceptrecid_v.pid_value pv_r1 = PIDVersioning(parent=conceptrecid_v) children_recids = [c.pid_value for c in pv_r1.children.all()] if not all(cr in recids for cr in children_recids): raise Exception('Children of the already upgraded record: {0} are ' 'not specified in the ordering: {1}' ''.format(children_recids, recids)) for (recid, record), (depid, deposit) in \ zip(recids_records, depids_deposits): # Remove old versioning schemes for non-base recids # Note: This will remove the child of the base-conceptrecid as well # but that's OK, since it will be added again afterwards in the # correct order. conceptrecid = PersistentIdentifier.get('recid', record['conceptrecid']) pv = PIDVersioning(parent=conceptrecid) pv.remove_child(recid) if conceptrecid.pid_value != conceptrecid_v_val: conceptrecid.delete() # Update the 'conceptrecid' and 'conceptdoi' in records and deposits record['conceptdoi'] = conceptdoi.pid_value record['conceptrecid'] = conceptrecid_v.pid_value record['communities'] = rec_comms record.commit() deposit['conceptdoi'] = conceptdoi.pid_value deposit['conceptrecid'] = conceptrecid_v.pid_value deposit['communities'] = dep_comms deposit.commit() # Add the child to the new versioning scheme pv_r1.insert_child(recid) pv_r1.update_redirect() db.session.commit() conceptrecid_v = PersistentIdentifier.get('recid', conceptrecid_v_val) pv = PIDVersioning(parent=conceptrecid_v) if current_app.config['DEPOSIT_DATACITE_MINTING_ENABLED']: datacite_register.delay(pv.last_child.pid_value, str(pv.last_child.object_uuid)) index_siblings(pv.last_child, with_deposits=True, eager=True)
def test_autoadd(app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Test basic workflow using Deposit and Communities API.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c3' (owned by user) through deposit v2 deposit_v2['communities'] = [ 'c1', 'c2', 'c3', ] deposit_v2['grants'] = [ { 'title': 'SomeGrant' }, ] deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() assert record_v2['grants'] == [ { 'title': 'SomeGrant' }, ] recid_v2_value = recid_v2.pid_value depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() assert record_v1.get('communities', []) == ['c3', 'grants_comm'] assert record_v2.get('communities', []) == ['c3', 'grants_comm'] assert deposit_v1.get('communities', []) == [ 'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo' ] assert deposit_v2.get('communities', []) == [ 'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo' ] c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') c3_api = ZenodoCommunity('c3') grants_comm_api = ZenodoCommunity('grants_comm') ecfunded_api = ZenodoCommunity('ecfunded') zenodo_api = ZenodoCommunity('zenodo') # Inclusion requests should be visible for both records assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c3_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0 assert c3_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0 assert grants_comm_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0 assert grants_comm_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0 assert ecfunded_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert ecfunded_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert zenodo_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert zenodo_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 # Accept to 'c1' through record_v2 (as originally requested), # and 'c2' through record_v1 (resolved through version) c1_api.accept_record(record_v2, pid=recid_v2) c2_api.accept_record(record_v1, pid=recid_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) # Accepting individual record to a community should propagate the changes # to all versions assert record_v1.get('communities', []) == ['c1', 'c2', 'c3', 'grants_comm'] assert record_v2.get('communities', []) == ['c1', 'c2', 'c3', 'grants_comm'] assert deposit_v1.get('communities', []) == [ 'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo' ] assert deposit_v2.get('communities', []) == [ 'c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo' ] # Removing 'c1'-'c3' from deposit_v1 should remove it from two published # records and other deposits as well depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1 = deposit_v1.edit() deposit_v1['communities'] = [] deposit_v1 = publish_and_expunge(db, deposit_v1) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) assert record_v1.get('communities', []) == [ 'grants_comm', ] assert record_v2.get('communities', []) == [ 'grants_comm', ] assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo'] assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo']
def test_autoadd(app, db, users, communities, deposit, deposit_file, communities_autoadd_enabled): """Test basic workflow using Deposit and Communities API.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1 = deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit depid_v2_value = depid_v2.pid_value deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c3' (owned by user) through deposit v2 deposit_v2['communities'] = ['c1', 'c2', 'c3', ] deposit_v2['grants'] = [{'title': 'SomeGrant'}, ] deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() assert record_v2['grants'] == [{'title': 'SomeGrant'}, ] recid_v2_value = recid_v2.pid_value depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() assert record_v1.get('communities', []) == ['c3', 'grants_comm'] assert record_v2.get('communities', []) == ['c3', 'grants_comm'] assert deposit_v1.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'] assert deposit_v2.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'] c1_api = ZenodoCommunity('c1') c2_api = ZenodoCommunity('c2') c3_api = ZenodoCommunity('c3') grants_comm_api = ZenodoCommunity('grants_comm') ecfunded_api = ZenodoCommunity('ecfunded') zenodo_api = ZenodoCommunity('zenodo') # Inclusion requests should be visible for both records assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 assert c3_api.get_comm_irs(record_v1, pid=recid_v1).count() == 0 assert c3_api.get_comm_irs(record_v2, pid=recid_v2).count() == 0 assert grants_comm_api.get_comm_irs( record_v1, pid=recid_v1).count() == 0 assert grants_comm_api.get_comm_irs( record_v2, pid=recid_v2).count() == 0 assert ecfunded_api.get_comm_irs( record_v1, pid=recid_v1).count() == 1 assert ecfunded_api.get_comm_irs( record_v2, pid=recid_v2).count() == 1 assert zenodo_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1 assert zenodo_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1 # Accept to 'c1' through record_v2 (as originally requested), # and 'c2' through record_v1 (resolved through version) c1_api.accept_record(record_v2, pid=recid_v2) c2_api.accept_record(record_v1, pid=recid_v1) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) # Accepting individual record to a community should propagate the changes # to all versions assert record_v1.get('communities', []) == ['c1', 'c2', 'c3', 'grants_comm'] assert record_v2.get('communities', []) == ['c1', 'c2', 'c3', 'grants_comm'] assert deposit_v1.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'] assert deposit_v2.get('communities', []) == ['c1', 'c2', 'c3', 'ecfunded', 'grants_comm', 'zenodo'] # Removing 'c1'-'c3' from deposit_v1 should remove it from two published # records and other deposits as well depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) deposit_v1 = deposit_v1.edit() deposit_v1['communities'] = [] deposit_v1 = publish_and_expunge(db, deposit_v1) depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value) recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) recid_v2, record_v2 = record_resolver.resolve(recid_v2_value) assert record_v1.get('communities', []) == ['grants_comm', ] assert record_v2.get('communities', []) == ['grants_comm', ] assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo'] assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm', 'zenodo']
def test_relations_serialization(app, db, deposit, deposit_file): """Serialize PID relations.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() expected = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 1 }] } assert serialize_relations(recid_v1) == expected deposit_v1.newversion() # Should contain "draft_child_deposit" information expected = { "version": [{ "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 1, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert serialize_relations(recid_v1) == expected # Publish the new version pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() # Should no longer contain "draft_child_deposit" info after publishing # and no longer be the last child expected = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 }] } assert serialize_relations(recid_v1) == expected # New version should be the last child now expected = { "version": [{ "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert serialize_relations(recid_v2) == expected
def test_versioning_indexing(db, es, deposit, deposit_file): """Test the indexing of 'version' relations.""" deposit_index_name = 'deposits-records-record-v1.0.0' records_index_name = 'records-record-v1.0.0' deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value RecordIndexer().index_by_id(str(record_v1.id)) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 1 assert len(s_rec) == 1 assert 'relations' in s_dep[0]['_source'] assert 'relations' in s_rec[0]['_source'] expected = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 1, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert s_dep[0]['_source']['relations'] == expected assert s_rec[0]['_source']['relations'] == expected deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid) deposit_v2.files['file.txt'] = BytesIO(b('file1')) depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 2 # Two deposits should be indexed assert len(s_rec) == 1 # One, since record does not exist yet s_dep1 = current_search.client.get(index=deposit_index_name, id=deposit_v1.id) s_dep2 = current_search.client.get(index=deposit_index_name, id=deposit_v2.id) expected_d1 = { "version": [{ "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "2" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 # For deposit, draft children are also counted }] } expected_d2 = { "version": [{ "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 2, # For deposit, draft children are also counted "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert s_dep1['_source']['relations'] == expected_d1 assert s_dep2['_source']['relations'] == expected_d2 deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) RecordIndexer().index_by_id(str(record_v2.id)) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 2 assert len(s_rec) == 2 s_dep1 = current_search.client.get(index=deposit_index_name, id=deposit_v1.id) s_dep2 = current_search.client.get(index=deposit_index_name, id=deposit_v2.id) s_rec1 = current_search.client.get(index=records_index_name, id=record_v1.id) s_rec2 = current_search.client.get(index=records_index_name, id=record_v2.id) expected_d1 = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 }] } expected_d2 = { "version": [{ "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert s_dep1['_source']['relations'] == expected_d1 assert s_dep2['_source']['relations'] == expected_d2 expected_r1 = { "version": [{ "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 }] } expected_r2 = { "version": [{ "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, }] } assert s_rec1['_source']['relations'] == expected_r1 assert s_rec2['_source']['relations'] == expected_r2
def delete_record(record_uuid, reason, user): """Delete the record and it's PIDs. :param record_uuid: UUID of the record to be removed. :param reason: Reason for removal. Either one of: 'spam', 'uploader', 'takedown' (see 'ZENODO_REMOVAL_REASONS' variable in config), otherwise using it as a verbatim "Reason" string. :param user: ID or email of the Zenodo user (admin) responsible for the removal. """ from invenio_github.models import ReleaseStatus if isinstance(user, text_type): user_id = User.query.filter_by(email=user).one().id elif isinstance(user, int): user_id = User.query.get(user).id else: raise TypeError("User cannot be determined from argument: {0}".format( user)) record = ZenodoRecord.get_record(record_uuid) # Remove the record from versioning and delete the recid recid = PersistentIdentifier.get('recid', record['recid']) pv = PIDVersioning(child=recid) pv.remove_child(recid) pv.update_redirect() recid.delete() # Remove the record from index try: RecordIndexer().delete(record) except NotFoundError: pass # Remove buckets record_bucket = record.files.bucket RecordsBuckets.query.filter_by(record_id=record.id).delete() record_bucket.locked = False record_bucket.remove() removal_reasons = dict(current_app.config['ZENODO_REMOVAL_REASONS']) if reason in removal_reasons: reason = removal_reasons[reason] depid, deposit = deposit_resolver.resolve(record['_deposit']['id']) try: doi = PersistentIdentifier.get('doi', record['doi']) except PIDDoesNotExistError: doi = None # Record OpenAIRE info try: original_id = openaire_original_id(record, openaire_type(record))[1] datasource_id = openaire_datasource_id(record) except PIDDoesNotExistError: original_id = None datasource_id = None if pv.children.count() == 0: conceptrecid = PersistentIdentifier.get('recid', record['conceptrecid']) conceptrecid.delete() new_last_child = None else: new_last_child = (pv.last_child.pid_value, str(pv.last_child.object_uuid)) if 'conceptdoi' in record: conceptdoi_value = record['conceptdoi'] else: conceptdoi_value = None # Completely delete the deposit # Deposit will be removed from index deposit.delete(delete_published=True) # Clear the record and put the deletion information record.clear() record.update({ 'removal_reason': reason, 'removed_by': user_id, }) record.commit() # Mark the relevant GitHub Release as deleted for ghr in record.model.github_releases: ghr.status = ReleaseStatus.DELETED db.session.commit() # After successful DB commit, sync the DOIs with DataCite datacite_inactivate.delay(doi.pid_value) if conceptdoi_value: if new_last_child: # Update last child (updates also conceptdoi) pid_value, rec_uuid = new_last_child datacite_register.delay(pid_value, rec_uuid) else: datacite_inactivate.delay(conceptdoi_value) # Also delete from OpenAIRE index if current_app.config['OPENAIRE_DIRECT_INDEXING_ENABLED'] and original_id \ and datasource_id: openaire_delete.delay(original_id=original_id, datasource_id=datasource_id)
def test_extra_formats_buckets( api, api_client, db, es, locations, json_extra_auth_headers, deposit_url, get_json, extra_auth_headers, json_headers, license_record, communities, resolver, minimal_deposit): """Test simple flow using REST API.""" headers = json_extra_auth_headers client = api_client test_data = minimal_deposit # Create deposit response = client.post( deposit_url, json=test_data, headers=headers) data = get_json(response, code=201) # Get identifier and links depid = data['record_id'] links = data['links'] # Upload 1 files response = client.put( links['bucket'] + '/test1.txt', data='ctx', headers=extra_auth_headers, ) assert response.status_code == 200 # Check for extra_formats bucket response = api_client.options( extra_formats_urls['deposit'].format(depid), headers=headers) data = get_json(response, code=200) # Check that no extra_formats bucket is present buckets = Bucket.query.all() assert len(buckets) == 1 # There are no extra_formats files assert data == [] use_extra_formats_functions( extra_auth_headers, api_client, get_json, depid=depid) buckets = Bucket.query.all() assert len(buckets) == 2 deposit = deposit_resolver.resolve(depid)[1] assert deposit['_buckets']['extra_formats'] == \ str(deposit.extra_formats.bucket.id) # Publish deposition response = client.post(links['publish'], headers=extra_auth_headers) data = get_json(response, code=202) first_version_recid = data['record_id'] # Get the list of the extra_formats files attached to this deposit response = api_client.options( extra_formats_urls['record'].format(first_version_recid)) data = get_json(response, code=200) assert data[0]['key'] == 'application/foo+xml' assert len(data) == 1 # Test actions and clear extra_formats bucket use_extra_formats_functions(extra_auth_headers, api_client, get_json, depid=depid, recid=first_version_recid) # Get newversion url data = get_json( client.get(links['self'], headers=extra_auth_headers), code=200 ) new_version_url = data['links']['newversion'] # New Version data = get_json( client.post(new_version_url, headers=extra_auth_headers), code=201) links = data['links'] # Get the list of the extra_formats files attached to the new deposit # Should be the same with the previous version response = api_client.options( extra_formats_urls['deposit'].format(depid), headers=extra_auth_headers ) data = get_json(response, code=200) assert data[0]['key'] == 'application/foo+xml' assert len(data) == 1 # Get latest version data = get_json( client.get(links['latest_draft'], headers=extra_auth_headers), code=200) links = data['links'] depid = data['record_id'] # Add a file to the new deposit get_json(client.put( links['bucket'] + '/newfile.txt', data='newfile', headers=extra_auth_headers, ), code=200) # Publish the new record response = client.post(links['publish'], headers=extra_auth_headers) data = get_json(response, code=202) links = data['links'] recid = data['record_id'] # Get the list of the extra_formats files attached to the new record response = api_client.options(extra_formats_urls['record'].format(recid)) data = get_json(response, code=200) assert data[0]['key'] == 'application/foo+xml' assert len(data) == 1 # Add file to extra_formats bucket response = api_client.put( extra_formats_urls['deposit'].format(recid), data='bar file content', headers=extra_formats_headers['bar'] + extra_auth_headers ) data = get_json(response, code=200) assert data['message'] == 'Extra format "application/bar+xml" updated.' # Get the list of the extra_formats files attached to the new record response = api_client.options(extra_formats_urls['record'].format(recid)) data = get_json(response, code=200) assert {f['key'] for f in data} == \ {'application/foo+xml', 'application/bar+xml'} # Get the list of the extra_formats files attached to the previous record # Make sure that the snapshots are independent response = api_client.options( extra_formats_urls['record'].format(first_version_recid)) data = get_json(response, code=200) first_record = record_resolver.resolve(first_version_recid)[1] new_record = record_resolver.resolve(recid)[1] assert first_record.extra_formats.bucket.id != \ new_record.extra_formats.bucket.id assert data[0]['key'] == 'application/foo+xml' assert len(data) == 1 # Test actions and clear extra_formats bucket of deposit use_extra_formats_functions( extra_auth_headers, api_client, get_json, depid=depid, recid=recid)
def test_extra_formats_buckets(api, api_client, db, es, locations, json_extra_auth_headers, deposit_url, get_json, extra_auth_headers, json_headers, license_record, communities, resolver, minimal_deposit): """Test simple flow using REST API.""" headers = json_extra_auth_headers client = api_client test_data = minimal_deposit # Create deposit response = client.post(deposit_url, json=test_data, headers=headers) data = get_json(response, code=201) # Get identifier and links depid = data['record_id'] links = data['links'] # Upload 1 files response = client.put( links['bucket'] + '/test1.txt', data='ctx', headers=extra_auth_headers, ) assert response.status_code == 200 # Check for extra_formats bucket response = api_client.options(extra_formats_urls['deposit'].format(depid), headers=headers) data = get_json(response, code=200) # Check that no extra_formats bucket is present buckets = Bucket.query.all() assert len(buckets) == 1 # There are no extra_formats files assert data == [] use_extra_formats_functions(extra_auth_headers, api_client, get_json, depid=depid) buckets = Bucket.query.all() assert len(buckets) == 2 deposit = deposit_resolver.resolve(depid)[1] assert deposit['_buckets']['extra_formats'] == \ str(deposit.extra_formats.bucket.id) # Publish deposition response = client.post(links['publish'], headers=extra_auth_headers) data = get_json(response, code=202) first_version_recid = data['record_id'] # Get the list of the extra_formats files attached to this deposit response = api_client.options( extra_formats_urls['record'].format(first_version_recid)) data = get_json(response, code=200) assert data[0]['key'] == 'application/foo+xml' assert len(data) == 1 # Test actions and clear extra_formats bucket use_extra_formats_functions(extra_auth_headers, api_client, get_json, depid=depid, recid=first_version_recid) # Get newversion url data = get_json(client.get(links['self'], headers=extra_auth_headers), code=200) new_version_url = data['links']['newversion'] # New Version data = get_json(client.post(new_version_url, headers=extra_auth_headers), code=201) links = data['links'] # Get the list of the extra_formats files attached to the new deposit # Should be the same with the previous version response = api_client.options(extra_formats_urls['deposit'].format(depid), headers=extra_auth_headers) data = get_json(response, code=200) assert data[0]['key'] == 'application/foo+xml' assert len(data) == 1 # Get latest version data = get_json(client.get(links['latest_draft'], headers=extra_auth_headers), code=200) links = data['links'] depid = data['record_id'] # Add a file to the new deposit get_json(client.put( links['bucket'] + '/newfile.txt', data='newfile', headers=extra_auth_headers, ), code=200) # Publish the new record response = client.post(links['publish'], headers=extra_auth_headers) data = get_json(response, code=202) links = data['links'] recid = data['record_id'] # Get the list of the extra_formats files attached to the new record response = api_client.options(extra_formats_urls['record'].format(recid)) data = get_json(response, code=200) assert data[0]['key'] == 'application/foo+xml' assert len(data) == 1 # Add file to extra_formats bucket response = api_client.put(extra_formats_urls['deposit'].format(recid), data='bar file content', headers=extra_formats_headers['bar'] + extra_auth_headers) data = get_json(response, code=200) assert data['message'] == 'Extra format "application/bar+xml" updated.' # Get the list of the extra_formats files attached to the new record response = api_client.options(extra_formats_urls['record'].format(recid)) data = get_json(response, code=200) assert {f['key'] for f in data} == \ {'application/foo+xml', 'application/bar+xml'} # Get the list of the extra_formats files attached to the previous record # Make sure that the snapshots are independent response = api_client.options( extra_formats_urls['record'].format(first_version_recid)) data = get_json(response, code=200) first_record = record_resolver.resolve(first_version_recid)[1] new_record = record_resolver.resolve(recid)[1] assert first_record.extra_formats.bucket.id != \ new_record.extra_formats.bucket.id assert data[0]['key'] == 'application/foo+xml' assert len(data) == 1 # Test actions and clear extra_formats bucket of deposit use_extra_formats_functions(extra_auth_headers, api_client, get_json, depid=depid, recid=recid)
def test_versioning_indexing(db, es, deposit, deposit_file): """Test the indexing of 'version' relations.""" deposit_index_name = 'deposits-records-record-v1.0.0' records_index_name = 'records-record-v1.0.0' deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() recid_v1_value = recid_v1.pid_value RecordIndexer().index_by_id(str(record_v1.id)) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 1 assert len(s_rec) == 1 assert 'relations' in s_dep[0]['_source'] assert 'relations' in s_rec[0]['_source'] expected = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 1, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert s_dep[0]['_source']['relations'] == expected assert s_rec[0]['_source']['relations'] == expected deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.object_uuid) depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 2 # Two deposits should be indexed assert len(s_rec) == 1 # One, since record does not exist yet s_dep1 = current_search.client.get( index=deposit_index_name, id=deposit_v1.id) s_dep2 = current_search.client.get( index=deposit_index_name, id=deposit_v2.id) expected_d1 = { "version": [ { "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "2" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 # For deposit, draft children are also counted } ] } expected_d2 = { "version": [ { "draft_child_deposit": { "pid_type": "depid", "pid_value": "3" }, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "2" }, "count": 2, # For deposit, draft children are also counted "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert s_dep1['_source']['relations'] == expected_d1 assert s_dep2['_source']['relations'] == expected_d2 deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) RecordIndexer().index_by_id(str(record_v2.id)) RecordIndexer().process_bulk_queue() current_search.flush_and_refresh(index=deposit_index_name) current_search.flush_and_refresh(index=records_index_name) s_dep = current_search.client.search( index=deposit_index_name)['hits']['hits'] s_rec = current_search.client.search( index=records_index_name)['hits']['hits'] assert len(s_dep) == 2 assert len(s_rec) == 2 s_dep1 = current_search.client.get( index=deposit_index_name, id=deposit_v1.id) s_dep2 = current_search.client.get( index=deposit_index_name, id=deposit_v2.id) s_rec1 = current_search.client.get( index=records_index_name, id=record_v1.id) s_rec2 = current_search.client.get( index=records_index_name, id=record_v2.id) expected_d1 = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 } ] } expected_d2 = { "version": [ { "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert s_dep1['_source']['relations'] == expected_d1 assert s_dep2['_source']['relations'] == expected_d2 expected_r1 = { "version": [ { "draft_child_deposit": None, "index": 0, "is_last": False, "last_child": { "pid_type": "recid", "pid_value": "3" }, "parent": { "pid_type": "recid", "pid_value": "1" }, "count": 2 } ] } expected_r2 = { "version": [ { "draft_child_deposit": None, "index": 1, "is_last": True, "last_child": { "pid_type": "recid", "pid_value": "3" }, "count": 2, "parent": { "pid_type": "recid", "pid_value": "1" }, } ] } assert s_rec1['_source']['relations'] == expected_r1 assert s_rec2['_source']['relations'] == expected_r2
def test_related_identifiers_serialization(app, db, deposit, deposit_file): """Serialize PID Relations to related identifiers.""" deposit_v1 = publish_and_expunge(db, deposit) depid_v1_value = deposit_v1['_deposit']['id'] recid_v1, record_v1 = deposit_v1.fetch_published() deposit_v1.newversion() pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) deposit_v2 = deposit_v2.edit() # 1. Request for 'c1' and 'c2' through deposit v2 deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) recid_v1, record_v1 = deposit_v1.fetch_published() depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value) rids = serialize_related_identifiers(recid_v1) expected_v1 = [ { 'scheme': 'doi', 'identifier': '10.5072/zenodo.1', 'relation': 'isPartOf' } # TODO: serialization of new version realtions is disabled # { # 'scheme': 'doi', # 'identifier': '10.5072/zenodo.3', # 'relation': 'isPreviousVersionOf' # } ] assert rids == expected_v1 rids = serialize_related_identifiers(recid_v2) expected_v2 = [ { 'scheme': 'doi', 'identifier': '10.5072/zenodo.1', 'relation': 'isPartOf' } # TODO: serialization of new version realtions is disabled # { # 'scheme': 'doi', # 'identifier': '10.5072/zenodo.2', # 'relation': 'isNewVersionOf' # } ] assert rids == expected_v2 parent_pid = PersistentIdentifier.get('recid', '1') rids = serialize_related_identifiers(parent_pid) expected_parent = [{ 'relation': 'hasPart', 'scheme': 'doi', 'identifier': '10.5072/zenodo.2' }, { 'relation': 'hasPart', 'scheme': 'doi', 'identifier': '10.5072/zenodo.3' }] assert rids == expected_parent