def test_create_file_no_required_index(create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, require_index_exists_on): """ With REQUIRE_FILE_INDEX_EXISTS = True. Test submitting a data file that does not exist in indexd (should raise an error and should not create an index or an alias). """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) # no record in indexd for this file get_index_uuid.return_value = None get_index_hash.return_value = None # creating raises an error resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) assert resp.status_code == 400 # no index or alias creation assert not create_index.called assert not create_alias.called # response assert_negative_response(resp) entity = assert_single_entity_from_response(resp) assert entity['action'] == 'create' path = '/v0/submission/CGCI/BLGSP/export/?format=json&ids={nid}'.format( nid=entity['id']) r = client.get(path, headers=submitter) data = r.json assert len(data) == 1
def test_create_file_no_required_index( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, require_index_exists_on, ): """ With REQUIRE_FILE_INDEX_EXISTS = True. Test submitting a data file that does not exist in indexd (should raise an error and should not create an index or an alias). """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) # no record in indexd for this file get_index_uuid.return_value = None get_index_hash.return_value = None # creating raises an error resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) assert resp.status_code == 400 # no index or alias creation assert not create_index.called assert not create_alias.called # response assert_negative_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "create"
def test_data_file_update_url_different_file_not_indexed( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp): """ Test submitting the different data (with NO id provided) and updating the URL field. HOWEVER the file hash and size in the new data do NOT match the previously submitted data. The file hash/size provided does NOT match an already indexed file. e.g. The file is not yet indexed. The asssumption is that the user is attempting to UPDATE the index with a new file but didn't provide a full id, just the same submitter_id as before. Without an ID provided, sheepdog falls back on secondary keys (being the submitter_id/project). There is already a match for that, BUT the provided file hash/size is different than the previously submitted one. """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = DEFAULT_UUID document.urls = [DEFAULT_URL] get_index_uuid.return_value = document # index yeilds no match given hash/size get_index_hash.return_value = None resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) entity = assert_single_entity_from_response(resp) # now submit again but change url new_url = 'some/new/url/location/to/add' updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) updated_file['urls'] = new_url updated_file['object_id'] = DEFAULT_UUID updated_file['md5sum'] = DEFAULT_FILE_HASH.replace('0', '2') updated_file['file_size'] = DEFAULT_FILE_SIZE + 1 resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=updated_file) # no index or alias creation assert not create_index.called assert not create_alias.called # make sure original url is still there and new url is NOT assert DEFAULT_URL in document.urls assert new_url not in document.urls # response assert_negative_response(resp) assert_single_entity_from_response(resp)
def test_data_file_already_indexed_id_provided( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, monkeypatch, ): """ Test submitting when the file is already indexed in the index client and an id is provided in the submission. NOTE: signpostclient will NOT create add a file to the index service if an ID is provided """ monkeypatch.setitem(flask.current_app.config, "USE_SIGNPOST", True) submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = "14fd1746-61bb-401a-96d2-342cfaf70000" get_index_uuid.return_value = document # signpostclient cannot find by hash/size get_index_hash.return_value = None # signpost create will return a document create_index.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid file = copy.deepcopy(DEFAULT_METADATA_FILE) file["id"] = document.did resp = submit_metadata_file( client, pg_driver, admin, submitter, cgci_blgsp, data=file ) # no index or alias creation assert not create_index.called assert not create_alias.called # no support for aliases assert not create_alias.called # response assert_negative_response(resp) assert_single_entity_from_response(resp)
def test_data_file_update_url_id_provided_different_file_already_indexed( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp): """ Test submitting the same data again (with the id provided) and updating the URL field (should get added to the indexed file in index service). HOWEVER the file hash and size in the new data MATCH A DIFFERENT FILE in the index service that does NOT have the id provided. The asssumption is that the user is attempting to UPDATE the index with a new file they've already submitted under a different id. FIXME At the moment, we do not allow updating like this """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document_with_id = MagicMock() document_with_id.did = DEFAULT_UUID document_with_id.urls = [DEFAULT_URL] different_file_matching_hash_and_size = MagicMock() different_file_matching_hash_and_size.did = '14fd1746-61bb-401a-96d2-342cfaf70000' different_file_matching_hash_and_size.urls = [DEFAULT_URL] get_index_uuid.return_value = document_with_id get_index_hash.return_value = different_file_matching_hash_and_size submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # now submit again but change url new_url = 'some/new/url/location/to/add' updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) updated_file['urls'] = new_url updated_file['id'] = DEFAULT_UUID updated_file['md5sum'] = DEFAULT_FILE_HASH.replace('0', '2') updated_file['file_size'] = DEFAULT_FILE_SIZE + 1 resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=updated_file) # no index or alias creation assert not create_index.called assert not create_alias.called # make sure original url is still there and new url is NOT assert DEFAULT_URL in document_with_id.urls assert DEFAULT_URL in different_file_matching_hash_and_size.urls assert new_url not in document_with_id.urls assert new_url not in different_file_matching_hash_and_size.urls # response assert_negative_response(resp) assert_single_entity_from_response(resp)
def test_data_file_update_url_invalid_id( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, ): """ Test submitting the same data again (with the WRONG id provided). i.e. ID provided doesn't match the id from the index service for the file found with the hash/size provided FIXME: the 1:1 between node id and index/file id is temporary so this test may need to be modified in the future """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = "14fd1746-61bb-401a-96d2-342cfaf70000" document.urls = [DEFAULT_URL] get_index_hash.return_value = document # the uuid provided doesn't have a matching indexed file get_index_uuid.return_value = None submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # now submit again but change url new_url = "some/new/url/location/to/add" updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) updated_file["urls"] = new_url updated_file["id"] = DEFAULT_UUID resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=updated_file) # no index or alias creation assert not create_index.called assert not create_alias.called # make sure original url is still there and new url is NOT assert DEFAULT_URL in document.urls assert new_url not in document.urls # response assert_negative_response(resp) assert_single_entity_from_response(resp)
def test_data_file_update_url_id_provided_different_file_not_indexed( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp): """ Test submitting the same data again (with the id provided) and updating the URL field. HOWEVER the file hash and size in the new data do NOT match the previously submitted data for the given ID. The file hash/size provided does NOT match an already indexed file. e.g. The file is not yet indexed. The asssumption is that the user is attempting to UPDATE the index with a new file. FIXME At the moment, we do not allow updating like this """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = DEFAULT_UUID document.urls = [DEFAULT_URL] get_index_uuid.return_value = document # index yeilds no match given hash/size get_index_hash.return_value = None submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # now submit again but change url new_url = 'some/new/url/location/to/add' updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) updated_file['urls'] = new_url updated_file['id'] = DEFAULT_UUID updated_file['md5sum'] = DEFAULT_FILE_HASH.replace('0', '2') updated_file['file_size'] = DEFAULT_FILE_SIZE + 1 resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=updated_file) # no index or alias creation assert not create_index.called assert not create_alias.called # make sure original url is still there and new url is NOT assert DEFAULT_URL in document.urls assert new_url not in document.urls # response assert_negative_response(resp) assert_single_entity_from_response(resp)
def test_data_file_not_indexed_id_provided( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, monkeypatch, ): """ Test node and data file creation when neither exist and an ID is provided. NOTE: signpostclient will NOT create add a file to the index service if an ID is provided """ monkeypatch.setitem(flask.current_app.config, "USE_SIGNPOST", True) submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) get_index_uuid.return_value = None get_index_hash.return_value = None # signpost create will return a document document = MagicMock() document.did = "14fd1746-61bb-401a-96d2-342cfaf70000" create_index.return_value = document file = copy.deepcopy(DEFAULT_METADATA_FILE) file["id"] = DEFAULT_UUID resp = submit_metadata_file( client, pg_driver, admin, submitter, cgci_blgsp, data=file ) # no index creation assert not create_index.called # no support for aliases assert not create_alias.called # response assert_negative_response(resp) assert_single_entity_from_response(resp)
def test_data_file_already_indexed_object_id_provided_no_hash( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, ): """ Test submitting when the file is already indexed in the index client, its object_id is provided in the submission, and the size and hash DO NOT match those of the indexed file. The empty acl means the file was just uploaded and the empty hash and size mean the file is not ready for metadata submission yet. The submission should fail. The acl and uploader field should NOT have been updated. """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) file = copy.deepcopy(DEFAULT_METADATA_FILE) # provide the object_id of an existing indexed file file["object_id"] = "14fd1746-61bb-401a-96d2-342cfaf70000" document = MagicMock() document.did = file["object_id"] document.size = None document.hashes = None document.acl = [] document.uploader = DEFAULT_SUBMITTER_ID get_index_uuid.return_value = document get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=file) # no index or alias creation assert not create_index.called assert not create_alias.called # response assert_negative_response(resp) entity = assert_single_entity_from_response(resp) # check that the acl and uploader fields have NOT been updated in indexd assert not document.acl assert document.uploader == DEFAULT_SUBMITTER_ID