def test_data_file_not_indexed(create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, monkeypatch): """ Test node and data file creation when neither exist and no ID is provided. """ monkeypatch.setitem(flask.current_app.config, 'USE_SIGNPOST', True) submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) get_index_uuid.return_value = None get_index_hash.return_value = None # signpost create will return a document document = MagicMock() document.did = '14fd1746-61bb-401a-96d2-342cfaf70000' create_index.return_value = document resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # index creation should be called with no args for SIGNPOST assert create_index.call_count == 1 args, kwargs = create_index.call_args_list[0] assert not args assert not kwargs # no support for aliases assert not create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity['action'] == 'create'
def test_data_file_update_url( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, ): """ Test submitting the same data again but updating the URL field (should get added to the indexed file in index service). """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = "14fd1746-61bb-401a-96d2-342cfaf70000" document.urls = [DEFAULT_URL] get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # now submit again but change url new_url = "some/new/url/location/to/add" updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) updated_file["urls"] = new_url resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=updated_file) # no index or alias creation assert not create_index.called assert not create_alias.called # make sure new url are in the document and patch gets called assert document.urls == [new_url] assert document.patch.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "update" assert entity["id"] != document.did
def test_data_file_update_multiple_urls(create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp): """ Test submitting the same data again but updating the URL field (should get added to the indexed file in index service). """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = '14fd1746-61bb-401a-96d2-342cfaf70000' document.urls = [DEFAULT_URL] get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # now submit again but change url new_url = 'some/new/url/location/to/add' another_new_url = 'some/other/url' updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) # comma separated list of urls INCLUDING the url that's already there updated_file['urls'] = DEFAULT_URL + ',' + new_url + ',' + another_new_url resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=updated_file) # no index or alias creation assert not create_index.called assert not create_alias.called # make sure original url and new url are in the document and patch gets called assert DEFAULT_URL in document.urls assert new_url in document.urls assert another_new_url in document.urls assert document.patch.called # make sure no duplicates were added assert len(document.urls) == 3 # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity['action'] == 'update'
def test_data_file_update_url_id_provided(create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp): """ Test submitting the same data again (with the id provided) and updating the URL field (should get added to the indexed file in index service). """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = '14fd1746-61bb-401a-96d2-342cfaf70000' document.urls = [DEFAULT_URL] get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # now submit again but change url new_url = 'some/new/url/location/to/add' updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) updated_file['urls'] = new_url updated_file['id'] = document.did resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=updated_file) # no index or alias creation assert not create_index.called assert not create_alias.called # make sure original url and new url are in the document and patch gets called assert DEFAULT_URL in document.urls assert new_url in document.urls assert document.patch.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity['action'] == 'update' # make sure uuid in node is the same as the uuid from index # FIXME this is a temporary solution so these tests will probably # need to change in the future assert entity['id'] == document.did
def test_data_file_already_indexed_id_provided( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, ): """ Test submitting when the file is already indexed in the index client and an id is provided in the submission. """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = "14fd1746-61bb-401a-96d2-342cfaf70000" get_index_uuid.return_value = document get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid file = copy.deepcopy(DEFAULT_METADATA_FILE) file["id"] = document.did resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=file) # no index or alias creation assert not create_index.called assert not create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "create" # make sure uuid in node is the same as the uuid from index # FIXME this is a temporary solution so these tests will probably # need to change in the future assert entity["id"] == document.did
def test_data_file_not_indexed_id_provided( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, require_index_exists_off, ): """ Test node and data file creation when neither exist and an ID is provided. That ID should be used for the node and file index creation """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) get_index_uuid.return_value = None get_index_hash.return_value = None file = copy.deepcopy(DEFAULT_METADATA_FILE) file["id"] = DEFAULT_UUID resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=file) # index creation assert create_index.call_count == 1 args, kwargs = create_index.call_args_list[0] assert "did" in kwargs did = kwargs["did"] assert "hashes" in kwargs assert kwargs["hashes"].get("md5") == DEFAULT_FILE_HASH assert "urls" in kwargs assert DEFAULT_URL in kwargs["urls"] # alias creation assert create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "create" # make sure uuid in node is the same as the uuid from index # FIXME this is a temporary solution so these tests will probably # need to change in the future assert did == DEFAULT_UUID assert entity["id"] == DEFAULT_UUID
def test_can_submit_data_file_with_asterisk_json( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, ): """ Test that we can submit a file when some fields have asterisks prepended """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = "14fd1746-61bb-401a-96d2-342cfaf70000" get_index_uuid.return_value = document get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid file = copy.deepcopy(DEFAULT_METADATA_FILE) file["id"] = document.did # insert asterisks before the property names for key in file.keys(): file["*{}".format(key)] = file.pop(key) resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=file) # no index or alias creation assert not create_index.called assert not create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "create"
def test_link_case_insensitivity( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, ): """ Test that links are case insensitive. """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) for i in range(16): document = MagicMock() document.did = "14fd1746-61bb-401a-96d2-342cfaf7000" + str(i) document.urls = [DEFAULT_URL] get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid updated_file = copy.deepcopy(DEFAULT_METADATA_FILE) updated_file["submitter_id"] = str(i) updated_file["experiments"]["submitter_id"] = "".join( random.choice([k.upper(), k.lower()]) for k in updated_file["experiments"]["submitter_id"]) resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=updated_file) # no index or alias creation assert not create_index.called assert not create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "create"
def test_data_file_already_indexed( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, ): """ Test submitting when the file is already indexed in the index client and no ID is provided. sheepdog should fall back on the hash/size of the file to find it in indexing service. """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = "14fd1746-61bb-401a-96d2-342cfaf70000" get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # no index or alias creation assert not create_index.called assert not create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "create" path = "/v0/submission/CGCI/BLGSP/export/?format=json&ids={nid}".format( nid=entity["id"]) r = client.get(path, headers=submitter) data = r.json assert len(data) == 1 assert data[0]["object_id"] == document.did assert data[0]["id"] != document.did
def test_data_file_not_indexed( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, require_index_exists_off, ): """ Test node and data file creation when neither exist and no ID is provided. """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) get_index_uuid.return_value = None get_index_hash.return_value = None resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # index creation assert create_index.call_count == 1 _, kwargs = create_index.call_args_list[0] assert "did" in kwargs did = kwargs["did"] assert "hashes" in kwargs assert kwargs["hashes"].get("md5") == DEFAULT_FILE_HASH assert "urls" in kwargs assert DEFAULT_URL in kwargs["urls"] # alias creation assert create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "create" path = "/v0/submission/CGCI/BLGSP/export/?format=json&ids={nid}".format( nid=entity["id"]) r = client.get(path, headers=submitter) data = r.json assert len(data) == 1 assert did == None
def test_data_file_already_indexed( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, monkeypatch, ): """ Test submitting when the file is already indexed in the index client and no ID is provided. sheepdog should fall back on the hash/size of the file to find it in indexing service. NOTE: signpostclient will create another file in the index service regardless of whether or not the file exists. signpostclient does not have capabilities of searching for files based on hash/size """ monkeypatch.setitem(flask.current_app.config, "USE_SIGNPOST", True) submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) # signpostclient cannot find by hash/size get_index_hash.return_value = None get_index_uuid.return_value = None # signpost create will return a document document = MagicMock() document.did = "14fd1746-61bb-401a-96d2-342cfaf70000" create_index.return_value = document resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # index creation should be called with no args for SIGNPOST assert create_index.call_count == 1 args, kwargs = create_index.call_args_list[0] assert not args assert not kwargs # no support for aliases assert not create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "create"
def test_data_file_already_indexed(create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp): """ Test submitting when the file is already indexed in the index client and no ID is provided. sheepdog should fall back on the hash/size of the file to find it in indexing service. """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) document = MagicMock() document.did = '14fd1746-61bb-401a-96d2-342cfaf70000' get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # no index or alias creation assert not create_index.called assert not create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity['action'] == 'create' # make sure uuid in node is the same as the uuid from index # FIXME this is a temporary solution so these tests will probably # need to change in the future assert entity['id'] == document.did
def test_data_file_not_indexed( create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, cgci_blgsp, require_index_exists_off): """ Test node and data file creation when neither exist and no ID is provided. """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) get_index_uuid.return_value = None get_index_hash.return_value = None resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp) # index creation assert create_index.call_count == 1 _, kwargs = create_index.call_args_list[0] assert 'did' in kwargs did = kwargs['did'] assert 'hashes' in kwargs assert kwargs['hashes'].get('md5') == DEFAULT_FILE_HASH assert 'urls' in kwargs assert DEFAULT_URL in kwargs['urls'] # alias creation assert create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity['action'] == 'create' # make sure uuid in node is the same as the uuid from index # FIXME this is a temporary solution so these tests will probably # need to change in the future assert entity['id'] == did
def test_data_file_already_indexed_object_id_provided_hash_match( update_acl_uploader_indexd, create_alias, create_index, get_index_uuid, get_index_hash, client, pg_driver, admin, submitter, submitter_name, cgci_blgsp, ): """ Test submitting when the file is already indexed in the index client, an id is provided in the submission, and the size and hash match those of the indexed file. The empty acl means the file was just uploaded. The submission should succeed but no new index should be created. The acl and uploader field should have been updated as part of the data upload flow. """ submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp) file = copy.deepcopy(DEFAULT_METADATA_FILE) # provide the object_id of an existing indexed file file["object_id"] = "14fd1746-61bb-401a-96d2-342cfaf70000" document = MagicMock() document.did = file["object_id"] document.size = file["file_size"] document.hashes = {"md5": file["md5sum"], "other": "abc123"} document.acl = [] document.uploader = submitter_name get_index_uuid.return_value = document get_index_hash.return_value = document # only return the correct document by uuid IF the uuid provided is # the one from above def get_index_by_uuid(uuid): if uuid == document.did: return document else: return None get_index_uuid.side_effect = get_index_by_uuid resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp, data=file) # no index or alias creation assert not create_index.called assert not create_alias.called # response assert_positive_response(resp) entity = assert_single_entity_from_response(resp) assert entity["action"] == "create" path = "/v0/submission/CGCI/BLGSP/export/?format=json&ids={nid}".format( nid=entity["id"]) r = client.get(path, headers=submitter) data = r.json assert len(data) == 1 assert data[0]["object_id"] == file["object_id"] # check that the acl and uploader fields have been updated in indexd assert update_acl_uploader_indexd.called