Ejemplo n.º 1
0
def test_data_file_not_indexed(create_alias, create_index, get_index_uuid,
                               get_index_hash, client, pg_driver, admin,
                               submitter, cgci_blgsp, monkeypatch):
    """
    Test node and data file creation when neither exist and no ID is provided.
    """
    monkeypatch.setitem(flask.current_app.config, 'USE_SIGNPOST', True)
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    get_index_uuid.return_value = None
    get_index_hash.return_value = None

    # signpost create will return a document
    document = MagicMock()
    document.did = '14fd1746-61bb-401a-96d2-342cfaf70000'
    create_index.return_value = document

    resp = submit_metadata_file(client, pg_driver, admin, submitter,
                                cgci_blgsp)

    # index creation should be called with no args for SIGNPOST
    assert create_index.call_count == 1
    args, kwargs = create_index.call_args_list[0]
    assert not args
    assert not kwargs

    # no support for aliases
    assert not create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity['action'] == 'create'
Ejemplo n.º 2
0
def test_data_file_update_url(
    create_alias,
    create_index,
    get_index_uuid,
    get_index_hash,
    client,
    pg_driver,
    admin,
    submitter,
    cgci_blgsp,
):
    """
    Test submitting the same data again but updating the URL field (should
    get added to the indexed file in index service).
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    document = MagicMock()
    document.did = "14fd1746-61bb-401a-96d2-342cfaf70000"
    document.urls = [DEFAULT_URL]
    get_index_hash.return_value = document

    # only return the correct document by uuid IF the uuid provided is
    # the one from above
    def get_index_by_uuid(uuid):
        if uuid == document.did:
            return document
        else:
            return None

    get_index_uuid.side_effect = get_index_by_uuid

    submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp)

    # now submit again but change url
    new_url = "some/new/url/location/to/add"
    updated_file = copy.deepcopy(DEFAULT_METADATA_FILE)
    updated_file["urls"] = new_url
    resp = submit_metadata_file(client,
                                pg_driver,
                                admin,
                                submitter,
                                cgci_blgsp,
                                data=updated_file)

    # no index or alias creation
    assert not create_index.called
    assert not create_alias.called

    # make sure new url are in the document and patch gets called
    assert document.urls == [new_url]
    assert document.patch.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity["action"] == "update"

    assert entity["id"] != document.did
Ejemplo n.º 3
0
def test_data_file_update_multiple_urls(create_alias, create_index,
                                        get_index_uuid, get_index_hash, client,
                                        pg_driver, admin, submitter,
                                        cgci_blgsp):
    """
    Test submitting the same data again but updating the URL field (should
    get added to the indexed file in index service).
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    document = MagicMock()
    document.did = '14fd1746-61bb-401a-96d2-342cfaf70000'
    document.urls = [DEFAULT_URL]
    get_index_hash.return_value = document

    # only return the correct document by uuid IF the uuid provided is
    # the one from above
    def get_index_by_uuid(uuid):
        if uuid == document.did:
            return document
        else:
            return None

    get_index_uuid.side_effect = get_index_by_uuid

    submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp)

    # now submit again but change url
    new_url = 'some/new/url/location/to/add'
    another_new_url = 'some/other/url'
    updated_file = copy.deepcopy(DEFAULT_METADATA_FILE)

    # comma separated list of urls INCLUDING the url that's already there
    updated_file['urls'] = DEFAULT_URL + ',' + new_url + ',' + another_new_url
    resp = submit_metadata_file(client,
                                pg_driver,
                                admin,
                                submitter,
                                cgci_blgsp,
                                data=updated_file)

    # no index or alias creation
    assert not create_index.called
    assert not create_alias.called

    # make sure original url and new url are in the document and patch gets called
    assert DEFAULT_URL in document.urls
    assert new_url in document.urls
    assert another_new_url in document.urls
    assert document.patch.called

    # make sure no duplicates were added
    assert len(document.urls) == 3

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity['action'] == 'update'
Ejemplo n.º 4
0
def test_data_file_update_url_id_provided(create_alias, create_index,
                                          get_index_uuid, get_index_hash,
                                          client, pg_driver, admin, submitter,
                                          cgci_blgsp):
    """
    Test submitting the same data again (with the id provided) and updating the
    URL field (should get added to the indexed file in index service).
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    document = MagicMock()
    document.did = '14fd1746-61bb-401a-96d2-342cfaf70000'
    document.urls = [DEFAULT_URL]
    get_index_hash.return_value = document

    # only return the correct document by uuid IF the uuid provided is
    # the one from above
    def get_index_by_uuid(uuid):
        if uuid == document.did:
            return document
        else:
            return None

    get_index_uuid.side_effect = get_index_by_uuid

    submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp)

    # now submit again but change url
    new_url = 'some/new/url/location/to/add'
    updated_file = copy.deepcopy(DEFAULT_METADATA_FILE)
    updated_file['urls'] = new_url
    updated_file['id'] = document.did
    resp = submit_metadata_file(client,
                                pg_driver,
                                admin,
                                submitter,
                                cgci_blgsp,
                                data=updated_file)

    # no index or alias creation
    assert not create_index.called
    assert not create_alias.called

    # make sure original url and new url are in the document and patch gets called
    assert DEFAULT_URL in document.urls
    assert new_url in document.urls
    assert document.patch.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity['action'] == 'update'

    # make sure uuid in node is the same as the uuid from index
    # FIXME this is a temporary solution so these tests will probably
    #       need to change in the future
    assert entity['id'] == document.did
Ejemplo n.º 5
0
def test_data_file_already_indexed_id_provided(
    create_alias,
    create_index,
    get_index_uuid,
    get_index_hash,
    client,
    pg_driver,
    admin,
    submitter,
    cgci_blgsp,
):
    """
    Test submitting when the file is already indexed in the index client and
    an id is provided in the submission.
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    document = MagicMock()
    document.did = "14fd1746-61bb-401a-96d2-342cfaf70000"
    get_index_uuid.return_value = document
    get_index_hash.return_value = document

    # only return the correct document by uuid IF the uuid provided is
    # the one from above
    def get_index_by_uuid(uuid):
        if uuid == document.did:
            return document
        else:
            return None

    get_index_uuid.side_effect = get_index_by_uuid

    file = copy.deepcopy(DEFAULT_METADATA_FILE)
    file["id"] = document.did
    resp = submit_metadata_file(client,
                                pg_driver,
                                admin,
                                submitter,
                                cgci_blgsp,
                                data=file)

    # no index or alias creation
    assert not create_index.called
    assert not create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity["action"] == "create"

    # make sure uuid in node is the same as the uuid from index
    # FIXME this is a temporary solution so these tests will probably
    #       need to change in the future
    assert entity["id"] == document.did
Ejemplo n.º 6
0
def test_data_file_not_indexed_id_provided(
    create_alias,
    create_index,
    get_index_uuid,
    get_index_hash,
    client,
    pg_driver,
    admin,
    submitter,
    cgci_blgsp,
    require_index_exists_off,
):
    """
    Test node and data file creation when neither exist and an ID is provided.
    That ID should be used for the node and file index creation
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    get_index_uuid.return_value = None
    get_index_hash.return_value = None

    file = copy.deepcopy(DEFAULT_METADATA_FILE)
    file["id"] = DEFAULT_UUID
    resp = submit_metadata_file(client,
                                pg_driver,
                                admin,
                                submitter,
                                cgci_blgsp,
                                data=file)

    # index creation
    assert create_index.call_count == 1
    args, kwargs = create_index.call_args_list[0]
    assert "did" in kwargs
    did = kwargs["did"]
    assert "hashes" in kwargs
    assert kwargs["hashes"].get("md5") == DEFAULT_FILE_HASH
    assert "urls" in kwargs
    assert DEFAULT_URL in kwargs["urls"]

    # alias creation
    assert create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity["action"] == "create"

    # make sure uuid in node is the same as the uuid from index
    # FIXME this is a temporary solution so these tests will probably
    #       need to change in the future
    assert did == DEFAULT_UUID
    assert entity["id"] == DEFAULT_UUID
Ejemplo n.º 7
0
def test_can_submit_data_file_with_asterisk_json(
    create_alias,
    create_index,
    get_index_uuid,
    get_index_hash,
    client,
    pg_driver,
    admin,
    submitter,
    cgci_blgsp,
):
    """
    Test that we can submit a file when some fields have asterisks prepended
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    document = MagicMock()
    document.did = "14fd1746-61bb-401a-96d2-342cfaf70000"
    get_index_uuid.return_value = document
    get_index_hash.return_value = document

    # only return the correct document by uuid IF the uuid provided is
    # the one from above
    def get_index_by_uuid(uuid):
        if uuid == document.did:
            return document
        else:
            return None

    get_index_uuid.side_effect = get_index_by_uuid

    file = copy.deepcopy(DEFAULT_METADATA_FILE)
    file["id"] = document.did

    # insert asterisks before the property names
    for key in file.keys():
        file["*{}".format(key)] = file.pop(key)

    resp = submit_metadata_file(client,
                                pg_driver,
                                admin,
                                submitter,
                                cgci_blgsp,
                                data=file)

    # no index or alias creation
    assert not create_index.called
    assert not create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity["action"] == "create"
Ejemplo n.º 8
0
def test_link_case_insensitivity(
    create_alias,
    create_index,
    get_index_uuid,
    get_index_hash,
    client,
    pg_driver,
    admin,
    submitter,
    cgci_blgsp,
):
    """
    Test that links are case insensitive.
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    for i in range(16):
        document = MagicMock()
        document.did = "14fd1746-61bb-401a-96d2-342cfaf7000" + str(i)
        document.urls = [DEFAULT_URL]
        get_index_hash.return_value = document

        # only return the correct document by uuid IF the uuid provided is
        # the one from above
        def get_index_by_uuid(uuid):
            if uuid == document.did:
                return document
            else:
                return None

        get_index_uuid.side_effect = get_index_by_uuid

        updated_file = copy.deepcopy(DEFAULT_METADATA_FILE)
        updated_file["submitter_id"] = str(i)
        updated_file["experiments"]["submitter_id"] = "".join(
            random.choice([k.upper(), k.lower()])
            for k in updated_file["experiments"]["submitter_id"])
        resp = submit_metadata_file(client,
                                    pg_driver,
                                    admin,
                                    submitter,
                                    cgci_blgsp,
                                    data=updated_file)

        # no index or alias creation
        assert not create_index.called
        assert not create_alias.called

        # response
        assert_positive_response(resp)
        entity = assert_single_entity_from_response(resp)
        assert entity["action"] == "create"
Ejemplo n.º 9
0
def test_data_file_already_indexed(
    create_alias,
    create_index,
    get_index_uuid,
    get_index_hash,
    client,
    pg_driver,
    admin,
    submitter,
    cgci_blgsp,
):
    """
    Test submitting when the file is already indexed in the index client and
    no ID is provided. sheepdog should fall back on the hash/size of the file
    to find it in indexing service.
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    document = MagicMock()
    document.did = "14fd1746-61bb-401a-96d2-342cfaf70000"
    get_index_hash.return_value = document

    # only return the correct document by uuid IF the uuid provided is
    # the one from above
    def get_index_by_uuid(uuid):
        if uuid == document.did:
            return document
        else:
            return None

    get_index_uuid.side_effect = get_index_by_uuid

    resp = submit_metadata_file(client, pg_driver, admin, submitter,
                                cgci_blgsp)

    # no index or alias creation
    assert not create_index.called
    assert not create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity["action"] == "create"

    path = "/v0/submission/CGCI/BLGSP/export/?format=json&ids={nid}".format(
        nid=entity["id"])
    r = client.get(path, headers=submitter)

    data = r.json
    assert len(data) == 1
    assert data[0]["object_id"] == document.did
    assert data[0]["id"] != document.did
Ejemplo n.º 10
0
def test_data_file_not_indexed(
    create_alias,
    create_index,
    get_index_uuid,
    get_index_hash,
    client,
    pg_driver,
    admin,
    submitter,
    cgci_blgsp,
    require_index_exists_off,
):
    """
    Test node and data file creation when neither exist and no ID is provided.
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    get_index_uuid.return_value = None
    get_index_hash.return_value = None

    resp = submit_metadata_file(client, pg_driver, admin, submitter,
                                cgci_blgsp)

    # index creation
    assert create_index.call_count == 1
    _, kwargs = create_index.call_args_list[0]
    assert "did" in kwargs
    did = kwargs["did"]
    assert "hashes" in kwargs
    assert kwargs["hashes"].get("md5") == DEFAULT_FILE_HASH
    assert "urls" in kwargs
    assert DEFAULT_URL in kwargs["urls"]

    # alias creation
    assert create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity["action"] == "create"

    path = "/v0/submission/CGCI/BLGSP/export/?format=json&ids={nid}".format(
        nid=entity["id"])
    r = client.get(path, headers=submitter)

    data = r.json
    assert len(data) == 1
    assert did == None
Ejemplo n.º 11
0
def test_data_file_already_indexed(
    create_alias,
    create_index,
    get_index_uuid,
    get_index_hash,
    client,
    pg_driver,
    admin,
    submitter,
    cgci_blgsp,
    monkeypatch,
):
    """
    Test submitting when the file is already indexed in the index client and
    no ID is provided. sheepdog should fall back on the hash/size of the file
    to find it in indexing service.

    NOTE: signpostclient will create another file in the index service regardless
          of whether or not the file exists. signpostclient does not
          have capabilities of searching for files based on hash/size
    """
    monkeypatch.setitem(flask.current_app.config, "USE_SIGNPOST", True)
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    # signpostclient cannot find by hash/size
    get_index_hash.return_value = None
    get_index_uuid.return_value = None

    # signpost create will return a document
    document = MagicMock()
    document.did = "14fd1746-61bb-401a-96d2-342cfaf70000"
    create_index.return_value = document

    resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp)

    # index creation should be called with no args for SIGNPOST
    assert create_index.call_count == 1
    args, kwargs = create_index.call_args_list[0]
    assert not args
    assert not kwargs

    # no support for aliases
    assert not create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity["action"] == "create"
Ejemplo n.º 12
0
def test_data_file_already_indexed(create_alias, create_index, get_index_uuid,
                                   get_index_hash, client, pg_driver, admin,
                                   submitter, cgci_blgsp):
    """
    Test submitting when the file is already indexed in the index client and
    no ID is provided. sheepdog should fall back on the hash/size of the file
    to find it in indexing service.
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    document = MagicMock()
    document.did = '14fd1746-61bb-401a-96d2-342cfaf70000'
    get_index_hash.return_value = document

    # only return the correct document by uuid IF the uuid provided is
    # the one from above
    def get_index_by_uuid(uuid):
        if uuid == document.did:
            return document
        else:
            return None

    get_index_uuid.side_effect = get_index_by_uuid

    resp = submit_metadata_file(client, pg_driver, admin, submitter,
                                cgci_blgsp)

    # no index or alias creation
    assert not create_index.called
    assert not create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity['action'] == 'create'

    # make sure uuid in node is the same as the uuid from index
    # FIXME this is a temporary solution so these tests will probably
    #       need to change in the future
    assert entity['id'] == document.did
Ejemplo n.º 13
0
def test_data_file_not_indexed(
        create_alias, create_index, get_index_uuid, get_index_hash,
        client, pg_driver, admin, submitter, cgci_blgsp,
        require_index_exists_off):
    """
    Test node and data file creation when neither exist and no ID is provided.
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    get_index_uuid.return_value = None
    get_index_hash.return_value = None

    resp = submit_metadata_file(client, pg_driver, admin, submitter, cgci_blgsp)

    # index creation
    assert create_index.call_count == 1
    _, kwargs = create_index.call_args_list[0]
    assert 'did' in kwargs
    did = kwargs['did']
    assert 'hashes' in kwargs
    assert kwargs['hashes'].get('md5') == DEFAULT_FILE_HASH
    assert 'urls' in kwargs
    assert DEFAULT_URL in kwargs['urls']

    # alias creation
    assert create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity['action'] == 'create'

    # make sure uuid in node is the same as the uuid from index
    # FIXME this is a temporary solution so these tests will probably
    #       need to change in the future
    assert entity['id'] == did
Ejemplo n.º 14
0
def test_data_file_already_indexed_object_id_provided_hash_match(
    update_acl_uploader_indexd,
    create_alias,
    create_index,
    get_index_uuid,
    get_index_hash,
    client,
    pg_driver,
    admin,
    submitter,
    submitter_name,
    cgci_blgsp,
):
    """
    Test submitting when the file is already indexed in the index client,
    an id is provided in the submission, and the size and hash match those
    of the indexed file. The empty acl means the file was just uploaded.

    The submission should succeed but no new index should be created. The acl
    and uploader field should have been updated as part of the data upload
    flow.
    """
    submit_first_experiment(client, pg_driver, admin, submitter, cgci_blgsp)

    file = copy.deepcopy(DEFAULT_METADATA_FILE)
    # provide the object_id of an existing indexed file
    file["object_id"] = "14fd1746-61bb-401a-96d2-342cfaf70000"

    document = MagicMock()
    document.did = file["object_id"]
    document.size = file["file_size"]
    document.hashes = {"md5": file["md5sum"], "other": "abc123"}
    document.acl = []
    document.uploader = submitter_name
    get_index_uuid.return_value = document
    get_index_hash.return_value = document

    # only return the correct document by uuid IF the uuid provided is
    # the one from above
    def get_index_by_uuid(uuid):
        if uuid == document.did:
            return document
        else:
            return None

    get_index_uuid.side_effect = get_index_by_uuid

    resp = submit_metadata_file(client,
                                pg_driver,
                                admin,
                                submitter,
                                cgci_blgsp,
                                data=file)

    # no index or alias creation
    assert not create_index.called
    assert not create_alias.called

    # response
    assert_positive_response(resp)
    entity = assert_single_entity_from_response(resp)
    assert entity["action"] == "create"

    path = "/v0/submission/CGCI/BLGSP/export/?format=json&ids={nid}".format(
        nid=entity["id"])
    r = client.get(path, headers=submitter)

    data = r.json
    assert len(data) == 1
    assert data[0]["object_id"] == file["object_id"]

    # check that the acl and uploader fields have been updated in indexd
    assert update_acl_uploader_indexd.called