Esempio n. 1
0
def test_cli_list_users(tmp_cli_runner):  # NOQA
    from dtool_lookup_server.utils import register_users
    from dtool_lookup_server.cli import list_users

    register_users([{
        "username": "******",
        "is_admin": True
    }, {
        "username": "******"
    }])

    result = tmp_cli_runner.invoke(list_users, [])
    assert result.exit_code == 0

    expected_content = [
        {
            "username": "******",
            "is_admin": True,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
        {
            "username": "******",
            "is_admin": False,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
    ]
    assert json.loads(result.output) == expected_content
Esempio n. 2
0
def register():
    """Register a user in the dtool lookup server.

    The user in the Authorization token needs to be admin. Returns 404 for
    non-admins.
    """
    username = get_jwt_identity()
    data = request.get_json()

    try:
        user = get_user_obj(username)
    except AuthenticationError:
        # Unregistered users should see 404.
        abort(404)

    # Non admin users should see 404.
    if not user.is_admin:
        abort(404)


#   # Make it idempotent.
#   if base_uri_exists(base_uri):
#       return "", 201

# There should be some validation of the input here...

    register_users(data)

    return "", 201
Esempio n. 3
0
def tmp_app_with_dependent_data(request):
    from dtool_lookup_server.config import Config
    from dtool_lookup_server import create_app, mongo, sql_db
    from dtool_lookup_server.utils import (
        register_users,
        register_base_uri,
        register_dataset,
        update_permissions,
    )

    tmp_mongo_db_name = random_string()

    config = {
        "FLASK_ENV": "development",
        "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
        "MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name),
        "SQLALCHEMY_TRACK_MODIFICATIONS": False,
        "JWT_ALGORITHM": "RS256",
        "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
        "JWT_TOKEN_LOCATION": "headers",
        "JWT_HEADER_NAME": "Authorization",
        "JWT_HEADER_TYPE": "Bearer",
    }

    app = create_app(config)

    # Ensure the sql database has been put into the context.
    app.app_context().push()

    # Populate the database.
    sql_db.Model.metadata.create_all(sql_db.engine)

    # Register some users.
    username = "******"
    register_users([
        dict(username=username),
    ])

    base_uri = "s3://snow-white"
    register_base_uri(base_uri)
    permissions = {
        "base_uri": base_uri,
        "users_with_search_permissions": [username],
        "users_with_register_permissions": [username]
    }
    update_permissions(permissions)

    for dataset_info in family_datasets(base_uri):
        register_dataset(dataset_info)

    @request.addfinalizer
    def teardown():
        mongo.cx.drop_database(tmp_mongo_db_name)
        sql_db.session.remove()

    return app.test_client()
Esempio n. 4
0
def test_cli_give_register_permission(tmp_cli_runner):  # NOQA

    from dtool_lookup_server.utils import (
        get_base_uri_obj,
        register_users,
        register_base_uri,
    )

    username1 = "sleepy"
    username2 = "dopey"
    base_uri_str = "s3://snow-white"
    register_users([{"username": username1}, {"username": username2}])
    register_base_uri(base_uri_str)

    from dtool_lookup_server.cli import give_register_permission

    result = tmp_cli_runner.invoke(give_register_permission,
                                   [username1, base_uri_str])
    assert result.exit_code == 0

    base_uri = get_base_uri_obj(base_uri_str)
    expected_content = {
        "base_uri": base_uri_str,
        "users_with_search_permissions": [],
        "users_with_register_permissions": [username1]
    }
    assert base_uri.as_dict() == expected_content

    result = tmp_cli_runner.invoke(give_register_permission,
                                   [username2, base_uri_str])
    assert result.exit_code == 0

    base_uri = get_base_uri_obj(base_uri_str)
    expected_content = {
        "base_uri": base_uri_str,
        "users_with_search_permissions": [],
        "users_with_register_permissions": [username1, username2]
    }
    assert base_uri.as_dict() == expected_content

    result = tmp_cli_runner.invoke(give_register_permission,
                                   [username2, base_uri_str])
    assert result.exit_code != 0
    assert "User '{}' already has register permissions".format(
        username2) in result.output  # NOQA

    result = tmp_cli_runner.invoke(give_register_permission,
                                   ["dopey", "s3://no-uri"])
    assert result.exit_code != 0
    assert "Base URI 's3://no-uri' not registered" in result.output

    result = tmp_cli_runner.invoke(give_register_permission,
                                   ["noone", base_uri_str])
    assert result.exit_code != 0
    assert "User 'noone' not registered" in result.output
Esempio n. 5
0
def register_user(username, is_admin):
    """Register a user in the dtool lookup server."""

    if user_exists(username):
        click.secho("User '{}' already registered".format(username),
                    fg="red",
                    err=True)
        sys.exit(1)

    users = [{"username": username, "is_admin": is_admin}]
    register_users(users)
Esempio n. 6
0
def test_register_dataset_without_created_at(tmp_app):   # NOQA
    from dtool_lookup_server import ValidationError
    from dtool_lookup_server.utils import (
        register_users,
        register_base_uri,
        update_permissions,
        register_dataset,
        get_admin_metadata_from_uri,
        get_readme_from_uri_by_user,
    )

    register_users([
        dict(username="******"),
        dict(username="******"),
    ])

    base_uri = "s3://snow-white"
    register_base_uri(base_uri)

    permissions = {
        "base_uri": base_uri,
        "users_with_search_permissions": ["grumpy", "sleepy"],
        "users_with_register_permissions": ["grumpy"],
    }
    update_permissions(permissions)

    uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
    uri = "{}/{}".format(base_uri, uuid)
    dataset_info = {
        "base_uri": base_uri,
        "uuid": uuid,
        "uri": uri,
        "name": "my-dataset",
        "type": "dataset",
        "readme": {"description": "test dataset"},
        "manifest": {
            "dtoolcore_version": "3.7.0",
            "hash_function": "md5sum_hexdigest",
            "items": {
                "e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": {
                    "hash": "d89117c9da2cc34586e183017cb14851",
                    "relpath": "U00096.3.rev.1.bt2",
                    "size_in_bytes": 5741810,
                    "utc_timestamp": 1536832115.0
                }
            }
        },
        "creator_username": "******",
        "frozen_at": 1536238185.881941,
        "annotations": {"software": "bowtie2"},
        "tags": ["rnaseq"],
    }

    register_dataset(dataset_info)

    # When missing, created_at will be set to frozen_at.
    expected_content = {
        "base_uri": base_uri,
        "uuid": uuid,
        "uri": uri,
        "name": "my-dataset",
        "creator_username": "******",
        "frozen_at": 1536238185.881941,
        "created_at": 1536238185.881941,
    }
    assert get_admin_metadata_from_uri(uri) == expected_content
    assert get_readme_from_uri_by_user("sleepy", uri) == dataset_info["readme"]

    with pytest.raises(ValidationError):
        register_dataset({"name": "not-all-required-metadata"})
Esempio n. 7
0
def test_register_too_large_metadata_dataset(tmp_app):  # NOQA

    from dtool_lookup_server import ValidationError
    from dtool_lookup_server.utils import (
        register_users,
        register_base_uri,
        update_permissions,
        register_dataset,
        get_admin_metadata_from_uri,
    )

    register_users([
        dict(username="******"),
        dict(username="******"),
    ])

    base_uri = "s3://snow-white"
    register_base_uri(base_uri)

    permissions = {
        "base_uri": base_uri,
        "users_with_search_permissions": ["grumpy", "sleepy"],
        "users_with_register_permissions": ["grumpy"],
    }
    update_permissions(permissions)

    uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
    uri = "{}/{}".format(base_uri, uuid)
    dataset_info = {
        "base_uri": base_uri,
        "uuid": uuid,
        "uri": uri,
        "name": "my-dataset",
        "type": "dataset",
        "manifest": {
            "dtoolcore_version": "3.7.0",
            "hash_function": "md5sum_hexdigest",
            "items": {
                "e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": {
                    "hash": "d89117c9da2cc34586e183017cb14851",
                    "relpath": "U00096.3.rev.1.bt2",
                    "size_in_bytes": 5741810,
                    "utc_timestamp": 1536832115.0
                }
            }
        },
        "creator_username": "******",
        "frozen_at": 1536238185.881941,
        "created_at": 1536236399.19497,
        "annotations": {"software": "bowtie2"},
        "tags": ["rnaseq"],
    }

    really_large_readme = {}
    for i in range(100000):
        key = "here_is_a_long_key_{}".format(i)
        value = "here_is_a_long_value_{}".format(i) * 10
        really_large_readme[key] = value

    dataset_info["readme"] = really_large_readme

    # The dataset_info is too large and raises:
    # pymongo.errors.DocumentTooLarge: BSON document too large (28978543 bytes)
    # - the connected server supports BSON document sizes up to 16793598 bytes.
    # See https://github.com/jic-dtool/dtool-lookup-server/issues/16
    # So the code catches this and raises dtool_lookup_server.ValidationError
    # instead.
    with pytest.raises(ValidationError):
        register_dataset(dataset_info)

    assert get_admin_metadata_from_uri(dataset_info["uri"]) is None
def test_user_management_helper_functions(tmp_app):  # NOQA

    from dtool_lookup_server.utils import (
        register_users,
        get_user_info,
        list_users,
    )

    # Create list of dictionaries of users.
    admin_username = "******"
    data_champion_username = "******"
    standard_user_username = "******"
    users = [
        {"username": admin_username, "is_admin": True},
        {"username": data_champion_username, "is_admin": False},
        {"username": standard_user_username},
    ]

    # Register the users.
    register_users(users)

    user_info = get_user_info(admin_username)
    expected_content = {
        "username": admin_username,
        "is_admin": True,
        "search_permissions_on_base_uris": [],
        "register_permissions_on_base_uris": []
    }
    assert user_info == expected_content

    user_info = get_user_info(data_champion_username)
    expected_content = {
        "username": data_champion_username,
        "is_admin": False,
        "search_permissions_on_base_uris": [],
        "register_permissions_on_base_uris": []
    }
    assert user_info == expected_content

    user_info = get_user_info(standard_user_username)
    expected_content = {
        "username": standard_user_username,
        "is_admin": False,
        "search_permissions_on_base_uris": [],
        "register_permissions_on_base_uris": []
    }
    assert user_info == expected_content

    # Test non-existing user.
    assert get_user_info("no-one") is None

    # Test registering input with an existing user present.
    new_username = "******"
    users = [{"username": data_champion_username}, {"username": new_username}]
    register_users(users)
    user_info = get_user_info(new_username)
    expected_content = {
        "username": new_username,
        "is_admin": False,
        "search_permissions_on_base_uris": [],
        "register_permissions_on_base_uris": []
    }
    assert user_info == expected_content

    # Test listing users.
    expected_content = [
        {
            "username": admin_username,
            "is_admin": True,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
        {
            "username": data_champion_username,
            "is_admin": False,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
        {
            "username": standard_user_username,
            "is_admin": False,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
        {
            "username": new_username,
            "is_admin": False,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
    ]
    assert list_users() == expected_content

    # Test deleting users.
    from dtool_lookup_server.utils import delete_users

    users_to_delete = [
        {
            "username": standard_user_username,
            "is_admin": False,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
        {
            "username": new_username,
            "is_admin": False,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
    ]
    delete_users(users_to_delete)

    expected_content = [
        {
            "username": admin_username,
            "is_admin": True,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
        {
            "username": data_champion_username,
            "is_admin": False,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
    ]
    assert list_users() == expected_content

    # Test updating users admin privileges.
    from dtool_lookup_server.utils import update_users

    users_to_update = [
        {"username": admin_username},  # The is_admin value defaults to False.
        {"username": data_champion_username, "is_admin": True},
        {"username": standard_user_username},  # Not in system so ignored.
    ]
    update_users(users_to_update)

    expected_content = [
        {
            "username": admin_username,
            "is_admin": False,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
        {
            "username": data_champion_username,
            "is_admin": True,
            "search_permissions_on_base_uris": [],
            "register_permissions_on_base_uris": []
        },
    ]
    assert list_users() == expected_content
Esempio n. 9
0
def tmp_app_with_data(request):

    from dtool_lookup_server import create_app, mongo, sql_db
    from dtool_lookup_server.utils import (
        register_users,
        register_base_uri,
        register_dataset,
        update_permissions,
    )

    tmp_mongo_db_name = random_string()

    config = {
        "FLASK_ENV": "development",
        "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
        "MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name),
        "SQLALCHEMY_TRACK_MODIFICATIONS": False,
        "JWT_ALGORITHM": "RS256",
        "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
        "JWT_TOKEN_LOCATION": "headers",
        "JWT_HEADER_NAME": "Authorization",
        "JWT_HEADER_TYPE": "Bearer",
    }

    app = create_app(config)

    # Ensure the sql database has been put into the context.
    app.app_context().push()

    # Populate the database.
    sql_db.Model.metadata.create_all(sql_db.engine)

    # Register some users.
    username = "******"
    register_users([
        dict(username=username),
        dict(username="******"),
        dict(username="******", is_admin=True)
    ])

    # Add base URIs and update permissions
    for base_uri in ["s3://snow-white", "s3://mr-men"]:
        register_base_uri(base_uri)
        permissions = {
            "base_uri": base_uri,
            "users_with_search_permissions": [username],
            "users_with_register_permissions": [username]
        }
        update_permissions(permissions)

    # Add some data to the database.
    for base_uri in ["s3://snow-white", "s3://mr-men"]:
        uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
        uri = "{}/{}".format(base_uri, uuid)
        dataset_info = {
            "base_uri": base_uri,
            "type": "dataset",
            "uuid": uuid,
            "uri": uri,
            "name": "bad-apples",
            "readme": {
                "descripton": "apples from queen"
            },
            "manifest": {
                "dtoolcore_version": "3.7.0",
                "hash_function": "md5sum_hexdigest",
                "items": {
                    "e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": {
                        "hash": "d89117c9da2cc34586e183017cb14851",
                        "relpath": "U00096.3.rev.1.bt2",
                        "size_in_bytes": 5741810,
                        "utc_timestamp": 1536832115.0
                    }
                }
            },
            "creator_username": "******",
            "frozen_at": 1536238185.881941,
            "annotations": {
                "type": "fruit"
            },
            "tags": ["evil", "fruit"],
        }
        register_dataset(dataset_info)

    base_uri = "s3://snow-white"
    uuid = "a2218059-5bd0-4690-b090-062faf08e046"
    uri = "{}/{}".format(base_uri, uuid)
    dataset_info = {
        "base_uri": base_uri,
        "type": "dataset",
        "uuid": uuid,
        "uri": uri,
        "name": "oranges",
        "readme": {
            "descripton": "oranges from queen"
        },
        "manifest": {
            "dtoolcore_version": "3.7.0",
            "hash_function": "md5sum_hexdigest",
            "items": {}
        },
        "creator_username": "******",
        "frozen_at": 1536238185.881941,
        "annotations": {
            "type": "fruit",
            "only_here": "crazystuff"
        },
        "tags": ["good", "fruit"],
    }
    register_dataset(dataset_info)

    @request.addfinalizer
    def teardown():
        mongo.cx.drop_database(tmp_mongo_db_name)
        sql_db.session.remove()

    return app.test_client()
def test_user_management_helper_functions(tmp_app):  # NOQA

    from dtool_lookup_server.utils import (
        register_users,
        register_base_uri,
        list_users,
        list_base_uris,
        update_permissions,
        get_permission_info,
    )

    base_uri = "s3://snow-white"

    usernames = ["snow-white", "dopey", "sleepy"]
    users = [{"username": u} for u in usernames]

    permissions = {
        "base_uri": base_uri,
        "users_with_search_permissions": usernames,
        "users_with_register_permissions": [usernames[0]],
    }

    register_base_uri(base_uri)
    register_users(users)
    update_permissions(permissions)

    assert get_permission_info(base_uri) == permissions

    expected_content = [{
        "username":
        "******",
        "is_admin":
        False,
        "search_permissions_on_base_uris": ["s3://snow-white"],
        "register_permissions_on_base_uris": ["s3://snow-white"]
    }, {
        "username": "******",
        "is_admin": False,
        "search_permissions_on_base_uris": ["s3://snow-white"],
        "register_permissions_on_base_uris": []
    }, {
        "username": "******",
        "is_admin": False,
        "search_permissions_on_base_uris": ["s3://snow-white"],
        "register_permissions_on_base_uris": []
    }]
    assert list_users() == expected_content

    expected_content = [{
        "base_uri":
        base_uri,
        "users_with_search_permissions": ["snow-white", "dopey", "sleepy"],
        "users_with_register_permissions": ["snow-white"]
    }]
    assert list_base_uris() == expected_content

    # Wipe the permissions again.
    permissions = {
        "base_uri": base_uri,
        "users_with_search_permissions": [],
        "users_with_register_permissions": [],
    }
    update_permissions(permissions)
    assert get_permission_info(base_uri) == permissions

    expected_content = [{
        "username": "******",
        "is_admin": False,
        "search_permissions_on_base_uris": [],
        "register_permissions_on_base_uris": []
    }, {
        "username": "******",
        "is_admin": False,
        "search_permissions_on_base_uris": [],
        "register_permissions_on_base_uris": []
    }, {
        "username": "******",
        "is_admin": False,
        "search_permissions_on_base_uris": [],
        "register_permissions_on_base_uris": []
    }]
    assert list_users() == expected_content

    expected_content = [{
        "base_uri": base_uri,
        "users_with_search_permissions": [],
        "users_with_register_permissions": []
    }]
    assert list_base_uris() == expected_content
Esempio n. 11
0
def tmp_app(request):

    from dtool_lookup_server import create_app, mongo, sql_db
    from dtool_lookup_server.utils import (
        register_users,
        register_base_uri,
        register_dataset,
        update_permissions,
    )

    # Create temporary sqlite URI.
    d = tempfile.mkdtemp()
    sqlite_uri = randome_sqlite_uri(d)

    # Create temporary mongodb name.
    tmp_mongo_db_name = random_string()

    config = {
        "SECRET_KEY": "secret",
        "FLASK_ENV": "development",
        "SQLALCHEMY_DATABASE_URI": sqlite_uri,
        "MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name),
        "SQLALCHEMY_TRACK_MODIFICATIONS": False,
        "JWT_ALGORITHM": "RS256",
        "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
        "JWT_TOKEN_LOCATION": "headers",
        "JWT_HEADER_NAME": "Authorization",
        "JWT_HEADER_TYPE": "Bearer",
    }

    app = create_app(config)

    # Ensure the sql database has been put into the context.
    app.app_context().push()

    # Populate the database.
    sql_db.Model.metadata.create_all(sql_db.engine)

    # Register some users.
    username = "******"
    register_users([
        dict(username=username),
        dict(username="******"),
    ])

    # Register base URIs and set permissions.
    base_uri_1 = "s3://snow-white"
    base_uri_2 = "s3://mr-men"
    for base_uri in [base_uri_1, base_uri_2]:
        register_base_uri(base_uri)
        permissions = {
            "base_uri": base_uri,
            "users_with_search_permissions": [username],
            "users_with_register_permissions": [],
        }
        update_permissions(permissions)

    dataset_info = generate_dataset_info(
        base_uri_1,
        "blue-shirt",
        {"color": "blue"}
    )
    register_dataset(dataset_info)

    dataset_info = generate_dataset_info(
        base_uri_2,
        "red-wavy-shirt",
        {
            "color": "red",
            "pattern": "wavy",
            "complex_ignored": ["lists", "are", "ignored"]
        }
    )
    register_dataset(dataset_info)

    dataset_info = generate_dataset_info(
        base_uri_1,
        "stripy-shirt",
        {
            "pattern": "stripey",
            "color": ["purple", "gray"]  # Complex data type so ignored
        }
    )
    register_dataset(dataset_info)

    dataset_info = generate_dataset_info(
        base_uri_1,
        "complex-shirt",
        {
            "pattern": ["lies", "circles"],  # Complex data type so ignored
            "color": ["purple", "gray"]  # Complex data type so ignored
        }
    )
    register_dataset(dataset_info)  # Whole dataset ignored by plugin.

    @request.addfinalizer
    def teardown():
        mongo.cx.drop_database(tmp_mongo_db_name)
        sql_db.session.remove()
        shutil.rmtree(d)

    return app.test_client()
Esempio n. 12
0
def test_list_datasets_by_user(tmp_app):  # NOQA

    from dtool_lookup_server.utils import (
        register_users,
        register_base_uri,
        update_permissions,
        register_dataset_admin_metadata,
        list_datasets_by_user,
    )

    base_uri_1 = "s3://snow-white"
    base_uri_2 = "s3://mr-men"
    register_base_uri(base_uri_1)
    register_base_uri(base_uri_2)

    username_1 = "dopey"
    username_2 = "grumpy"
    register_users([{"username": un} for un in (username_1, username_2)])

    uuid_1 = "11111111-1111-1111-1111-111111111111"
    uuid_2 = "22222222-2222-2222-2222-222222222222"
    uri_1 = "{}/{}".format(base_uri_1, uuid_1)
    uri_2 = "{}/{}".format(base_uri_2, uuid_2)

    admin_metadata_1 = {
        "base_uri": base_uri_1,
        "uuid": uuid_1,
        "uri": uri_1,
        "name": "ds_1",
        "creator_username": "******",
        "frozen_at": 1536238185.881941,
        "created_at": 1536236399.19497,
    }
    admin_metadata_2 = {
        "base_uri": base_uri_2,
        "uuid": uuid_2,
        "uri": uri_2,
        "name": "ds_2",
        "creator_username": "******",
        "frozen_at": 1536238185.881941,
        "created_at": 1536236399.19497,
    }
    register_dataset_admin_metadata(admin_metadata_1)
    register_dataset_admin_metadata(admin_metadata_2)

    permissions_1 = {
        "base_uri": base_uri_1,
        "users_with_search_permissions": [username_1],
        "users_with_register_permissions": []
    }
    permissions_2 = {
        "base_uri": base_uri_2,
        "users_with_search_permissions": [username_1, username_2],
        "users_with_register_permissions": []
    }
    update_permissions(permissions_1)
    update_permissions(permissions_2)

    expected_content = [admin_metadata_1, admin_metadata_2]
    assert list_datasets_by_user(username_1) == expected_content

    expected_content = [admin_metadata_2]
    assert list_datasets_by_user(username_2) == expected_content