def test_cli_list_users(tmp_cli_runner): # NOQA from dtool_lookup_server.utils import register_users from dtool_lookup_server.cli import list_users register_users([{ "username": "******", "is_admin": True }, { "username": "******" }]) result = tmp_cli_runner.invoke(list_users, []) assert result.exit_code == 0 expected_content = [ { "username": "******", "is_admin": True, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": "******", "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, ] assert json.loads(result.output) == expected_content
def register(): """Register a user in the dtool lookup server. The user in the Authorization token needs to be admin. Returns 404 for non-admins. """ username = get_jwt_identity() data = request.get_json() try: user = get_user_obj(username) except AuthenticationError: # Unregistered users should see 404. abort(404) # Non admin users should see 404. if not user.is_admin: abort(404) # # Make it idempotent. # if base_uri_exists(base_uri): # return "", 201 # There should be some validation of the input here... register_users(data) return "", 201
def tmp_app_with_dependent_data(request): from dtool_lookup_server.config import Config from dtool_lookup_server import create_app, mongo, sql_db from dtool_lookup_server.utils import ( register_users, register_base_uri, register_dataset, update_permissions, ) tmp_mongo_db_name = random_string() config = { "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", "MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name), "SQLALCHEMY_TRACK_MODIFICATIONS": False, "JWT_ALGORITHM": "RS256", "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY, "JWT_TOKEN_LOCATION": "headers", "JWT_HEADER_NAME": "Authorization", "JWT_HEADER_TYPE": "Bearer", } app = create_app(config) # Ensure the sql database has been put into the context. app.app_context().push() # Populate the database. sql_db.Model.metadata.create_all(sql_db.engine) # Register some users. username = "******" register_users([ dict(username=username), ]) base_uri = "s3://snow-white" register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": [username], "users_with_register_permissions": [username] } update_permissions(permissions) for dataset_info in family_datasets(base_uri): register_dataset(dataset_info) @request.addfinalizer def teardown(): mongo.cx.drop_database(tmp_mongo_db_name) sql_db.session.remove() return app.test_client()
def test_cli_give_register_permission(tmp_cli_runner): # NOQA from dtool_lookup_server.utils import ( get_base_uri_obj, register_users, register_base_uri, ) username1 = "sleepy" username2 = "dopey" base_uri_str = "s3://snow-white" register_users([{"username": username1}, {"username": username2}]) register_base_uri(base_uri_str) from dtool_lookup_server.cli import give_register_permission result = tmp_cli_runner.invoke(give_register_permission, [username1, base_uri_str]) assert result.exit_code == 0 base_uri = get_base_uri_obj(base_uri_str) expected_content = { "base_uri": base_uri_str, "users_with_search_permissions": [], "users_with_register_permissions": [username1] } assert base_uri.as_dict() == expected_content result = tmp_cli_runner.invoke(give_register_permission, [username2, base_uri_str]) assert result.exit_code == 0 base_uri = get_base_uri_obj(base_uri_str) expected_content = { "base_uri": base_uri_str, "users_with_search_permissions": [], "users_with_register_permissions": [username1, username2] } assert base_uri.as_dict() == expected_content result = tmp_cli_runner.invoke(give_register_permission, [username2, base_uri_str]) assert result.exit_code != 0 assert "User '{}' already has register permissions".format( username2) in result.output # NOQA result = tmp_cli_runner.invoke(give_register_permission, ["dopey", "s3://no-uri"]) assert result.exit_code != 0 assert "Base URI 's3://no-uri' not registered" in result.output result = tmp_cli_runner.invoke(give_register_permission, ["noone", base_uri_str]) assert result.exit_code != 0 assert "User 'noone' not registered" in result.output
def register_user(username, is_admin): """Register a user in the dtool lookup server.""" if user_exists(username): click.secho("User '{}' already registered".format(username), fg="red", err=True) sys.exit(1) users = [{"username": username, "is_admin": is_admin}] register_users(users)
def test_register_dataset_without_created_at(tmp_app): # NOQA from dtool_lookup_server import ValidationError from dtool_lookup_server.utils import ( register_users, register_base_uri, update_permissions, register_dataset, get_admin_metadata_from_uri, get_readme_from_uri_by_user, ) register_users([ dict(username="******"), dict(username="******"), ]) base_uri = "s3://snow-white" register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": ["grumpy", "sleepy"], "users_with_register_permissions": ["grumpy"], } update_permissions(permissions) uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337" uri = "{}/{}".format(base_uri, uuid) dataset_info = { "base_uri": base_uri, "uuid": uuid, "uri": uri, "name": "my-dataset", "type": "dataset", "readme": {"description": "test dataset"}, "manifest": { "dtoolcore_version": "3.7.0", "hash_function": "md5sum_hexdigest", "items": { "e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": { "hash": "d89117c9da2cc34586e183017cb14851", "relpath": "U00096.3.rev.1.bt2", "size_in_bytes": 5741810, "utc_timestamp": 1536832115.0 } } }, "creator_username": "******", "frozen_at": 1536238185.881941, "annotations": {"software": "bowtie2"}, "tags": ["rnaseq"], } register_dataset(dataset_info) # When missing, created_at will be set to frozen_at. expected_content = { "base_uri": base_uri, "uuid": uuid, "uri": uri, "name": "my-dataset", "creator_username": "******", "frozen_at": 1536238185.881941, "created_at": 1536238185.881941, } assert get_admin_metadata_from_uri(uri) == expected_content assert get_readme_from_uri_by_user("sleepy", uri) == dataset_info["readme"] with pytest.raises(ValidationError): register_dataset({"name": "not-all-required-metadata"})
def test_register_too_large_metadata_dataset(tmp_app): # NOQA from dtool_lookup_server import ValidationError from dtool_lookup_server.utils import ( register_users, register_base_uri, update_permissions, register_dataset, get_admin_metadata_from_uri, ) register_users([ dict(username="******"), dict(username="******"), ]) base_uri = "s3://snow-white" register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": ["grumpy", "sleepy"], "users_with_register_permissions": ["grumpy"], } update_permissions(permissions) uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337" uri = "{}/{}".format(base_uri, uuid) dataset_info = { "base_uri": base_uri, "uuid": uuid, "uri": uri, "name": "my-dataset", "type": "dataset", "manifest": { "dtoolcore_version": "3.7.0", "hash_function": "md5sum_hexdigest", "items": { "e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": { "hash": "d89117c9da2cc34586e183017cb14851", "relpath": "U00096.3.rev.1.bt2", "size_in_bytes": 5741810, "utc_timestamp": 1536832115.0 } } }, "creator_username": "******", "frozen_at": 1536238185.881941, "created_at": 1536236399.19497, "annotations": {"software": "bowtie2"}, "tags": ["rnaseq"], } really_large_readme = {} for i in range(100000): key = "here_is_a_long_key_{}".format(i) value = "here_is_a_long_value_{}".format(i) * 10 really_large_readme[key] = value dataset_info["readme"] = really_large_readme # The dataset_info is too large and raises: # pymongo.errors.DocumentTooLarge: BSON document too large (28978543 bytes) # - the connected server supports BSON document sizes up to 16793598 bytes. # See https://github.com/jic-dtool/dtool-lookup-server/issues/16 # So the code catches this and raises dtool_lookup_server.ValidationError # instead. with pytest.raises(ValidationError): register_dataset(dataset_info) assert get_admin_metadata_from_uri(dataset_info["uri"]) is None
def test_user_management_helper_functions(tmp_app): # NOQA from dtool_lookup_server.utils import ( register_users, get_user_info, list_users, ) # Create list of dictionaries of users. admin_username = "******" data_champion_username = "******" standard_user_username = "******" users = [ {"username": admin_username, "is_admin": True}, {"username": data_champion_username, "is_admin": False}, {"username": standard_user_username}, ] # Register the users. register_users(users) user_info = get_user_info(admin_username) expected_content = { "username": admin_username, "is_admin": True, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] } assert user_info == expected_content user_info = get_user_info(data_champion_username) expected_content = { "username": data_champion_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] } assert user_info == expected_content user_info = get_user_info(standard_user_username) expected_content = { "username": standard_user_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] } assert user_info == expected_content # Test non-existing user. assert get_user_info("no-one") is None # Test registering input with an existing user present. new_username = "******" users = [{"username": data_champion_username}, {"username": new_username}] register_users(users) user_info = get_user_info(new_username) expected_content = { "username": new_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] } assert user_info == expected_content # Test listing users. expected_content = [ { "username": admin_username, "is_admin": True, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": data_champion_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": standard_user_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": new_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, ] assert list_users() == expected_content # Test deleting users. from dtool_lookup_server.utils import delete_users users_to_delete = [ { "username": standard_user_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": new_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, ] delete_users(users_to_delete) expected_content = [ { "username": admin_username, "is_admin": True, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": data_champion_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, ] assert list_users() == expected_content # Test updating users admin privileges. from dtool_lookup_server.utils import update_users users_to_update = [ {"username": admin_username}, # The is_admin value defaults to False. {"username": data_champion_username, "is_admin": True}, {"username": standard_user_username}, # Not in system so ignored. ] update_users(users_to_update) expected_content = [ { "username": admin_username, "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": data_champion_username, "is_admin": True, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, ] assert list_users() == expected_content
def tmp_app_with_data(request): from dtool_lookup_server import create_app, mongo, sql_db from dtool_lookup_server.utils import ( register_users, register_base_uri, register_dataset, update_permissions, ) tmp_mongo_db_name = random_string() config = { "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", "MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name), "SQLALCHEMY_TRACK_MODIFICATIONS": False, "JWT_ALGORITHM": "RS256", "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY, "JWT_TOKEN_LOCATION": "headers", "JWT_HEADER_NAME": "Authorization", "JWT_HEADER_TYPE": "Bearer", } app = create_app(config) # Ensure the sql database has been put into the context. app.app_context().push() # Populate the database. sql_db.Model.metadata.create_all(sql_db.engine) # Register some users. username = "******" register_users([ dict(username=username), dict(username="******"), dict(username="******", is_admin=True) ]) # Add base URIs and update permissions for base_uri in ["s3://snow-white", "s3://mr-men"]: register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": [username], "users_with_register_permissions": [username] } update_permissions(permissions) # Add some data to the database. for base_uri in ["s3://snow-white", "s3://mr-men"]: uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337" uri = "{}/{}".format(base_uri, uuid) dataset_info = { "base_uri": base_uri, "type": "dataset", "uuid": uuid, "uri": uri, "name": "bad-apples", "readme": { "descripton": "apples from queen" }, "manifest": { "dtoolcore_version": "3.7.0", "hash_function": "md5sum_hexdigest", "items": { "e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": { "hash": "d89117c9da2cc34586e183017cb14851", "relpath": "U00096.3.rev.1.bt2", "size_in_bytes": 5741810, "utc_timestamp": 1536832115.0 } } }, "creator_username": "******", "frozen_at": 1536238185.881941, "annotations": { "type": "fruit" }, "tags": ["evil", "fruit"], } register_dataset(dataset_info) base_uri = "s3://snow-white" uuid = "a2218059-5bd0-4690-b090-062faf08e046" uri = "{}/{}".format(base_uri, uuid) dataset_info = { "base_uri": base_uri, "type": "dataset", "uuid": uuid, "uri": uri, "name": "oranges", "readme": { "descripton": "oranges from queen" }, "manifest": { "dtoolcore_version": "3.7.0", "hash_function": "md5sum_hexdigest", "items": {} }, "creator_username": "******", "frozen_at": 1536238185.881941, "annotations": { "type": "fruit", "only_here": "crazystuff" }, "tags": ["good", "fruit"], } register_dataset(dataset_info) @request.addfinalizer def teardown(): mongo.cx.drop_database(tmp_mongo_db_name) sql_db.session.remove() return app.test_client()
def test_user_management_helper_functions(tmp_app): # NOQA from dtool_lookup_server.utils import ( register_users, register_base_uri, list_users, list_base_uris, update_permissions, get_permission_info, ) base_uri = "s3://snow-white" usernames = ["snow-white", "dopey", "sleepy"] users = [{"username": u} for u in usernames] permissions = { "base_uri": base_uri, "users_with_search_permissions": usernames, "users_with_register_permissions": [usernames[0]], } register_base_uri(base_uri) register_users(users) update_permissions(permissions) assert get_permission_info(base_uri) == permissions expected_content = [{ "username": "******", "is_admin": False, "search_permissions_on_base_uris": ["s3://snow-white"], "register_permissions_on_base_uris": ["s3://snow-white"] }, { "username": "******", "is_admin": False, "search_permissions_on_base_uris": ["s3://snow-white"], "register_permissions_on_base_uris": [] }, { "username": "******", "is_admin": False, "search_permissions_on_base_uris": ["s3://snow-white"], "register_permissions_on_base_uris": [] }] assert list_users() == expected_content expected_content = [{ "base_uri": base_uri, "users_with_search_permissions": ["snow-white", "dopey", "sleepy"], "users_with_register_permissions": ["snow-white"] }] assert list_base_uris() == expected_content # Wipe the permissions again. permissions = { "base_uri": base_uri, "users_with_search_permissions": [], "users_with_register_permissions": [], } update_permissions(permissions) assert get_permission_info(base_uri) == permissions expected_content = [{ "username": "******", "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": "******", "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": "******", "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }] assert list_users() == expected_content expected_content = [{ "base_uri": base_uri, "users_with_search_permissions": [], "users_with_register_permissions": [] }] assert list_base_uris() == expected_content
def tmp_app(request): from dtool_lookup_server import create_app, mongo, sql_db from dtool_lookup_server.utils import ( register_users, register_base_uri, register_dataset, update_permissions, ) # Create temporary sqlite URI. d = tempfile.mkdtemp() sqlite_uri = randome_sqlite_uri(d) # Create temporary mongodb name. tmp_mongo_db_name = random_string() config = { "SECRET_KEY": "secret", "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": sqlite_uri, "MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name), "SQLALCHEMY_TRACK_MODIFICATIONS": False, "JWT_ALGORITHM": "RS256", "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY, "JWT_TOKEN_LOCATION": "headers", "JWT_HEADER_NAME": "Authorization", "JWT_HEADER_TYPE": "Bearer", } app = create_app(config) # Ensure the sql database has been put into the context. app.app_context().push() # Populate the database. sql_db.Model.metadata.create_all(sql_db.engine) # Register some users. username = "******" register_users([ dict(username=username), dict(username="******"), ]) # Register base URIs and set permissions. base_uri_1 = "s3://snow-white" base_uri_2 = "s3://mr-men" for base_uri in [base_uri_1, base_uri_2]: register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": [username], "users_with_register_permissions": [], } update_permissions(permissions) dataset_info = generate_dataset_info( base_uri_1, "blue-shirt", {"color": "blue"} ) register_dataset(dataset_info) dataset_info = generate_dataset_info( base_uri_2, "red-wavy-shirt", { "color": "red", "pattern": "wavy", "complex_ignored": ["lists", "are", "ignored"] } ) register_dataset(dataset_info) dataset_info = generate_dataset_info( base_uri_1, "stripy-shirt", { "pattern": "stripey", "color": ["purple", "gray"] # Complex data type so ignored } ) register_dataset(dataset_info) dataset_info = generate_dataset_info( base_uri_1, "complex-shirt", { "pattern": ["lies", "circles"], # Complex data type so ignored "color": ["purple", "gray"] # Complex data type so ignored } ) register_dataset(dataset_info) # Whole dataset ignored by plugin. @request.addfinalizer def teardown(): mongo.cx.drop_database(tmp_mongo_db_name) sql_db.session.remove() shutil.rmtree(d) return app.test_client()
def test_list_datasets_by_user(tmp_app): # NOQA from dtool_lookup_server.utils import ( register_users, register_base_uri, update_permissions, register_dataset_admin_metadata, list_datasets_by_user, ) base_uri_1 = "s3://snow-white" base_uri_2 = "s3://mr-men" register_base_uri(base_uri_1) register_base_uri(base_uri_2) username_1 = "dopey" username_2 = "grumpy" register_users([{"username": un} for un in (username_1, username_2)]) uuid_1 = "11111111-1111-1111-1111-111111111111" uuid_2 = "22222222-2222-2222-2222-222222222222" uri_1 = "{}/{}".format(base_uri_1, uuid_1) uri_2 = "{}/{}".format(base_uri_2, uuid_2) admin_metadata_1 = { "base_uri": base_uri_1, "uuid": uuid_1, "uri": uri_1, "name": "ds_1", "creator_username": "******", "frozen_at": 1536238185.881941, "created_at": 1536236399.19497, } admin_metadata_2 = { "base_uri": base_uri_2, "uuid": uuid_2, "uri": uri_2, "name": "ds_2", "creator_username": "******", "frozen_at": 1536238185.881941, "created_at": 1536236399.19497, } register_dataset_admin_metadata(admin_metadata_1) register_dataset_admin_metadata(admin_metadata_2) permissions_1 = { "base_uri": base_uri_1, "users_with_search_permissions": [username_1], "users_with_register_permissions": [] } permissions_2 = { "base_uri": base_uri_2, "users_with_search_permissions": [username_1, username_2], "users_with_register_permissions": [] } update_permissions(permissions_1) update_permissions(permissions_2) expected_content = [admin_metadata_1, admin_metadata_2] assert list_datasets_by_user(username_1) == expected_content expected_content = [admin_metadata_2] assert list_datasets_by_user(username_2) == expected_content