def give_register_permission(username, base_uri): """Give a user register permission on a base URI.""" if not base_uri_exists(base_uri): click.secho("Base URI '{}' not registered".format(base_uri), fg="red", err=True) sys.exit(1) if not user_exists(username): click.secho("User '{}' not registered".format(username), fg="red", err=True) sys.exit(1) permissions = get_permission_info(base_uri) if username in permissions["users_with_register_permissions"]: click.secho( "User '{}' already has register permissions".format(username), fg="red", err=True) sys.exit(1) permissions["users_with_register_permissions"].append(username) update_permissions(permissions)
def tmp_app_with_dependent_data(request): from dtool_lookup_server.config import Config from dtool_lookup_server import create_app, mongo, sql_db from dtool_lookup_server.utils import ( register_users, register_base_uri, register_dataset, update_permissions, ) tmp_mongo_db_name = random_string() config = { "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", "MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name), "SQLALCHEMY_TRACK_MODIFICATIONS": False, "JWT_ALGORITHM": "RS256", "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY, "JWT_TOKEN_LOCATION": "headers", "JWT_HEADER_NAME": "Authorization", "JWT_HEADER_TYPE": "Bearer", } app = create_app(config) # Ensure the sql database has been put into the context. app.app_context().push() # Populate the database. sql_db.Model.metadata.create_all(sql_db.engine) # Register some users. username = "******" register_users([ dict(username=username), ]) base_uri = "s3://snow-white" register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": [username], "users_with_register_permissions": [username] } update_permissions(permissions) for dataset_info in family_datasets(base_uri): register_dataset(dataset_info) @request.addfinalizer def teardown(): mongo.cx.drop_database(tmp_mongo_db_name) sql_db.session.remove() return app.test_client()
def test_register_dataset_without_created_at(tmp_app): # NOQA from dtool_lookup_server import ValidationError from dtool_lookup_server.utils import ( register_users, register_base_uri, update_permissions, register_dataset, get_admin_metadata_from_uri, get_readme_from_uri_by_user, ) register_users([ dict(username="******"), dict(username="******"), ]) base_uri = "s3://snow-white" register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": ["grumpy", "sleepy"], "users_with_register_permissions": ["grumpy"], } update_permissions(permissions) uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337" uri = "{}/{}".format(base_uri, uuid) dataset_info = { "base_uri": base_uri, "uuid": uuid, "uri": uri, "name": "my-dataset", "type": "dataset", "readme": {"description": "test dataset"}, "manifest": { "dtoolcore_version": "3.7.0", "hash_function": "md5sum_hexdigest", "items": { "e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": { "hash": "d89117c9da2cc34586e183017cb14851", "relpath": "U00096.3.rev.1.bt2", "size_in_bytes": 5741810, "utc_timestamp": 1536832115.0 } } }, "creator_username": "******", "frozen_at": 1536238185.881941, "annotations": {"software": "bowtie2"}, "tags": ["rnaseq"], } register_dataset(dataset_info) # When missing, created_at will be set to frozen_at. expected_content = { "base_uri": base_uri, "uuid": uuid, "uri": uri, "name": "my-dataset", "creator_username": "******", "frozen_at": 1536238185.881941, "created_at": 1536238185.881941, } assert get_admin_metadata_from_uri(uri) == expected_content assert get_readme_from_uri_by_user("sleepy", uri) == dataset_info["readme"] with pytest.raises(ValidationError): register_dataset({"name": "not-all-required-metadata"})
def test_register_too_large_metadata_dataset(tmp_app): # NOQA from dtool_lookup_server import ValidationError from dtool_lookup_server.utils import ( register_users, register_base_uri, update_permissions, register_dataset, get_admin_metadata_from_uri, ) register_users([ dict(username="******"), dict(username="******"), ]) base_uri = "s3://snow-white" register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": ["grumpy", "sleepy"], "users_with_register_permissions": ["grumpy"], } update_permissions(permissions) uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337" uri = "{}/{}".format(base_uri, uuid) dataset_info = { "base_uri": base_uri, "uuid": uuid, "uri": uri, "name": "my-dataset", "type": "dataset", "manifest": { "dtoolcore_version": "3.7.0", "hash_function": "md5sum_hexdigest", "items": { "e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": { "hash": "d89117c9da2cc34586e183017cb14851", "relpath": "U00096.3.rev.1.bt2", "size_in_bytes": 5741810, "utc_timestamp": 1536832115.0 } } }, "creator_username": "******", "frozen_at": 1536238185.881941, "created_at": 1536236399.19497, "annotations": {"software": "bowtie2"}, "tags": ["rnaseq"], } really_large_readme = {} for i in range(100000): key = "here_is_a_long_key_{}".format(i) value = "here_is_a_long_value_{}".format(i) * 10 really_large_readme[key] = value dataset_info["readme"] = really_large_readme # The dataset_info is too large and raises: # pymongo.errors.DocumentTooLarge: BSON document too large (28978543 bytes) # - the connected server supports BSON document sizes up to 16793598 bytes. # See https://github.com/jic-dtool/dtool-lookup-server/issues/16 # So the code catches this and raises dtool_lookup_server.ValidationError # instead. with pytest.raises(ValidationError): register_dataset(dataset_info) assert get_admin_metadata_from_uri(dataset_info["uri"]) is None
def tmp_app_with_data(request): from dtool_lookup_server import create_app, mongo, sql_db from dtool_lookup_server.utils import ( register_users, register_base_uri, register_dataset, update_permissions, ) tmp_mongo_db_name = random_string() config = { "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", "MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name), "SQLALCHEMY_TRACK_MODIFICATIONS": False, "JWT_ALGORITHM": "RS256", "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY, "JWT_TOKEN_LOCATION": "headers", "JWT_HEADER_NAME": "Authorization", "JWT_HEADER_TYPE": "Bearer", } app = create_app(config) # Ensure the sql database has been put into the context. app.app_context().push() # Populate the database. sql_db.Model.metadata.create_all(sql_db.engine) # Register some users. username = "******" register_users([ dict(username=username), dict(username="******"), dict(username="******", is_admin=True) ]) # Add base URIs and update permissions for base_uri in ["s3://snow-white", "s3://mr-men"]: register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": [username], "users_with_register_permissions": [username] } update_permissions(permissions) # Add some data to the database. for base_uri in ["s3://snow-white", "s3://mr-men"]: uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337" uri = "{}/{}".format(base_uri, uuid) dataset_info = { "base_uri": base_uri, "type": "dataset", "uuid": uuid, "uri": uri, "name": "bad-apples", "readme": { "descripton": "apples from queen" }, "manifest": { "dtoolcore_version": "3.7.0", "hash_function": "md5sum_hexdigest", "items": { "e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": { "hash": "d89117c9da2cc34586e183017cb14851", "relpath": "U00096.3.rev.1.bt2", "size_in_bytes": 5741810, "utc_timestamp": 1536832115.0 } } }, "creator_username": "******", "frozen_at": 1536238185.881941, "annotations": { "type": "fruit" }, "tags": ["evil", "fruit"], } register_dataset(dataset_info) base_uri = "s3://snow-white" uuid = "a2218059-5bd0-4690-b090-062faf08e046" uri = "{}/{}".format(base_uri, uuid) dataset_info = { "base_uri": base_uri, "type": "dataset", "uuid": uuid, "uri": uri, "name": "oranges", "readme": { "descripton": "oranges from queen" }, "manifest": { "dtoolcore_version": "3.7.0", "hash_function": "md5sum_hexdigest", "items": {} }, "creator_username": "******", "frozen_at": 1536238185.881941, "annotations": { "type": "fruit", "only_here": "crazystuff" }, "tags": ["good", "fruit"], } register_dataset(dataset_info) @request.addfinalizer def teardown(): mongo.cx.drop_database(tmp_mongo_db_name) sql_db.session.remove() return app.test_client()
def test_user_management_helper_functions(tmp_app): # NOQA from dtool_lookup_server.utils import ( register_users, register_base_uri, list_users, list_base_uris, update_permissions, get_permission_info, ) base_uri = "s3://snow-white" usernames = ["snow-white", "dopey", "sleepy"] users = [{"username": u} for u in usernames] permissions = { "base_uri": base_uri, "users_with_search_permissions": usernames, "users_with_register_permissions": [usernames[0]], } register_base_uri(base_uri) register_users(users) update_permissions(permissions) assert get_permission_info(base_uri) == permissions expected_content = [{ "username": "******", "is_admin": False, "search_permissions_on_base_uris": ["s3://snow-white"], "register_permissions_on_base_uris": ["s3://snow-white"] }, { "username": "******", "is_admin": False, "search_permissions_on_base_uris": ["s3://snow-white"], "register_permissions_on_base_uris": [] }, { "username": "******", "is_admin": False, "search_permissions_on_base_uris": ["s3://snow-white"], "register_permissions_on_base_uris": [] }] assert list_users() == expected_content expected_content = [{ "base_uri": base_uri, "users_with_search_permissions": ["snow-white", "dopey", "sleepy"], "users_with_register_permissions": ["snow-white"] }] assert list_base_uris() == expected_content # Wipe the permissions again. permissions = { "base_uri": base_uri, "users_with_search_permissions": [], "users_with_register_permissions": [], } update_permissions(permissions) assert get_permission_info(base_uri) == permissions expected_content = [{ "username": "******", "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": "******", "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }, { "username": "******", "is_admin": False, "search_permissions_on_base_uris": [], "register_permissions_on_base_uris": [] }] assert list_users() == expected_content expected_content = [{ "base_uri": base_uri, "users_with_search_permissions": [], "users_with_register_permissions": [] }] assert list_base_uris() == expected_content
def tmp_app(request): from dtool_lookup_server import create_app, mongo, sql_db from dtool_lookup_server.utils import ( register_users, register_base_uri, register_dataset, update_permissions, ) # Create temporary sqlite URI. d = tempfile.mkdtemp() sqlite_uri = randome_sqlite_uri(d) # Create temporary mongodb name. tmp_mongo_db_name = random_string() config = { "SECRET_KEY": "secret", "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": sqlite_uri, "MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name), "SQLALCHEMY_TRACK_MODIFICATIONS": False, "JWT_ALGORITHM": "RS256", "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY, "JWT_TOKEN_LOCATION": "headers", "JWT_HEADER_NAME": "Authorization", "JWT_HEADER_TYPE": "Bearer", } app = create_app(config) # Ensure the sql database has been put into the context. app.app_context().push() # Populate the database. sql_db.Model.metadata.create_all(sql_db.engine) # Register some users. username = "******" register_users([ dict(username=username), dict(username="******"), ]) # Register base URIs and set permissions. base_uri_1 = "s3://snow-white" base_uri_2 = "s3://mr-men" for base_uri in [base_uri_1, base_uri_2]: register_base_uri(base_uri) permissions = { "base_uri": base_uri, "users_with_search_permissions": [username], "users_with_register_permissions": [], } update_permissions(permissions) dataset_info = generate_dataset_info( base_uri_1, "blue-shirt", {"color": "blue"} ) register_dataset(dataset_info) dataset_info = generate_dataset_info( base_uri_2, "red-wavy-shirt", { "color": "red", "pattern": "wavy", "complex_ignored": ["lists", "are", "ignored"] } ) register_dataset(dataset_info) dataset_info = generate_dataset_info( base_uri_1, "stripy-shirt", { "pattern": "stripey", "color": ["purple", "gray"] # Complex data type so ignored } ) register_dataset(dataset_info) dataset_info = generate_dataset_info( base_uri_1, "complex-shirt", { "pattern": ["lies", "circles"], # Complex data type so ignored "color": ["purple", "gray"] # Complex data type so ignored } ) register_dataset(dataset_info) # Whole dataset ignored by plugin. @request.addfinalizer def teardown(): mongo.cx.drop_database(tmp_mongo_db_name) sql_db.session.remove() shutil.rmtree(d) return app.test_client()
def test_notify_route(tmp_app_with_users, tmp_dir_fixture): # NOQA bucket_name = 'bucket' # Add local directory as base URI and assign URI to the bucket base_uri = sanitise_uri(tmp_dir_fixture) register_base_uri(base_uri) update_permissions({ 'base_uri': base_uri, 'users_with_search_permissions': ['snow-white'], 'users_with_register_permissions': ['snow-white'], }) Config.BUCKET_TO_BASE_URI[bucket_name] = base_uri # Create test dataset name = "my_dataset" admin_metadata = generate_admin_metadata(name) dest_uri = DiskStorageBroker.generate_uri(name=name, uuid=admin_metadata["uuid"], base_uri=tmp_dir_fixture) sample_data_path = os.path.join(TEST_SAMPLE_DATA) local_file_path = os.path.join(sample_data_path, 'tiny.png') # Create a minimal dataset proto_dataset = ProtoDataSet(uri=dest_uri, admin_metadata=admin_metadata, config_path=None) proto_dataset.create() readme = 'abc: def' proto_dataset.put_readme(readme) proto_dataset.put_item(local_file_path, 'tiny.png') proto_dataset.freeze() # Read in a dataset dataset = DataSet.from_uri(dest_uri) expected_identifier = generate_identifier('tiny.png') assert expected_identifier in dataset.identifiers assert len(dataset.identifiers) == 1 # Tell plugin that dataset has been created r = tmp_app_with_users.post( "/elastic-search/notify/all/{}".format(name), json={ 'bucket': bucket_name, 'metadata': dataset._admin_metadata }, ) assert r.status_code == 200 # Check that dataset has actually been registered datasets = list_datasets_by_user('snow-white') assert len(datasets) == 1 assert datasets[0]['base_uri'] == base_uri assert datasets[0]['uri'] == dest_uri assert datasets[0]['uuid'] == admin_metadata['uuid'] assert datasets[0]['name'] == name # Check README check_readme = get_readme_from_uri_by_user('snow-white', dest_uri) assert check_readme == yaml.load(readme) # Update README new_readme = 'ghi: jkl' dataset.put_readme(new_readme) # Notify plugin about updated name r = tmp_app_with_users.post( "/elastic-search/notify/all/{}".format(name), json={ 'bucket': bucket_name, 'metadata': dataset._admin_metadata }, ) assert r.status_code == 200 # Check dataset datasets = list_datasets_by_user('snow-white') assert len(datasets) == 1 assert datasets[0]['base_uri'] == base_uri assert datasets[0]['uri'] == dest_uri assert datasets[0]['uuid'] == admin_metadata['uuid'] assert datasets[0]['name'] == name # Check that README has actually been changed check_readme = get_readme_from_uri_by_user('snow-white', dest_uri) assert check_readme == yaml.load(new_readme) # Tell plugin that dataset has been deleted r = tmp_app_with_users.delete( "/elastic-search/notify/all/{}_{}/dtool".format( bucket_name, admin_metadata['uuid'])) assert r.status_code == 200 # Check that dataset has been deleted datasets = list_datasets_by_user('snow-white') assert len(datasets) == 0
def test_list_datasets_by_user(tmp_app): # NOQA from dtool_lookup_server.utils import ( register_users, register_base_uri, update_permissions, register_dataset_admin_metadata, list_datasets_by_user, ) base_uri_1 = "s3://snow-white" base_uri_2 = "s3://mr-men" register_base_uri(base_uri_1) register_base_uri(base_uri_2) username_1 = "dopey" username_2 = "grumpy" register_users([{"username": un} for un in (username_1, username_2)]) uuid_1 = "11111111-1111-1111-1111-111111111111" uuid_2 = "22222222-2222-2222-2222-222222222222" uri_1 = "{}/{}".format(base_uri_1, uuid_1) uri_2 = "{}/{}".format(base_uri_2, uuid_2) admin_metadata_1 = { "base_uri": base_uri_1, "uuid": uuid_1, "uri": uri_1, "name": "ds_1", "creator_username": "******", "frozen_at": 1536238185.881941, "created_at": 1536236399.19497, } admin_metadata_2 = { "base_uri": base_uri_2, "uuid": uuid_2, "uri": uri_2, "name": "ds_2", "creator_username": "******", "frozen_at": 1536238185.881941, "created_at": 1536236399.19497, } register_dataset_admin_metadata(admin_metadata_1) register_dataset_admin_metadata(admin_metadata_2) permissions_1 = { "base_uri": base_uri_1, "users_with_search_permissions": [username_1], "users_with_register_permissions": [] } permissions_2 = { "base_uri": base_uri_2, "users_with_search_permissions": [username_1, username_2], "users_with_register_permissions": [] } update_permissions(permissions_1) update_permissions(permissions_2) expected_content = [admin_metadata_1, admin_metadata_2] assert list_datasets_by_user(username_1) == expected_content expected_content = [admin_metadata_2] assert list_datasets_by_user(username_2) == expected_content