def test_sql_dataset_helper_functions(tmp_app): # NOQA
from dtool_lookup_server import ValidationError
from dtool_lookup_server.utils import (
register_base_uri,
register_dataset_admin_metadata,
get_admin_metadata_from_uri,
list_admin_metadata_in_base_uri,
)
base_uri = "s3://snow-white"
uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
uri = "{}/{}".format(base_uri, uuid)
admin_metadata = {
"base_uri": base_uri,
"uuid": uuid,
"uri": uri,
"name": "my-dataset",
"creator_username": "******",
"frozen_at": 1536238185.881941,
"created_at": 1536236399.19497,
}
# BaseURI not registered yet.
with pytest.raises(ValidationError):
register_dataset_admin_metadata(admin_metadata)
register_base_uri(base_uri)
register_dataset_admin_metadata(admin_metadata)
assert get_admin_metadata_from_uri(uri) == admin_metadata
expected_content = [admin_metadata]
assert list_admin_metadata_in_base_uri(base_uri) == expected_content
def register():
"""Register a base URI.
The user needs to be admin. Returns 404 for non-admins.
"""
username = get_jwt_identity()
data = request.get_json()
base_uri = data["base_uri"]
try:
user = get_user_obj(username)
except AuthenticationError:
# Unregistered users should see 404.
abort(404)
# Non admin users should see 404.
if not user.is_admin:
abort(404)
# Make it idempotent.
if base_uri_exists(base_uri):
return "", 201
register_base_uri(base_uri)
return "", 201
def tmp_app_with_dependent_data(request):
from dtool_lookup_server.config import Config
from dtool_lookup_server import create_app, mongo, sql_db
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
register_dataset,
update_permissions,
)
tmp_mongo_db_name = random_string()
config = {
"FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
"MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name),
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
"JWT_ALGORITHM": "RS256",
"JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
"JWT_TOKEN_LOCATION": "headers",
"JWT_HEADER_NAME": "Authorization",
"JWT_HEADER_TYPE": "Bearer",
}
app = create_app(config)
# Ensure the sql database has been put into the context.
app.app_context().push()
# Populate the database.
sql_db.Model.metadata.create_all(sql_db.engine)
# Register some users.
username = "******"
register_users([
dict(username=username),
])
base_uri = "s3://snow-white"
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": [username],
"users_with_register_permissions": [username]
}
update_permissions(permissions)
for dataset_info in family_datasets(base_uri):
register_dataset(dataset_info)
@request.addfinalizer
def teardown():
mongo.cx.drop_database(tmp_mongo_db_name)
sql_db.session.remove()
return app.test_client()
def add_base_uri(base_uri):
"""Register a base URI in the dtool lookup server."""
base_uri = dtoolcore.utils.sanitise_uri(base_uri)
if base_uri_exists(base_uri):
click.secho("Base URI '{}' already registered".format(base_uri),
fg="red",
err=True)
sys.exit(1)
register_base_uri(base_uri)
def test_cli_give_register_permission(tmp_cli_runner): # NOQA
from dtool_lookup_server.utils import (
get_base_uri_obj,
register_users,
register_base_uri,
)
username1 = "sleepy"
username2 = "dopey"
base_uri_str = "s3://snow-white"
register_users([{"username": username1}, {"username": username2}])
register_base_uri(base_uri_str)
from dtool_lookup_server.cli import give_register_permission
result = tmp_cli_runner.invoke(give_register_permission,
[username1, base_uri_str])
assert result.exit_code == 0
base_uri = get_base_uri_obj(base_uri_str)
expected_content = {
"base_uri": base_uri_str,
"users_with_search_permissions": [],
"users_with_register_permissions": [username1]
}
assert base_uri.as_dict() == expected_content
result = tmp_cli_runner.invoke(give_register_permission,
[username2, base_uri_str])
assert result.exit_code == 0
base_uri = get_base_uri_obj(base_uri_str)
expected_content = {
"base_uri": base_uri_str,
"users_with_search_permissions": [],
"users_with_register_permissions": [username1, username2]
}
assert base_uri.as_dict() == expected_content
result = tmp_cli_runner.invoke(give_register_permission,
[username2, base_uri_str])
assert result.exit_code != 0
assert "User '{}' already has register permissions".format(
username2) in result.output # NOQA
result = tmp_cli_runner.invoke(give_register_permission,
["dopey", "s3://no-uri"])
assert result.exit_code != 0
assert "Base URI 's3://no-uri' not registered" in result.output
result = tmp_cli_runner.invoke(give_register_permission,
["noone", base_uri_str])
assert result.exit_code != 0
assert "User 'noone' not registered" in result.output
def test_uri_management_helper_functions(tmp_app): # NOQA
from dtool_lookup_server.utils import (
register_base_uri,
list_base_uris,
)
base_uri = "s3://snow-white"
register_base_uri(base_uri)
expected_content = [{
"base_uri": base_uri,
"users_with_search_permissions": [],
"users_with_register_permissions": []
}]
assert list_base_uris() == expected_content
def test_cli_list_base_uri(tmp_cli_runner): # NOQA
from dtool_lookup_server.utils import register_base_uri
base_uris = ["s3://snow-white", "s3://mr-men"]
for uri in base_uris:
register_base_uri(uri)
from dtool_lookup_server.cli import list_base_uris
result = tmp_cli_runner.invoke(list_base_uris, [])
assert result.exit_code == 0
expected_content = sorted([{
"base_uri": uri,
"users_with_search_permissions": [],
"users_with_register_permissions": []
} for uri in base_uris],
key=itemgetter("base_uri"))
actual = sorted(json.loads(result.output), key=itemgetter("base_uri"))
assert actual == expected_content
def test_register_dataset_without_created_at(tmp_app): # NOQA
from dtool_lookup_server import ValidationError
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
update_permissions,
register_dataset,
get_admin_metadata_from_uri,
get_readme_from_uri_by_user,
)
register_users([
dict(username="******"),
dict(username="******"),
])
base_uri = "s3://snow-white"
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": ["grumpy", "sleepy"],
"users_with_register_permissions": ["grumpy"],
}
update_permissions(permissions)
uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
uri = "{}/{}".format(base_uri, uuid)
dataset_info = {
"base_uri": base_uri,
"uuid": uuid,
"uri": uri,
"name": "my-dataset",
"type": "dataset",
"readme": {"description": "test dataset"},
"manifest": {
"dtoolcore_version": "3.7.0",
"hash_function": "md5sum_hexdigest",
"items": {
"e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": {
"hash": "d89117c9da2cc34586e183017cb14851",
"relpath": "U00096.3.rev.1.bt2",
"size_in_bytes": 5741810,
"utc_timestamp": 1536832115.0
}
}
},
"creator_username": "******",
"frozen_at": 1536238185.881941,
"annotations": {"software": "bowtie2"},
"tags": ["rnaseq"],
}
register_dataset(dataset_info)
# When missing, created_at will be set to frozen_at.
expected_content = {
"base_uri": base_uri,
"uuid": uuid,
"uri": uri,
"name": "my-dataset",
"creator_username": "******",
"frozen_at": 1536238185.881941,
"created_at": 1536238185.881941,
}
assert get_admin_metadata_from_uri(uri) == expected_content
assert get_readme_from_uri_by_user("sleepy", uri) == dataset_info["readme"]
with pytest.raises(ValidationError):
register_dataset({"name": "not-all-required-metadata"})
def test_register_too_large_metadata_dataset(tmp_app): # NOQA
from dtool_lookup_server import ValidationError
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
update_permissions,
register_dataset,
get_admin_metadata_from_uri,
)
register_users([
dict(username="******"),
dict(username="******"),
])
base_uri = "s3://snow-white"
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": ["grumpy", "sleepy"],
"users_with_register_permissions": ["grumpy"],
}
update_permissions(permissions)
uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
uri = "{}/{}".format(base_uri, uuid)
dataset_info = {
"base_uri": base_uri,
"uuid": uuid,
"uri": uri,
"name": "my-dataset",
"type": "dataset",
"manifest": {
"dtoolcore_version": "3.7.0",
"hash_function": "md5sum_hexdigest",
"items": {
"e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": {
"hash": "d89117c9da2cc34586e183017cb14851",
"relpath": "U00096.3.rev.1.bt2",
"size_in_bytes": 5741810,
"utc_timestamp": 1536832115.0
}
}
},
"creator_username": "******",
"frozen_at": 1536238185.881941,
"created_at": 1536236399.19497,
"annotations": {"software": "bowtie2"},
"tags": ["rnaseq"],
}
really_large_readme = {}
for i in range(100000):
key = "here_is_a_long_key_{}".format(i)
value = "here_is_a_long_value_{}".format(i) * 10
really_large_readme[key] = value
dataset_info["readme"] = really_large_readme
# The dataset_info is too large and raises:
# pymongo.errors.DocumentTooLarge: BSON document too large (28978543 bytes)
# - the connected server supports BSON document sizes up to 16793598 bytes.
# See https://github.com/jic-dtool/dtool-lookup-server/issues/16
# So the code catches this and raises dtool_lookup_server.ValidationError
# instead.
with pytest.raises(ValidationError):
register_dataset(dataset_info)
assert get_admin_metadata_from_uri(dataset_info["uri"]) is None
def tmp_app_with_data(request):
from dtool_lookup_server import create_app, mongo, sql_db
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
register_dataset,
update_permissions,
)
tmp_mongo_db_name = random_string()
config = {
"FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
"MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name),
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
"JWT_ALGORITHM": "RS256",
"JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
"JWT_TOKEN_LOCATION": "headers",
"JWT_HEADER_NAME": "Authorization",
"JWT_HEADER_TYPE": "Bearer",
}
app = create_app(config)
# Ensure the sql database has been put into the context.
app.app_context().push()
# Populate the database.
sql_db.Model.metadata.create_all(sql_db.engine)
# Register some users.
username = "******"
register_users([
dict(username=username),
dict(username="******"),
dict(username="******", is_admin=True)
])
# Add base URIs and update permissions
for base_uri in ["s3://snow-white", "s3://mr-men"]:
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": [username],
"users_with_register_permissions": [username]
}
update_permissions(permissions)
# Add some data to the database.
for base_uri in ["s3://snow-white", "s3://mr-men"]:
uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
uri = "{}/{}".format(base_uri, uuid)
dataset_info = {
"base_uri": base_uri,
"type": "dataset",
"uuid": uuid,
"uri": uri,
"name": "bad-apples",
"readme": {
"descripton": "apples from queen"
},
"manifest": {
"dtoolcore_version": "3.7.0",
"hash_function": "md5sum_hexdigest",
"items": {
"e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": {
"hash": "d89117c9da2cc34586e183017cb14851",
"relpath": "U00096.3.rev.1.bt2",
"size_in_bytes": 5741810,
"utc_timestamp": 1536832115.0
}
}
},
"creator_username": "******",
"frozen_at": 1536238185.881941,
"annotations": {
"type": "fruit"
},
"tags": ["evil", "fruit"],
}
register_dataset(dataset_info)
base_uri = "s3://snow-white"
uuid = "a2218059-5bd0-4690-b090-062faf08e046"
uri = "{}/{}".format(base_uri, uuid)
dataset_info = {
"base_uri": base_uri,
"type": "dataset",
"uuid": uuid,
"uri": uri,
"name": "oranges",
"readme": {
"descripton": "oranges from queen"
},
"manifest": {
"dtoolcore_version": "3.7.0",
"hash_function": "md5sum_hexdigest",
"items": {}
},
"creator_username": "******",
"frozen_at": 1536238185.881941,
"annotations": {
"type": "fruit",
"only_here": "crazystuff"
},
"tags": ["good", "fruit"],
}
register_dataset(dataset_info)
@request.addfinalizer
def teardown():
mongo.cx.drop_database(tmp_mongo_db_name)
sql_db.session.remove()
return app.test_client()
def test_user_management_helper_functions(tmp_app): # NOQA
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
list_users,
list_base_uris,
update_permissions,
get_permission_info,
)
base_uri = "s3://snow-white"
usernames = ["snow-white", "dopey", "sleepy"]
users = [{"username": u} for u in usernames]
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": usernames,
"users_with_register_permissions": [usernames[0]],
}
register_base_uri(base_uri)
register_users(users)
update_permissions(permissions)
assert get_permission_info(base_uri) == permissions
expected_content = [{
"username":
"******",
"is_admin":
False,
"search_permissions_on_base_uris": ["s3://snow-white"],
"register_permissions_on_base_uris": ["s3://snow-white"]
}, {
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": ["s3://snow-white"],
"register_permissions_on_base_uris": []
}, {
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": ["s3://snow-white"],
"register_permissions_on_base_uris": []
}]
assert list_users() == expected_content
expected_content = [{
"base_uri":
base_uri,
"users_with_search_permissions": ["snow-white", "dopey", "sleepy"],
"users_with_register_permissions": ["snow-white"]
}]
assert list_base_uris() == expected_content
# Wipe the permissions again.
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": [],
"users_with_register_permissions": [],
}
update_permissions(permissions)
assert get_permission_info(base_uri) == permissions
expected_content = [{
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}, {
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}, {
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}]
assert list_users() == expected_content
expected_content = [{
"base_uri": base_uri,
"users_with_search_permissions": [],
"users_with_register_permissions": []
}]
assert list_base_uris() == expected_content
def tmp_app(request):
from dtool_lookup_server import create_app, mongo, sql_db
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
register_dataset,
update_permissions,
)
# Create temporary sqlite URI.
d = tempfile.mkdtemp()
sqlite_uri = randome_sqlite_uri(d)
# Create temporary mongodb name.
tmp_mongo_db_name = random_string()
config = {
"SECRET_KEY": "secret",
"FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": sqlite_uri,
"MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name),
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
"JWT_ALGORITHM": "RS256",
"JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
"JWT_TOKEN_LOCATION": "headers",
"JWT_HEADER_NAME": "Authorization",
"JWT_HEADER_TYPE": "Bearer",
}
app = create_app(config)
# Ensure the sql database has been put into the context.
app.app_context().push()
# Populate the database.
sql_db.Model.metadata.create_all(sql_db.engine)
# Register some users.
username = "******"
register_users([
dict(username=username),
dict(username="******"),
])
# Register base URIs and set permissions.
base_uri_1 = "s3://snow-white"
base_uri_2 = "s3://mr-men"
for base_uri in [base_uri_1, base_uri_2]:
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": [username],
"users_with_register_permissions": [],
}
update_permissions(permissions)
dataset_info = generate_dataset_info(
base_uri_1,
"blue-shirt",
{"color": "blue"}
)
register_dataset(dataset_info)
dataset_info = generate_dataset_info(
base_uri_2,
"red-wavy-shirt",
{
"color": "red",
"pattern": "wavy",
"complex_ignored": ["lists", "are", "ignored"]
}
)
register_dataset(dataset_info)
dataset_info = generate_dataset_info(
base_uri_1,
"stripy-shirt",
{
"pattern": "stripey",
"color": ["purple", "gray"] # Complex data type so ignored
}
)
register_dataset(dataset_info)
dataset_info = generate_dataset_info(
base_uri_1,
"complex-shirt",
{
"pattern": ["lies", "circles"], # Complex data type so ignored
"color": ["purple", "gray"] # Complex data type so ignored
}
)
register_dataset(dataset_info) # Whole dataset ignored by plugin.
@request.addfinalizer
def teardown():
mongo.cx.drop_database(tmp_mongo_db_name)
sql_db.session.remove()
shutil.rmtree(d)
return app.test_client()
def test_notify_route(tmp_app_with_users, tmp_dir_fixture): # NOQA
bucket_name = 'bucket'
# Add local directory as base URI and assign URI to the bucket
base_uri = sanitise_uri(tmp_dir_fixture)
register_base_uri(base_uri)
update_permissions({
'base_uri': base_uri,
'users_with_search_permissions': ['snow-white'],
'users_with_register_permissions': ['snow-white'],
})
Config.BUCKET_TO_BASE_URI[bucket_name] = base_uri
# Create test dataset
name = "my_dataset"
admin_metadata = generate_admin_metadata(name)
dest_uri = DiskStorageBroker.generate_uri(name=name,
uuid=admin_metadata["uuid"],
base_uri=tmp_dir_fixture)
sample_data_path = os.path.join(TEST_SAMPLE_DATA)
local_file_path = os.path.join(sample_data_path, 'tiny.png')
# Create a minimal dataset
proto_dataset = ProtoDataSet(uri=dest_uri,
admin_metadata=admin_metadata,
config_path=None)
proto_dataset.create()
readme = 'abc: def'
proto_dataset.put_readme(readme)
proto_dataset.put_item(local_file_path, 'tiny.png')
proto_dataset.freeze()
# Read in a dataset
dataset = DataSet.from_uri(dest_uri)
expected_identifier = generate_identifier('tiny.png')
assert expected_identifier in dataset.identifiers
assert len(dataset.identifiers) == 1
# Tell plugin that dataset has been created
r = tmp_app_with_users.post(
"/elastic-search/notify/all/{}".format(name),
json={
'bucket': bucket_name,
'metadata': dataset._admin_metadata
},
)
assert r.status_code == 200
# Check that dataset has actually been registered
datasets = list_datasets_by_user('snow-white')
assert len(datasets) == 1
assert datasets[0]['base_uri'] == base_uri
assert datasets[0]['uri'] == dest_uri
assert datasets[0]['uuid'] == admin_metadata['uuid']
assert datasets[0]['name'] == name
# Check README
check_readme = get_readme_from_uri_by_user('snow-white', dest_uri)
assert check_readme == yaml.load(readme)
# Update README
new_readme = 'ghi: jkl'
dataset.put_readme(new_readme)
# Notify plugin about updated name
r = tmp_app_with_users.post(
"/elastic-search/notify/all/{}".format(name),
json={
'bucket': bucket_name,
'metadata': dataset._admin_metadata
},
)
assert r.status_code == 200
# Check dataset
datasets = list_datasets_by_user('snow-white')
assert len(datasets) == 1
assert datasets[0]['base_uri'] == base_uri
assert datasets[0]['uri'] == dest_uri
assert datasets[0]['uuid'] == admin_metadata['uuid']
assert datasets[0]['name'] == name
# Check that README has actually been changed
check_readme = get_readme_from_uri_by_user('snow-white', dest_uri)
assert check_readme == yaml.load(new_readme)
# Tell plugin that dataset has been deleted
r = tmp_app_with_users.delete(
"/elastic-search/notify/all/{}_{}/dtool".format(
bucket_name, admin_metadata['uuid']))
assert r.status_code == 200
# Check that dataset has been deleted
datasets = list_datasets_by_user('snow-white')
assert len(datasets) == 0
def test_list_datasets_by_user(tmp_app): # NOQA
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
update_permissions,
register_dataset_admin_metadata,
list_datasets_by_user,
)
base_uri_1 = "s3://snow-white"
base_uri_2 = "s3://mr-men"
register_base_uri(base_uri_1)
register_base_uri(base_uri_2)
username_1 = "dopey"
username_2 = "grumpy"
register_users([{"username": un} for un in (username_1, username_2)])
uuid_1 = "11111111-1111-1111-1111-111111111111"
uuid_2 = "22222222-2222-2222-2222-222222222222"
uri_1 = "{}/{}".format(base_uri_1, uuid_1)
uri_2 = "{}/{}".format(base_uri_2, uuid_2)
admin_metadata_1 = {
"base_uri": base_uri_1,
"uuid": uuid_1,
"uri": uri_1,
"name": "ds_1",
"creator_username": "******",
"frozen_at": 1536238185.881941,
"created_at": 1536236399.19497,
}
admin_metadata_2 = {
"base_uri": base_uri_2,
"uuid": uuid_2,
"uri": uri_2,
"name": "ds_2",
"creator_username": "******",
"frozen_at": 1536238185.881941,
"created_at": 1536236399.19497,
}
register_dataset_admin_metadata(admin_metadata_1)
register_dataset_admin_metadata(admin_metadata_2)
permissions_1 = {
"base_uri": base_uri_1,
"users_with_search_permissions": [username_1],
"users_with_register_permissions": []
}
permissions_2 = {
"base_uri": base_uri_2,
"users_with_search_permissions": [username_1, username_2],
"users_with_register_permissions": []
}
update_permissions(permissions_1)
update_permissions(permissions_2)
expected_content = [admin_metadata_1, admin_metadata_2]
assert list_datasets_by_user(username_1) == expected_content
expected_content = [admin_metadata_2]
assert list_datasets_by_user(username_2) == expected_content
