def test_cli_list_users(tmp_cli_runner): # NOQA
from dtool_lookup_server.utils import register_users
from dtool_lookup_server.cli import list_users
register_users([{
"username": "******",
"is_admin": True
}, {
"username": "******"
}])
result = tmp_cli_runner.invoke(list_users, [])
assert result.exit_code == 0
expected_content = [
{
"username": "******",
"is_admin": True,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
{
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
]
assert json.loads(result.output) == expected_content
def register():
"""Register a user in the dtool lookup server.
The user in the Authorization token needs to be admin. Returns 404 for
non-admins.
"""
username = get_jwt_identity()
data = request.get_json()
try:
user = get_user_obj(username)
except AuthenticationError:
# Unregistered users should see 404.
abort(404)
# Non admin users should see 404.
if not user.is_admin:
abort(404)
# # Make it idempotent.
# if base_uri_exists(base_uri):
# return "", 201
# There should be some validation of the input here...
register_users(data)
return "", 201
def tmp_app_with_dependent_data(request):
from dtool_lookup_server.config import Config
from dtool_lookup_server import create_app, mongo, sql_db
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
register_dataset,
update_permissions,
)
tmp_mongo_db_name = random_string()
config = {
"FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
"MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name),
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
"JWT_ALGORITHM": "RS256",
"JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
"JWT_TOKEN_LOCATION": "headers",
"JWT_HEADER_NAME": "Authorization",
"JWT_HEADER_TYPE": "Bearer",
}
app = create_app(config)
# Ensure the sql database has been put into the context.
app.app_context().push()
# Populate the database.
sql_db.Model.metadata.create_all(sql_db.engine)
# Register some users.
username = "******"
register_users([
dict(username=username),
])
base_uri = "s3://snow-white"
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": [username],
"users_with_register_permissions": [username]
}
update_permissions(permissions)
for dataset_info in family_datasets(base_uri):
register_dataset(dataset_info)
@request.addfinalizer
def teardown():
mongo.cx.drop_database(tmp_mongo_db_name)
sql_db.session.remove()
return app.test_client()
def test_cli_give_register_permission(tmp_cli_runner): # NOQA
from dtool_lookup_server.utils import (
get_base_uri_obj,
register_users,
register_base_uri,
)
username1 = "sleepy"
username2 = "dopey"
base_uri_str = "s3://snow-white"
register_users([{"username": username1}, {"username": username2}])
register_base_uri(base_uri_str)
from dtool_lookup_server.cli import give_register_permission
result = tmp_cli_runner.invoke(give_register_permission,
[username1, base_uri_str])
assert result.exit_code == 0
base_uri = get_base_uri_obj(base_uri_str)
expected_content = {
"base_uri": base_uri_str,
"users_with_search_permissions": [],
"users_with_register_permissions": [username1]
}
assert base_uri.as_dict() == expected_content
result = tmp_cli_runner.invoke(give_register_permission,
[username2, base_uri_str])
assert result.exit_code == 0
base_uri = get_base_uri_obj(base_uri_str)
expected_content = {
"base_uri": base_uri_str,
"users_with_search_permissions": [],
"users_with_register_permissions": [username1, username2]
}
assert base_uri.as_dict() == expected_content
result = tmp_cli_runner.invoke(give_register_permission,
[username2, base_uri_str])
assert result.exit_code != 0
assert "User '{}' already has register permissions".format(
username2) in result.output # NOQA
result = tmp_cli_runner.invoke(give_register_permission,
["dopey", "s3://no-uri"])
assert result.exit_code != 0
assert "Base URI 's3://no-uri' not registered" in result.output
result = tmp_cli_runner.invoke(give_register_permission,
["noone", base_uri_str])
assert result.exit_code != 0
assert "User 'noone' not registered" in result.output
def register_user(username, is_admin):
"""Register a user in the dtool lookup server."""
if user_exists(username):
click.secho("User '{}' already registered".format(username),
fg="red",
err=True)
sys.exit(1)
users = [{"username": username, "is_admin": is_admin}]
register_users(users)
def test_register_dataset_without_created_at(tmp_app): # NOQA
from dtool_lookup_server import ValidationError
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
update_permissions,
register_dataset,
get_admin_metadata_from_uri,
get_readme_from_uri_by_user,
)
register_users([
dict(username="******"),
dict(username="******"),
])
base_uri = "s3://snow-white"
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": ["grumpy", "sleepy"],
"users_with_register_permissions": ["grumpy"],
}
update_permissions(permissions)
uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
uri = "{}/{}".format(base_uri, uuid)
dataset_info = {
"base_uri": base_uri,
"uuid": uuid,
"uri": uri,
"name": "my-dataset",
"type": "dataset",
"readme": {"description": "test dataset"},
"manifest": {
"dtoolcore_version": "3.7.0",
"hash_function": "md5sum_hexdigest",
"items": {
"e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": {
"hash": "d89117c9da2cc34586e183017cb14851",
"relpath": "U00096.3.rev.1.bt2",
"size_in_bytes": 5741810,
"utc_timestamp": 1536832115.0
}
}
},
"creator_username": "******",
"frozen_at": 1536238185.881941,
"annotations": {"software": "bowtie2"},
"tags": ["rnaseq"],
}
register_dataset(dataset_info)
# When missing, created_at will be set to frozen_at.
expected_content = {
"base_uri": base_uri,
"uuid": uuid,
"uri": uri,
"name": "my-dataset",
"creator_username": "******",
"frozen_at": 1536238185.881941,
"created_at": 1536238185.881941,
}
assert get_admin_metadata_from_uri(uri) == expected_content
assert get_readme_from_uri_by_user("sleepy", uri) == dataset_info["readme"]
with pytest.raises(ValidationError):
register_dataset({"name": "not-all-required-metadata"})
def test_register_too_large_metadata_dataset(tmp_app): # NOQA
from dtool_lookup_server import ValidationError
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
update_permissions,
register_dataset,
get_admin_metadata_from_uri,
)
register_users([
dict(username="******"),
dict(username="******"),
])
base_uri = "s3://snow-white"
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": ["grumpy", "sleepy"],
"users_with_register_permissions": ["grumpy"],
}
update_permissions(permissions)
uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
uri = "{}/{}".format(base_uri, uuid)
dataset_info = {
"base_uri": base_uri,
"uuid": uuid,
"uri": uri,
"name": "my-dataset",
"type": "dataset",
"manifest": {
"dtoolcore_version": "3.7.0",
"hash_function": "md5sum_hexdigest",
"items": {
"e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": {
"hash": "d89117c9da2cc34586e183017cb14851",
"relpath": "U00096.3.rev.1.bt2",
"size_in_bytes": 5741810,
"utc_timestamp": 1536832115.0
}
}
},
"creator_username": "******",
"frozen_at": 1536238185.881941,
"created_at": 1536236399.19497,
"annotations": {"software": "bowtie2"},
"tags": ["rnaseq"],
}
really_large_readme = {}
for i in range(100000):
key = "here_is_a_long_key_{}".format(i)
value = "here_is_a_long_value_{}".format(i) * 10
really_large_readme[key] = value
dataset_info["readme"] = really_large_readme
# The dataset_info is too large and raises:
# pymongo.errors.DocumentTooLarge: BSON document too large (28978543 bytes)
# - the connected server supports BSON document sizes up to 16793598 bytes.
# See https://github.com/jic-dtool/dtool-lookup-server/issues/16
# So the code catches this and raises dtool_lookup_server.ValidationError
# instead.
with pytest.raises(ValidationError):
register_dataset(dataset_info)
assert get_admin_metadata_from_uri(dataset_info["uri"]) is None
def test_user_management_helper_functions(tmp_app): # NOQA
from dtool_lookup_server.utils import (
register_users,
get_user_info,
list_users,
)
# Create list of dictionaries of users.
admin_username = "******"
data_champion_username = "******"
standard_user_username = "******"
users = [
{"username": admin_username, "is_admin": True},
{"username": data_champion_username, "is_admin": False},
{"username": standard_user_username},
]
# Register the users.
register_users(users)
user_info = get_user_info(admin_username)
expected_content = {
"username": admin_username,
"is_admin": True,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}
assert user_info == expected_content
user_info = get_user_info(data_champion_username)
expected_content = {
"username": data_champion_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}
assert user_info == expected_content
user_info = get_user_info(standard_user_username)
expected_content = {
"username": standard_user_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}
assert user_info == expected_content
# Test non-existing user.
assert get_user_info("no-one") is None
# Test registering input with an existing user present.
new_username = "******"
users = [{"username": data_champion_username}, {"username": new_username}]
register_users(users)
user_info = get_user_info(new_username)
expected_content = {
"username": new_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}
assert user_info == expected_content
# Test listing users.
expected_content = [
{
"username": admin_username,
"is_admin": True,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
{
"username": data_champion_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
{
"username": standard_user_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
{
"username": new_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
]
assert list_users() == expected_content
# Test deleting users.
from dtool_lookup_server.utils import delete_users
users_to_delete = [
{
"username": standard_user_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
{
"username": new_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
]
delete_users(users_to_delete)
expected_content = [
{
"username": admin_username,
"is_admin": True,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
{
"username": data_champion_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
]
assert list_users() == expected_content
# Test updating users admin privileges.
from dtool_lookup_server.utils import update_users
users_to_update = [
{"username": admin_username}, # The is_admin value defaults to False.
{"username": data_champion_username, "is_admin": True},
{"username": standard_user_username}, # Not in system so ignored.
]
update_users(users_to_update)
expected_content = [
{
"username": admin_username,
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
{
"username": data_champion_username,
"is_admin": True,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
},
]
assert list_users() == expected_content
def tmp_app_with_data(request):
from dtool_lookup_server import create_app, mongo, sql_db
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
register_dataset,
update_permissions,
)
tmp_mongo_db_name = random_string()
config = {
"FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
"MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name),
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
"JWT_ALGORITHM": "RS256",
"JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
"JWT_TOKEN_LOCATION": "headers",
"JWT_HEADER_NAME": "Authorization",
"JWT_HEADER_TYPE": "Bearer",
}
app = create_app(config)
# Ensure the sql database has been put into the context.
app.app_context().push()
# Populate the database.
sql_db.Model.metadata.create_all(sql_db.engine)
# Register some users.
username = "******"
register_users([
dict(username=username),
dict(username="******"),
dict(username="******", is_admin=True)
])
# Add base URIs and update permissions
for base_uri in ["s3://snow-white", "s3://mr-men"]:
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": [username],
"users_with_register_permissions": [username]
}
update_permissions(permissions)
# Add some data to the database.
for base_uri in ["s3://snow-white", "s3://mr-men"]:
uuid = "af6727bf-29c7-43dd-b42f-a5d7ede28337"
uri = "{}/{}".format(base_uri, uuid)
dataset_info = {
"base_uri": base_uri,
"type": "dataset",
"uuid": uuid,
"uri": uri,
"name": "bad-apples",
"readme": {
"descripton": "apples from queen"
},
"manifest": {
"dtoolcore_version": "3.7.0",
"hash_function": "md5sum_hexdigest",
"items": {
"e4cc3a7dc281c3d89ed4553293c4b4b110dc9bf3": {
"hash": "d89117c9da2cc34586e183017cb14851",
"relpath": "U00096.3.rev.1.bt2",
"size_in_bytes": 5741810,
"utc_timestamp": 1536832115.0
}
}
},
"creator_username": "******",
"frozen_at": 1536238185.881941,
"annotations": {
"type": "fruit"
},
"tags": ["evil", "fruit"],
}
register_dataset(dataset_info)
base_uri = "s3://snow-white"
uuid = "a2218059-5bd0-4690-b090-062faf08e046"
uri = "{}/{}".format(base_uri, uuid)
dataset_info = {
"base_uri": base_uri,
"type": "dataset",
"uuid": uuid,
"uri": uri,
"name": "oranges",
"readme": {
"descripton": "oranges from queen"
},
"manifest": {
"dtoolcore_version": "3.7.0",
"hash_function": "md5sum_hexdigest",
"items": {}
},
"creator_username": "******",
"frozen_at": 1536238185.881941,
"annotations": {
"type": "fruit",
"only_here": "crazystuff"
},
"tags": ["good", "fruit"],
}
register_dataset(dataset_info)
@request.addfinalizer
def teardown():
mongo.cx.drop_database(tmp_mongo_db_name)
sql_db.session.remove()
return app.test_client()
def test_user_management_helper_functions(tmp_app): # NOQA
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
list_users,
list_base_uris,
update_permissions,
get_permission_info,
)
base_uri = "s3://snow-white"
usernames = ["snow-white", "dopey", "sleepy"]
users = [{"username": u} for u in usernames]
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": usernames,
"users_with_register_permissions": [usernames[0]],
}
register_base_uri(base_uri)
register_users(users)
update_permissions(permissions)
assert get_permission_info(base_uri) == permissions
expected_content = [{
"username":
"******",
"is_admin":
False,
"search_permissions_on_base_uris": ["s3://snow-white"],
"register_permissions_on_base_uris": ["s3://snow-white"]
}, {
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": ["s3://snow-white"],
"register_permissions_on_base_uris": []
}, {
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": ["s3://snow-white"],
"register_permissions_on_base_uris": []
}]
assert list_users() == expected_content
expected_content = [{
"base_uri":
base_uri,
"users_with_search_permissions": ["snow-white", "dopey", "sleepy"],
"users_with_register_permissions": ["snow-white"]
}]
assert list_base_uris() == expected_content
# Wipe the permissions again.
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": [],
"users_with_register_permissions": [],
}
update_permissions(permissions)
assert get_permission_info(base_uri) == permissions
expected_content = [{
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}, {
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}, {
"username": "******",
"is_admin": False,
"search_permissions_on_base_uris": [],
"register_permissions_on_base_uris": []
}]
assert list_users() == expected_content
expected_content = [{
"base_uri": base_uri,
"users_with_search_permissions": [],
"users_with_register_permissions": []
}]
assert list_base_uris() == expected_content
def tmp_app(request):
from dtool_lookup_server import create_app, mongo, sql_db
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
register_dataset,
update_permissions,
)
# Create temporary sqlite URI.
d = tempfile.mkdtemp()
sqlite_uri = randome_sqlite_uri(d)
# Create temporary mongodb name.
tmp_mongo_db_name = random_string()
config = {
"SECRET_KEY": "secret",
"FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": sqlite_uri,
"MONGO_URI": "mongodb://localhost:27017/{}".format(tmp_mongo_db_name),
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
"JWT_ALGORITHM": "RS256",
"JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
"JWT_TOKEN_LOCATION": "headers",
"JWT_HEADER_NAME": "Authorization",
"JWT_HEADER_TYPE": "Bearer",
}
app = create_app(config)
# Ensure the sql database has been put into the context.
app.app_context().push()
# Populate the database.
sql_db.Model.metadata.create_all(sql_db.engine)
# Register some users.
username = "******"
register_users([
dict(username=username),
dict(username="******"),
])
# Register base URIs and set permissions.
base_uri_1 = "s3://snow-white"
base_uri_2 = "s3://mr-men"
for base_uri in [base_uri_1, base_uri_2]:
register_base_uri(base_uri)
permissions = {
"base_uri": base_uri,
"users_with_search_permissions": [username],
"users_with_register_permissions": [],
}
update_permissions(permissions)
dataset_info = generate_dataset_info(
base_uri_1,
"blue-shirt",
{"color": "blue"}
)
register_dataset(dataset_info)
dataset_info = generate_dataset_info(
base_uri_2,
"red-wavy-shirt",
{
"color": "red",
"pattern": "wavy",
"complex_ignored": ["lists", "are", "ignored"]
}
)
register_dataset(dataset_info)
dataset_info = generate_dataset_info(
base_uri_1,
"stripy-shirt",
{
"pattern": "stripey",
"color": ["purple", "gray"] # Complex data type so ignored
}
)
register_dataset(dataset_info)
dataset_info = generate_dataset_info(
base_uri_1,
"complex-shirt",
{
"pattern": ["lies", "circles"], # Complex data type so ignored
"color": ["purple", "gray"] # Complex data type so ignored
}
)
register_dataset(dataset_info) # Whole dataset ignored by plugin.
@request.addfinalizer
def teardown():
mongo.cx.drop_database(tmp_mongo_db_name)
sql_db.session.remove()
shutil.rmtree(d)
return app.test_client()
def test_list_datasets_by_user(tmp_app): # NOQA
from dtool_lookup_server.utils import (
register_users,
register_base_uri,
update_permissions,
register_dataset_admin_metadata,
list_datasets_by_user,
)
base_uri_1 = "s3://snow-white"
base_uri_2 = "s3://mr-men"
register_base_uri(base_uri_1)
register_base_uri(base_uri_2)
username_1 = "dopey"
username_2 = "grumpy"
register_users([{"username": un} for un in (username_1, username_2)])
uuid_1 = "11111111-1111-1111-1111-111111111111"
uuid_2 = "22222222-2222-2222-2222-222222222222"
uri_1 = "{}/{}".format(base_uri_1, uuid_1)
uri_2 = "{}/{}".format(base_uri_2, uuid_2)
admin_metadata_1 = {
"base_uri": base_uri_1,
"uuid": uuid_1,
"uri": uri_1,
"name": "ds_1",
"creator_username": "******",
"frozen_at": 1536238185.881941,
"created_at": 1536236399.19497,
}
admin_metadata_2 = {
"base_uri": base_uri_2,
"uuid": uuid_2,
"uri": uri_2,
"name": "ds_2",
"creator_username": "******",
"frozen_at": 1536238185.881941,
"created_at": 1536236399.19497,
}
register_dataset_admin_metadata(admin_metadata_1)
register_dataset_admin_metadata(admin_metadata_2)
permissions_1 = {
"base_uri": base_uri_1,
"users_with_search_permissions": [username_1],
"users_with_register_permissions": []
}
permissions_2 = {
"base_uri": base_uri_2,
"users_with_search_permissions": [username_1, username_2],
"users_with_register_permissions": []
}
update_permissions(permissions_1)
update_permissions(permissions_2)
expected_content = [admin_metadata_1, admin_metadata_2]
assert list_datasets_by_user(username_1) == expected_content
expected_content = [admin_metadata_2]
assert list_datasets_by_user(username_2) == expected_content