def test_get_permission(cidc_api, clean_db, monkeypatch):
"""Check that getting a single permission by ID works as expected."""
mock_gcloud_client(monkeypatch)
current_user_id, other_user_id = setup_permissions(cidc_api, monkeypatch)
with cidc_api.app_context():
current_user_perm = Permissions.find_for_user(current_user_id)[0]
other_user_perm = Permissions.find_for_user(other_user_id)[0]
client = cidc_api.test_client()
# Check that getting a permission that doesn't exist yields 404
res = client.get("permissions/123212321")
assert res.status_code == 404
# Check that a non-admin getting another user's permission yields 404
res = client.get(f"permissions/{other_user_perm.id}")
assert res.status_code == 404
# Check that a non-admin can get their own permission
res = client.get(f"permissions/{current_user_perm.id}")
assert res.status_code == 200
assert res.json == PermissionSchema().dump(current_user_perm)
# Check that an admin can get another user's permission
make_admin(current_user_id, cidc_api)
res = client.get(f"permissions/{other_user_perm.id}")
assert res.status_code == 200
assert res.json == PermissionSchema().dump(other_user_perm)
def test_get_downloadable_file(cidc_api, clean_db, monkeypatch):
"""Check that getting a single file works as expected."""
user_id = setup_user(cidc_api, monkeypatch)
file_id_1, file_id_2 = setup_downloadable_files(cidc_api)
client = cidc_api.test_client()
# Non-admins get 401s for single files they don't have permision to view
res = client.get(f"/downloadable_files/{file_id_1}")
assert res.status_code == 401
# Give the user one permission
with cidc_api.app_context():
perm = Permissions(
granted_to_user=user_id,
trial_id=trial_id_1,
upload_type=upload_types[0],
granted_by_user=user_id,
)
perm.insert()
# Non-admins can get single files that they have permision to view
res = client.get(f"/downloadable_files/{file_id_1}")
assert res.status_code == 200
assert res.json["id"] == file_id_1
# Admins can get any file regardless of permissions
make_admin(user_id, cidc_api)
res = client.get(f"/downloadable_files/{file_id_2}")
assert res.status_code == 200
assert res.json["id"] == file_id_2
# Non-existent files yield 404
res = client.get(f"/downloadable_files/123212321")
assert res.status_code == 404
def test_permissions_revoke_all_iam_permissions(clean_db, monkeypatch):
"""
Smoke test that Permissions.revoke_all_iam_permissions calls revoke_download_access the right arguments.
"""
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
upload_types = ["wes_bam", "ihc", "rna_fastq", "plasma"]
for upload_type in upload_types:
Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type=upload_type,
granted_by_user=user.id,
).insert()
Permissions.revoke_all_iam_permissions()
gcloud_client.revoke_download_access.assert_has_calls(
[call(user.email, trial.trial_id, upload_type) for upload_type in upload_types]
)
# not called on admins or nci biobank users
gcloud_client.revoke_download_access.reset_mock()
for role in [CIDCRole.ADMIN.value, CIDCRole.NCI_BIOBANK_USER.value]:
user.role = role
user.update()
Permissions.revoke_all_iam_permissions()
gcloud_client.revoke_download_access.assert_not_called()
def create_permission(uid, assay):
Permissions(
granted_by_user=uid,
granted_to_user=uid,
trial_id=TRIAL_ID,
upload_type=assay,
).insert()
def grant_upload_permission(user_id, upload_type, cidc_api):
with cidc_api.app_context():
Permissions(
granted_by_user=user_id,
granted_to_user=user_id,
trial_id=trial_id,
upload_type=upload_type,
).insert()
def test_get_related_files(cidc_api, clean_db, monkeypatch):
"""Check that the related_files endpoint calls `get_related_files`"""
user_id = setup_user(cidc_api, monkeypatch)
file_id_1, file_id_2 = setup_downloadable_files(cidc_api)
client = cidc_api.test_client()
# Add an additional file that is related to file 1
object_url = "/foo/bar"
with cidc_api.app_context():
DownloadableFiles(
trial_id=trial_id_1,
upload_type="wes",
object_url=object_url,
facet_group=
"/wes/r2_L.fastq.gz", # this is what makes this file "related"
uploaded_timestamp=datetime.now(),
file_size_bytes=0,
).insert()
# Non-admins get 401s when requesting related files they don't have permission to view
res = client.get(f"/downloadable_files/{file_id_1}/related_files")
assert res.status_code == 401
# Give the user one permission
with cidc_api.app_context():
perm = Permissions(
granted_to_user=user_id,
trial_id=trial_id_1,
upload_type=upload_types[0],
granted_by_user=user_id,
)
perm.insert()
# Non-admins can get related files that they have permision to view
res = client.get(f"/downloadable_files/{file_id_1}/related_files")
assert res.status_code == 200
assert len(res.json["_items"]) == 1 # file 1 has 1 related file
assert res.json["_items"][0]["object_url"] == object_url
# Admins can get related files without permissions
make_admin(user_id, cidc_api)
res = client.get(f"/downloadable_files/{file_id_2}/related_files")
assert res.status_code == 200
assert len(res.json["_items"]) == 0 # file 2 has 0 related file
def test_get_download_url(cidc_api, clean_db, monkeypatch):
"""Check that generating a GCS signed URL works as expected"""
user_id = setup_user(cidc_api, monkeypatch)
file_id, _ = setup_downloadable_files(cidc_api)
client = cidc_api.test_client()
# A query missing the required parameters should yield 422
res = client.get("/downloadable_files/download_url")
assert res.status_code == 422
assert res.json["_error"]["message"]["query"]["id"] == [
"Missing data for required field."
]
# A missing file should yield 404
res = client.get("/downloadable_files/download_url?id=123212321")
assert res.status_code == 404
# No permission should also yield 404
res = client.get(f"/downloadable_files/download_url?id={file_id}")
assert res.status_code == 404
with cidc_api.app_context():
perm = Permissions(
granted_to_user=user_id,
trial_id=trial_id_1,
upload_type=upload_types[0],
granted_by_user=user_id,
)
perm.insert()
test_url = "foo"
monkeypatch.setattr("cidc_api.shared.gcloud_client.get_signed_url",
lambda *args: test_url)
res = client.get(f"/downloadable_files/download_url?id={file_id}")
assert res.status_code == 200
assert res.json == test_url
# network viewers aren't allowed to get download urls
make_role(user_id, CIDCRole.NETWORK_VIEWER.value, cidc_api)
res = client.get(f"/downloadable_files/download_url?id={file_id}")
assert res.status_code == 401
def setup_db_records(cidc_api):
extra = {"_etag": ETAG}
with cidc_api.app_context():
Users(**users["json"], **extra).insert(compute_etag=False)
TrialMetadata(**trial_metadata["json"], **extra).insert(compute_etag=False)
DownloadableFiles(**downloadable_files["json"], **extra).insert(
compute_etag=False
)
Permissions(**permissions["json"], **extra).insert(compute_etag=False)
UploadJobs(**upload_jobs["json"], **extra).insert(compute_etag=False)
def test_user_get_data_access_report(clean_db, monkeypatch):
"""Test that user data access info is collected as expected"""
mock_gcloud_client(monkeypatch)
admin_user = Users(
email="*****@*****.**",
organization="CIDC",
approval_date=datetime.now(),
role=CIDCRole.ADMIN.value,
)
admin_user.insert()
cimac_user = Users(
email="*****@*****.**",
organization="DFCI",
approval_date=datetime.now(),
role=CIDCRole.CIMAC_USER.value,
)
cimac_user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
upload_types = ["wes_bam", "ihc"]
# Note that admins don't need permissions to view data,
# so we're deliberately issuing unnecessary permissions here.
for user in [admin_user, cimac_user]:
for t in upload_types:
Permissions(
granted_to_user=user.id,
granted_by_user=admin_user.id,
trial_id=trial.trial_id,
upload_type=t,
).insert()
bio = io.BytesIO()
result_df = Users.get_data_access_report(bio)
bio.seek(0)
# Make sure bytes were written to the BytesIO instance
assert bio.getbuffer().nbytes > 0
# Make sure report data has expected info
assert set(result_df.columns) == set(
["email", "role", "organization", "trial_id", "permissions"]
)
for user in [admin_user, cimac_user]:
user_df = result_df[result_df.email == user.email]
assert set([user.role]) == set(user_df.role)
assert set([user.organization]) == set(user_df.organization)
if user == admin_user:
assert set(["*"]) == set(user_df.permissions)
else:
assert set(user_df.permissions).issubset(["wes_bam,ihc", "ihc,wes_bam"])
def create_trial(n, grant_perm=False):
trial_id = f"test-trial-{n}"
metadata_json = {
"protocol_identifier":
trial_id,
"participants": [] if n == 2 else [{
"cimac_participant_id":
"CTTTPP1",
"participant_id":
"x",
"samples": [{
"cimac_id": f"CTTTPP1SS.01",
"sample_location": "",
"type_of_primary_container": "Other",
"type_of_sample": "Other",
"collection_event_name": "",
"parent_sample_id": "",
}],
}],
"allowed_collection_event_names": [""],
"allowed_cohort_names": [],
"assays": {},
"analysis": {},
"shipments": [],
}
trial = TrialMetadata(trial_id=trial_id, metadata_json=metadata_json)
trial.insert()
if grant_perm and user_id:
Permissions(
granted_to_user=user_id,
trial_id=trial.trial_id,
upload_type="olink",
granted_by_user=user_id,
).insert()
Permissions(
granted_to_user=user_id,
trial_id=trial.trial_id,
upload_type="ihc",
granted_by_user=user_id,
).insert()
return trial.id
def refresh_download_permissions(*args):
"""
Extend the expiry date for GCS download permissions belonging to users
who accessed the system in the last 2 (or so) days. If we don't do this,
users whose accounts are still active might lose GCS download permission prematurely.
"""
active_today = lambda q: q.filter(
# Provide a 3 day window to ensure we don't miss anyone
# if, e.g., this function fails to run on a certain day.
Users._accessed
> datetime.today() - timedelta(days=3)
)
with sqlalchemy_session() as session:
active_users = Users.list(
page_size=Users.count(session=session, filter_=active_today),
session=session,
filter_=active_today,
)
for user in active_users:
print(f"Refreshing IAM download permissions for {user.email}")
Permissions.grant_iam_permissions(user, session=session)
def test_permissions_grant_iam_permissions(clean_db, monkeypatch):
"""
Smoke test that Permissions.grant_iam_permissions calls grant_download_access with the right arguments.
"""
refresh_intake_access = MagicMock()
monkeypatch.setattr(
"cidc_api.models.models.refresh_intake_access", refresh_intake_access
)
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**", role=CIDCRole.NETWORK_VIEWER.value)
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
upload_types = ["wes_bam", "ihc", "rna_fastq", "plasma"]
for upload_type in upload_types:
Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type=upload_type,
granted_by_user=user.id,
).insert()
# IAM permissions not granted to network viewers
Permissions.grant_iam_permissions(user=user)
gcloud_client.grant_download_access.assert_not_called()
# IAM permissions should be granted for any other role
user.role = CIDCRole.CIMAC_USER.value
Permissions.grant_iam_permissions(user=user)
for upload_type in upload_types:
assert (
call(user.email, trial.trial_id, upload_type)
in gcloud_client.grant_download_access.call_args_list
)
refresh_intake_access.assert_called_once_with(user.email)
def test_permissions_broad_perms(clean_db, monkeypatch):
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
other_trial = TrialMetadata(
trial_id="other-trial",
metadata_json={**METADATA, "protocol_identifier": "other-trial"},
)
other_trial.insert()
for ut in ["wes_fastq", "olink"]:
for tid in [trial.trial_id, other_trial.trial_id]:
Permissions(
granted_to_user=user.id,
trial_id=tid,
upload_type=ut,
granted_by_user=user.id,
).insert()
# Can't insert a permission for access to all trials and assays
with pytest.raises(ValueError, match="must have a trial id or upload type"):
Permissions(granted_to_user=user.id, granted_by_user=user.id).insert()
# Inserting a trial-level permission should delete other more specific related perms.
trial_query = clean_db.query(Permissions).filter(
Permissions.trial_id == trial.trial_id
)
assert trial_query.count() == 2
Permissions(
trial_id=trial.trial_id, granted_to_user=user.id, granted_by_user=user.id
).insert()
assert trial_query.count() == 1
perm = trial_query.one()
assert perm.trial_id == trial.trial_id
assert perm.upload_type is None
# Inserting an upload-level permission should delete other more specific related perms.
olink_query = clean_db.query(Permissions).filter(Permissions.upload_type == "olink")
assert olink_query.count() == 1
assert olink_query.one().trial_id == other_trial.trial_id
Permissions(
upload_type="olink", granted_to_user=user.id, granted_by_user=user.id
).insert()
assert olink_query.count() == 1
perm = olink_query.one()
assert perm.trial_id is None
assert perm.upload_type == "olink"
# Getting perms for a particular user-trial-type returns broader perms
perm = Permissions.find_for_user_trial_type(user.id, trial.trial_id, "ihc")
assert perm is not None and perm.upload_type is None
perm = Permissions.find_for_user_trial_type(user.id, "some random trial", "olink")
assert perm is not None and perm.trial_id is None
def test_list_downloadable_files(cidc_api, clean_db, monkeypatch):
"""Check that getting a list of files works as expected"""
user_id = setup_user(cidc_api, monkeypatch)
file_id_1, file_id_2 = setup_downloadable_files(cidc_api)
client = cidc_api.test_client()
# Non-admins can't get files they don't have permissions for
res = client.get("/downloadable_files")
assert res.status_code == 200
assert len(res.json["_items"]) == 0
assert res.json["_meta"]["total"] == 0
# Test a couple different permission configurations that should
# produce the same result in this case.
for kwargs in [
{
"trial_id": trial_id_1,
"upload_type": upload_types[0]
},
{
"upload_type": upload_types[0]
},
{
"trial_id": trial_id_1
},
]:
# Give the user one permission
with cidc_api.app_context():
perm = Permissions(granted_to_user=user_id,
granted_by_user=user_id,
**kwargs)
perm.insert()
# Non-admins can view files for which they have permission
res = client.get("/downloadable_files")
assert res.status_code == 200
assert len(res.json["_items"]) == 1
assert res.json["_meta"]["total"] == 1
assert res.json["_items"][0]["id"] == file_id_1
# Non-admin filter queries exclude files they aren't allowed to view
res = client.get(
f"/downloadable_files?facets=Assay Type|CyTOF|Analysis Results")
assert res.status_code == 200
assert len(res.json["_items"]) == 0
assert res.json["_meta"]["total"] == 0
# Admins and NCI biobank users can view all files regardless of their permissions
for role in [CIDCRole.ADMIN.value, CIDCRole.NCI_BIOBANK_USER.value]:
make_role(user_id, role, cidc_api)
res = client.get("/downloadable_files")
assert res.status_code == 200
assert len(res.json["_items"]) == 2
assert res.json["_meta"]["total"] == 2
assert set([f["id"] for f in res.json["_items"]
]) == set([file_id_1, file_id_2])
# Admin filter queries include any files that fit the criteria
res = client.get(
f"/downloadable_files?facets=Assay Type|CyTOF|Analysis Results")
assert res.status_code == 200
assert len(res.json["_items"]) == 1
assert res.json["_meta"]["total"] == 1
assert res.json["_items"][0]["id"] == file_id_2
# Make sure it's possible to sort by file extension
res = client.get(
f"/downloadable_files?sort_field=file_ext&sort_direction=asc")
assert res.status_code == 200
assert [f["file_ext"] for f in res.json["_items"]] == ["bam", "zip"]
# Make sure it's possible to sort by data category
res = client.get(
f"/downloadable_files?sort_field=data_category&sort_direction=asc")
assert res.status_code == 200
assert [f["data_category"] for f in res.json["_items"]] == [
"CyTOF|Analysis Results",
"WES|Source",
]
def test_get_filter_facets(cidc_api, clean_db, monkeypatch):
"""Check that getting filter facets works as expected"""
user_id = setup_user(cidc_api, monkeypatch)
setup_downloadable_files(cidc_api)
client = cidc_api.test_client()
def check_facet_counts(facets, wes_count=0, cytof_count=0):
for facet, subfacets in facets["Assay Type"].items():
if not isinstance(subfacets, list):
subfacets = [subfacets]
for subfacet in subfacets:
if facet == "WES" and subfacet["label"] == "Source":
assert subfacet["count"] == wes_count
elif facet == "CyTOF" and subfacet[
"label"] == "Analysis Results":
assert subfacet["count"] == cytof_count
else:
assert subfacet["count"] == 0
# Non-admins' facets take into account their permissions
with cidc_api.app_context():
perm = Permissions(
granted_to_user=user_id,
trial_id=trial_id_1,
upload_type=upload_types[0],
granted_by_user=user_id,
)
perm.insert()
res = client.get("/downloadable_files/filter_facets")
assert res.status_code == 200
check_facet_counts(res.json["facets"], wes_count=1, cytof_count=0)
# Admins' facets include all files, regardless of permissions
make_admin(user_id, cidc_api)
res = client.get("/downloadable_files/filter_facets")
assert res.status_code == 200
assert sorted(res.json["trial_ids"], key=lambda x: x["label"]) == sorted(
[{
"label": trial_id_1,
"count": 1
}, {
"label": trial_id_2,
"count": 1
}],
key=lambda x: x["label"],
)
check_facet_counts(res.json["facets"], wes_count=1, cytof_count=1)
# Trial facets are governed by data category facets
res = client.get(
"/downloadable_files/filter_facets?facets=Assay Type|WES|Germline")
assert res.status_code == 200
assert res.json["trial_ids"] == []
check_facet_counts(res.json["facets"], wes_count=1, cytof_count=1)
# Data category facets are governed by trial facets
res = client.get(
"/downloadable_files/filter_facets?trial_ids=someothertrial")
assert res.status_code == 200
res.json["trial_ids"] == []
check_facet_counts(res.json["facets"], wes_count=0, cytof_count=0)
def test_create_compressed_batch(cidc_api, clean_db, monkeypatch):
user_id = setup_user(cidc_api, monkeypatch)
file_id_1, file_id_2 = setup_downloadable_files(cidc_api)
with cidc_api.app_context():
url_1 = DownloadableFiles.find_by_id(file_id_1).object_url
url_2 = DownloadableFiles.find_by_id(file_id_2).object_url
client = cidc_api.test_client()
url = "/downloadable_files/compressed_batch"
# A JSON body containing a file ID list must be provided
res = client.post(url)
assert res.status_code == 422
# User has no permissions, so no files should be found
short_file_list = {"file_ids": [file_id_1, file_id_2]}
res = client.post(url, json=short_file_list)
assert res.status_code == 404
# Give the user one permission
with cidc_api.app_context():
perm = Permissions(
granted_to_user=user_id,
trial_id=trial_id_1,
upload_type=upload_types[0],
granted_by_user=user_id,
)
perm.insert()
# Mock GCS client
blob = MagicMock()
bucket = MagicMock()
bucket.blob.return_value = blob
monkeypatch.setattr(
"cidc_api.resources.downloadable_files.gcloud_client._get_bucket",
lambda _: bucket,
)
signed_url = "fake/signed/url"
monkeypatch.setattr(
"cidc_api.resources.downloadable_files.gcloud_client.get_signed_url",
lambda *_: signed_url,
)
# User has one permission, s0 the endpoint should try to create
# a compressed batch file with the single file the user has
# access to in it.
res = client.post(url, json=short_file_list)
assert res.status_code == 200
assert res.json == signed_url
print(bucket.get_blob.call_args_list)
bucket.get_blob.assert_called_with(url_1)
blob.upload_from_filename.assert_called_once()
bucket.reset_mock()
blob.reset_mock()
make_admin(user_id, cidc_api)
# Admin has access to both files, but together they are too large
res = client.post(url, json=short_file_list)
assert res.status_code == 400
assert "batch too large" in res.json["_error"]["message"]
bucket.get_blob.assert_not_called()
blob.upload_from_filename.assert_not_called()
# Decrease the size of one of the files and try again
with cidc_api.app_context():
df = DownloadableFiles.find_by_id(file_id_1)
df.file_size_bytes = 1
df.update()
res = client.post(url, json=short_file_list)
assert res.status_code == 200
assert res.json == signed_url
assert call(url_1) in bucket.get_blob.call_args_list
assert call(url_2) in bucket.get_blob.call_args_list
blob.upload_from_filename.assert_called_once()
def test_get_filelist(cidc_api, clean_db, monkeypatch):
"""Check that getting a filelist.tsv works as expected"""
user_id = setup_user(cidc_api, monkeypatch)
file_id_1, file_id_2 = setup_downloadable_files(cidc_api)
client = cidc_api.test_client()
url = "/downloadable_files/filelist"
# A JSON body containing a file ID list must be provided
res = client.post(url)
assert res.status_code == 422
# User has no permissions, so no files should be found
short_file_list = {"file_ids": [file_id_1, file_id_2]}
res = client.post(url, json=short_file_list)
assert res.status_code == 404
# Give the user one permission
with cidc_api.app_context():
perm = Permissions(
granted_to_user=user_id,
trial_id=trial_id_1,
upload_type=upload_types[0],
granted_by_user=user_id,
)
perm.insert()
# User has one permission, so the filelist should contain a single file
res = client.post(url, json=short_file_list)
assert res.status_code == 200
assert "text/tsv" in res.headers["Content-Type"]
assert "filename=filelist.tsv" in res.headers["Content-Disposition"]
assert res.data.decode("utf-8") == (
f"gs://{GOOGLE_DATA_BUCKET}/{trial_id_1}/wes/.../reads_123.bam\t{trial_id_1}_wes_..._reads_123.bam\n"
)
# Admins don't need permissions to get files
make_admin(user_id, cidc_api)
res = client.post(url, json=short_file_list)
assert res.status_code == 200
assert res.data.decode("utf-8") == (
f"gs://{GOOGLE_DATA_BUCKET}/{trial_id_1}/wes/.../reads_123.bam\t{trial_id_1}_wes_..._reads_123.bam\n"
f"gs://{GOOGLE_DATA_BUCKET}/{trial_id_2}/cytof/.../analysis.zip\t{trial_id_2}_cytof_..._analysis.zip\n"
)
# Clear inserted file records
with cidc_api.app_context():
clean_db.query(DownloadableFiles).delete()
# Filelists don't get paginated
ids = []
with cidc_api.app_context():
for id in range(1000):
df = DownloadableFiles(
trial_id=trial_id_1,
object_url=str(id),
upload_type="",
file_size_bytes=0,
uploaded_timestamp=datetime.now(),
)
df.insert()
ids.append(df.id)
res = client.post(url, json={"file_ids": ids})
assert res.status_code == 200
# newly inserted files + EOF newline
assert len(res.data.decode("utf-8").split("\n")) == len(ids) + 1
def assert_sqla_matching_fields(rec1, rec2):
received = rec1.__dict__
expected = rec2.__dict__
received.pop("_sa_instance_state")
expected.pop("_sa_instance_state")
assert received == expected
perm_json = {
"upload_type": "olink",
"granted_by_user": 1,
"granted_to_user": 2,
"trial_id": "sometrial",
}
perm_record = Permissions(**perm_json)
def test_unmarshal_request(empty_app: Flask):
"""Check that unmarshal_request validates and loads request JSON as expected."""
s, p, a = 1, 2, 3
@unmarshal_request(PermissionSchema(), "permission_record", load_sqla=True)
def endpoint_sqla(some, positional, args, permission_record):
assert some == s and positional == p and args == a
return permission_record
@unmarshal_request(PermissionSchema(),
"permission_record",
load_sqla=False)
def endpoint_json(some, positional, args, permission_record):
def test_list_trials(cidc_api, clean_db, monkeypatch):
"""Check that listing trials works as expected"""
mock_gcloud_client(monkeypatch)
user_id = setup_user(cidc_api, monkeypatch)
trial_1, trial_2 = setup_trial_metadata(cidc_api, user_id)
client = cidc_api.test_client()
# A CIMAC user can list trials that they're allowed to see via
# granular permissions
res = client.get("/trial_metadata")
assert res.status_code == 200
assert len(res.json["_items"]) == 1
assert res.json["_items"][0]["id"] == trial_1
assert "file_bundle" not in res.json["_items"][0]
assert "num_participants" not in res.json["_items"][0]
assert "num_samples" not in res.json["_items"][0]
# A CIMAC user with a cross-trial permission can list all trials
with cidc_api.app_context():
Permissions(granted_by_user=user_id,
granted_to_user=user_id,
upload_type="ihc").insert()
res = client.get("/trial_metadata")
assert res.status_code == 200
assert len(res.json["_items"]) == 2
# Allowed users can get all trials
for role in trial_modifier_roles:
make_role(user_id, role, cidc_api)
res = client.get("/trial_metadata")
assert res.status_code == 200
assert len(res.json["_items"]) == 2
assert res.json["_meta"]["total"] == 2
assert set([t["id"]
for t in res.json["_items"]]) == set([trial_1, trial_2])
assert not any("file_bundle" in t for t in res.json["_items"])
# Passing the URL param include_file_bundles=true works on an
# as-available basis - if trials have no files associated with them,
# they won't have a file bundle in the response
res = client.get("/trial_metadata?include_file_bundles=true")
assert res.status_code == 200
assert len(res.json["_items"]) == 2
assert "file_bundle" not in res.json["_items"][0]
# Add some files...
with cidc_api.app_context():
# for trial 1
for id, (type, facet_group) in enumerate([
("cytof_10021_9204", "/cytof_10021_9204/spike_in.fcs"),
("cytof_10021_9204", "/cytof_10021_9204/source_.fcs"),
(
"cytof_10021_9204",
"/cytof_analysis/combined_cell_counts_profiling.csv",
),
("wes", "/wes/r1_L.fastq.gz"),
]):
DownloadableFiles(
id=id,
trial_id="test-trial-1",
facet_group=facet_group,
object_url=f"test-trial-1/{facet_group}",
upload_type=type,
file_size_bytes=0,
uploaded_timestamp=datetime.now(),
).insert()
# for trial 2
for id_minus_4, (type, facet_group) in enumerate([
("participants info", "csv|participants info"),
("mif", "/mif/roi_/cell_seg_data.txt"),
]):
DownloadableFiles(
id=id_minus_4 + 4,
trial_id="test-trial-2",
facet_group=facet_group,
object_url=f"test-trial-2/{facet_group}",
upload_type=type,
file_size_bytes=0,
uploaded_timestamp=datetime.now(),
).insert()
# Listing trials with populated file bundles (also, check that sorting and counting participants/samples works)
res = client.get(
"/trial_metadata?include_file_bundles=true&include_counts=true&sort_field=trial_id&sort_direction=asc"
)
assert res.status_code == 200
assert len(res.json["_items"]) == 2
assert res.json["_items"][0]
[trial_json_1, trial_json_2] = res.json["_items"]
assert set(trial_json_1["file_bundle"]["CyTOF"]["source"]) == set([0, 1])
assert trial_json_1["file_bundle"]["CyTOF"]["analysis"] == [2]
assert trial_json_1["file_bundle"]["WES"]["source"] == [3]
assert trial_json_1["num_samples"] == 1
assert trial_json_1["num_participants"] == 1
assert trial_json_2["file_bundle"]["Participants Info"]["clinical"] == [4]
assert trial_json_2["file_bundle"]["mIF"]["analysis"] == [5]
assert trial_json_2["num_samples"] == 0
assert trial_json_2["num_participants"] == 0
# Filtering by trial id seems to work when file bundles are included
res = client.get(
"/trial_metadata?include_file_bundles=true&trial_ids=test-trial-1")
assert res.status_code == 200
assert len(res.json["_items"]) == 1
assert res.json["_items"][0]["trial_id"] == "test-trial-1"
# Pagination seems to work when file bundles are included
res = client.get("/trial_metadata?include_file_bundles=true&page_size=1")
assert res.status_code == 200
assert len(res.json["_items"]) == 1
# Metadata blobs are pruned as expected
res = client.get("/trial_metadata")
assert res.status_code == 200
metadata_json = res.json["_items"][0]["metadata_json"]
assert metadata_json.get("participants") is None
assert metadata_json.get("assays") is None
assert metadata_json.get("analysis") is None
assert metadata_json.get("shipments") is None
def test_permissions_delete(clean_db, monkeypatch, caplog):
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=user.id,
)
perm.insert()
# Deleting a record by a user doesn't exist leads to an error
gcloud_client.reset_mocks()
with pytest.raises(NoResultFound, match="no user with id"):
perm.delete(deleted_by=999999)
# Deletion of an existing permission leads to no error
gcloud_client.reset_mocks()
with caplog.at_level(logging.DEBUG):
perm.delete(deleted_by=user.id)
gcloud_client.revoke_download_access.assert_called_once()
gcloud_client.grant_download_access.assert_not_called()
assert any(
log_record.message.strip()
== f"admin-action: {user.email} removed from {user.email} the permission wes_bam on {trial.trial_id}"
for log_record in caplog.records
)
# Deleting an already-deleted record is idempotent
gcloud_client.reset_mocks()
perm.delete(deleted_by=user)
gcloud_client.revoke_download_access.assert_called_once()
gcloud_client.grant_download_access.assert_not_called()
# Deleting a record whose user doesn't exist leads to an error
gcloud_client.reset_mocks()
with pytest.raises(NoResultFound, match="no user with id"):
Permissions(granted_to_user=999999).delete(deleted_by=user)
gcloud_client.revoke_download_access.assert_not_called()
gcloud_client.grant_download_access.assert_not_called()
# If revoking a permission from a "network-viewer", no GCS IAM actions are taken
gcloud_client.revoke_download_access.reset_mock()
user.role = CIDCRole.NETWORK_VIEWER.value
user.update()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="ihc",
granted_by_user=user.id,
)
perm.insert()
perm.delete(deleted_by=user)
gcloud_client.revoke_download_access.assert_not_called()
def test_permissions_insert(clean_db, monkeypatch, caplog):
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
_insert = MagicMock()
monkeypatch.setattr(CommonColumns, "insert", _insert)
# if upload_type is invalid
with pytest.raises(ValueError, match="invalid upload type"):
Permissions(upload_type="foo", granted_to_user=user.id, trial_id=trial.trial_id)
# if don't give granted_by_user
perm = Permissions(
granted_to_user=user.id, trial_id=trial.trial_id, upload_type="wes_bam"
)
with pytest.raises(IntegrityError, match="`granted_by_user` user must be given"):
perm.insert()
_insert.assert_not_called()
# if give bad granted_by_user
_insert.reset_mock()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=999999,
)
with pytest.raises(IntegrityError, match="`granted_by_user` user must exist"):
perm.insert()
_insert.assert_not_called()
# if give bad granted_to_user
_insert.reset_mock()
perm = Permissions(
granted_to_user=999999,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=user.id,
)
with pytest.raises(IntegrityError, match="`granted_to_user` user must exist"):
perm.insert()
_insert.assert_not_called()
# This one will work
_insert.reset_mock()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=user.id,
)
with caplog.at_level(logging.DEBUG):
perm.insert()
_insert.assert_called_once()
assert any(
log_record.message.strip()
== f"admin-action: {user.email} gave {user.email} the permission wes_bam on {trial.trial_id}"
for log_record in caplog.records
)
gcloud_client.grant_download_access.assert_called_once()
# If granting a permission to a "network-viewer", no GCS IAM actions are taken
_insert.reset_mock()
gcloud_client.grant_download_access.reset_mock()
user.role = CIDCRole.NETWORK_VIEWER.value
user.update()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="ihc",
granted_by_user=user.id,
)
perm.insert()
_insert.assert_called_once()
gcloud_client.grant_download_access.assert_not_called()
def test_create_permission(cidc_api, clean_db, monkeypatch):
"""Check that creating a new permission works as expected."""
gcloud_client = mock_gcloud_client(monkeypatch)
current_user_id, other_user_id = setup_permissions(cidc_api, monkeypatch)
client = cidc_api.test_client()
# Non-admins should be blocked from posting to this endpoint
gcloud_client.reset_mocks()
res = client.post("permissions")
assert res.status_code == 401
assert "not authorized to access this endpoint" in res.json["_error"][
"message"]
gcloud_client.grant_download_access.assert_not_called()
gcloud_client.revoke_download_access.assert_not_called()
make_admin(current_user_id, cidc_api)
perm = {
"granted_to_user": other_user_id,
"trial_id": TRIAL_ID,
"upload_type": "ihc",
}
# When an IAM grant error occurs, the permission db record shouldn't be created
gcloud_client.reset_mocks()
gcloud_client.grant_download_access.side_effect = Exception("oops")
res = client.post("permissions", json=perm)
assert "IAM grant failed" in res.json["_error"]["message"]
assert res.status_code == 500
with cidc_api.app_context():
assert clean_db.query(Permissions).filter_by(**perm).all() == []
gcloud_client.grant_download_access.side_effect = None
# Admins can't create permissions with invalid upload types
gcloud_client.reset_mocks()
res = client.post("permissions", json={**perm, "upload_type": "foo"})
assert res.status_code == 422
assert "invalid upload type: foo" in res.json["_error"]["message"]
# Admins should be able to create new permissions
gcloud_client.reset_mocks()
res = client.post("permissions", json=perm)
assert res.status_code == 201
assert "id" in res.json
assert {**res.json, **perm} == res.json
with cidc_api.app_context():
assert Permissions.find_by_id(res.json["id"])
gcloud_client.grant_download_access.assert_called_once()
gcloud_client.revoke_download_access.assert_not_called()
# Re-insertion is not allowed
gcloud_client.reset_mocks()
res = client.post("permissions", json=perm)
assert res.status_code == 400
assert "unique constraint" in res.json["_error"]["message"]
gcloud_client.grant_download_access.assert_not_called()
gcloud_client.revoke_download_access.assert_not_called()
# The permission grantee must exist
gcloud_client.reset_mocks()
perm["granted_to_user"] = 999999999 # user doesn't exist
res = client.post("permissions", json=perm)
assert res.status_code == 400
assert "user must exist, but no user found" in res.json["_error"][
"message"]
gcloud_client.grant_download_access.assert_not_called()
gcloud_client.revoke_download_access.assert_not_called()
with cidc_api.app_context():
clean_db.query(Permissions).delete()
clean_db.commit()
# The permission grantee must have <= GOOGLE_MAX_DOWNLOAD_PERMISSIONS
perm["granted_to_user"] = current_user_id
inserts_fail_eventually = False
upload_types = list(ALL_UPLOAD_TYPES)
for i in range(GOOGLE_MAX_DOWNLOAD_PERMISSIONS + 1):
gcloud_client.reset_mocks()
perm["upload_type"] = upload_types[i]
res = client.post("permissions", json=perm)
if res.status_code != 201:
assert res.status_code == 400
assert (
"greater than or equal to the maximum number of allowed granular permissions"
in res.json["_error"]["message"])
gcloud_client.grant_download_access.assert_not_called()
gcloud_client.revoke_download_access.assert_not_called()
inserts_fail_eventually = True
break
assert inserts_fail_eventually
def test_delete_permission(cidc_api, clean_db, monkeypatch):
"""Check that deleting a permission works as expected."""
gcloud_client = mock_gcloud_client(monkeypatch)
current_user_id, other_user_id = setup_permissions(cidc_api, monkeypatch)
with cidc_api.app_context():
perm = Permissions.find_for_user(current_user_id)[0]
client = cidc_api.test_client()
# Non-admins are not allowed to delete
gcloud_client.reset_mock()
res = client.delete(f"permissions/{perm.id}")
assert res.status_code == 401
assert "not authorized to access this endpoint" in res.json["_error"][
"message"]
gcloud_client.grant_download_access.assert_not_called()
gcloud_client.revoke_download_access.assert_not_called()
make_admin(current_user_id, cidc_api)
# Requester must supply an If-Match header
gcloud_client.reset_mock()
res = client.delete(f"permissions/{perm.id}")
assert res.status_code == 428
gcloud_client.grant_download_access.assert_not_called()
gcloud_client.revoke_download_access.assert_not_called()
headers = {"If-Match": "foobar"}
# Returns NotFound if no record exists
gcloud_client.reset_mock()
res = client.delete(f"permissions/1232123", headers=headers)
assert res.status_code == 404
gcloud_client.grant_download_access.assert_not_called()
gcloud_client.revoke_download_access.assert_not_called()
# A mismatched ETag leads to a PreconditionFailed error
gcloud_client.reset_mock()
res = client.delete(f"permissions/{perm.id}", headers=headers)
assert res.status_code == 412
gcloud_client.grant_download_access.assert_not_called()
gcloud_client.revoke_download_access.assert_not_called()
headers["If-Match"] = perm._etag
# A well-formed delete request fails if IAM revoke fails
gcloud_client.reset_mock()
gcloud_client.revoke_download_access.side_effect = Exception("oops")
res = client.delete(f"permissions/{perm.id}", headers=headers)
assert "IAM revoke failed" in res.json["_error"]["message"]
assert res.status_code == 500
with cidc_api.app_context():
assert Permissions.find_by_id(perm.id) is not None
gcloud_client.revoke_download_access.side_effect = None
# A matching ETag leads to a successful deletion
gcloud_client.reset_mock()
res = client.delete(f"permissions/{perm.id}", headers=headers)
assert res.status_code == 204
with cidc_api.app_context():
assert Permissions.find_by_id(perm.id) is None
gcloud_client.grant_download_access.assert_not_called()
gcloud_client.revoke_download_access.assert_called_once()