def test_permissions_revoke_all_iam_permissions(clean_db, monkeypatch):
"""
Smoke test that Permissions.revoke_all_iam_permissions calls revoke_download_access the right arguments.
"""
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
upload_types = ["wes_bam", "ihc", "rna_fastq", "plasma"]
for upload_type in upload_types:
Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type=upload_type,
granted_by_user=user.id,
).insert()
Permissions.revoke_all_iam_permissions()
gcloud_client.revoke_download_access.assert_has_calls(
[call(user.email, trial.trial_id, upload_type) for upload_type in upload_types]
)
# not called on admins or nci biobank users
gcloud_client.revoke_download_access.reset_mock()
for role in [CIDCRole.ADMIN.value, CIDCRole.NCI_BIOBANK_USER.value]:
user.role = role
user.update()
Permissions.revoke_all_iam_permissions()
gcloud_client.revoke_download_access.assert_not_called()
def mock_admin_user(cidc_api, monkeypatch) -> int:
user = Users(**{**users["json"], "email": "*****@*****.**", "id": None})
mock_current_user(user, monkeypatch)
with cidc_api.app_context():
user.insert()
return user.id
def setup_trial_and_user(cidc_api, monkeypatch) -> int:
"""
Insert a trial and a cimac-user into the database, and set the user
as the current user.
"""
# this is necessary for adding/removing permissions from this user
# without trying to contact GCP
mock_gcloud_client(monkeypatch)
user = Users(email=user_email,
role=CIDCRole.CIMAC_USER.value,
approval_date=datetime.now())
mock_current_user(user, monkeypatch)
with cidc_api.app_context():
TrialMetadata(
trial_id="test_trial",
metadata_json={
prism.PROTOCOL_ID_FIELD_NAME: trial_id,
"participants": [],
"allowed_cohort_names": ["Arm_Z"],
"allowed_collection_event_names": [],
},
).insert()
user.insert()
return user.id
def test_poll_upload_merge_status(cidc_api, clean_db, monkeypatch):
"""
Check pull_upload_merge_status endpoint behavior
"""
user_id = setup_trial_and_user(cidc_api, monkeypatch)
with cidc_api.app_context():
user = Users.find_by_id(user_id)
make_cimac_biofx_user(user_id, cidc_api)
metadata = {PROTOCOL_ID_FIELD_NAME: trial_id}
with cidc_api.app_context():
other_user = Users(email="*****@*****.**")
other_user.insert()
upload_job = UploadJobs.create(
upload_type="wes",
uploader_email=user.email,
gcs_file_map={},
metadata=metadata,
gcs_xlsx_uri="",
)
upload_job.insert()
upload_job_id = upload_job.id
client = cidc_api.test_client()
# Upload not found
res = client.get(
f"/ingestion/poll_upload_merge_status/12345?token={upload_job.token}")
assert res.status_code == 404
upload_job_url = (
f"/ingestion/poll_upload_merge_status/{upload_job_id}?token={upload_job.token}"
)
# Upload not-yet-ready
res = client.get(upload_job_url)
assert res.status_code == 200
assert "retry_in" in res.json and res.json["retry_in"] == 5
assert "status" not in res.json
test_details = "A human-friendly reason for this "
for status in [
UploadJobStatus.MERGE_COMPLETED.value,
UploadJobStatus.MERGE_FAILED.value,
]:
# Simulate cloud function merge status update
with cidc_api.app_context():
upload_job._set_status_no_validation(status)
upload_job.status_details = test_details
upload_job.update()
# Upload ready
res = client.get(upload_job_url)
assert res.status_code == 200
assert "retry_in" not in res.json
assert "status" in res.json and res.json["status"] == status
assert ("status_details" in res.json
and res.json["status_details"] == test_details)
def test_permissions_delete(clean_db, monkeypatch, caplog):
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=user.id,
)
perm.insert()
# Deleting a record by a user doesn't exist leads to an error
gcloud_client.reset_mocks()
with pytest.raises(NoResultFound, match="no user with id"):
perm.delete(deleted_by=999999)
# Deletion of an existing permission leads to no error
gcloud_client.reset_mocks()
with caplog.at_level(logging.DEBUG):
perm.delete(deleted_by=user.id)
gcloud_client.revoke_download_access.assert_called_once()
gcloud_client.grant_download_access.assert_not_called()
assert any(
log_record.message.strip()
== f"admin-action: {user.email} removed from {user.email} the permission wes_bam on {trial.trial_id}"
for log_record in caplog.records
)
# Deleting an already-deleted record is idempotent
gcloud_client.reset_mocks()
perm.delete(deleted_by=user)
gcloud_client.revoke_download_access.assert_called_once()
gcloud_client.grant_download_access.assert_not_called()
# Deleting a record whose user doesn't exist leads to an error
gcloud_client.reset_mocks()
with pytest.raises(NoResultFound, match="no user with id"):
Permissions(granted_to_user=999999).delete(deleted_by=user)
gcloud_client.revoke_download_access.assert_not_called()
gcloud_client.grant_download_access.assert_not_called()
# If revoking a permission from a "network-viewer", no GCS IAM actions are taken
gcloud_client.revoke_download_access.reset_mock()
user.role = CIDCRole.NETWORK_VIEWER.value
user.update()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="ihc",
granted_by_user=user.id,
)
perm.insert()
perm.delete(deleted_by=user)
gcloud_client.revoke_download_access.assert_not_called()
def test_user_get_data_access_report(clean_db, monkeypatch):
"""Test that user data access info is collected as expected"""
mock_gcloud_client(monkeypatch)
admin_user = Users(
email="*****@*****.**",
organization="CIDC",
approval_date=datetime.now(),
role=CIDCRole.ADMIN.value,
)
admin_user.insert()
cimac_user = Users(
email="*****@*****.**",
organization="DFCI",
approval_date=datetime.now(),
role=CIDCRole.CIMAC_USER.value,
)
cimac_user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
upload_types = ["wes_bam", "ihc"]
# Note that admins don't need permissions to view data,
# so we're deliberately issuing unnecessary permissions here.
for user in [admin_user, cimac_user]:
for t in upload_types:
Permissions(
granted_to_user=user.id,
granted_by_user=admin_user.id,
trial_id=trial.trial_id,
upload_type=t,
).insert()
bio = io.BytesIO()
result_df = Users.get_data_access_report(bio)
bio.seek(0)
# Make sure bytes were written to the BytesIO instance
assert bio.getbuffer().nbytes > 0
# Make sure report data has expected info
assert set(result_df.columns) == set(
["email", "role", "organization", "trial_id", "permissions"]
)
for user in [admin_user, cimac_user]:
user_df = result_df[result_df.email == user.email]
assert set([user.role]) == set(user_df.role)
assert set([user.organization]) == set(user_df.organization)
if user == admin_user:
assert set(["*"]) == set(user_df.permissions)
else:
assert set(user_df.permissions).issubset(["wes_bam,ihc", "ihc,wes_bam"])
def setup_user(cidc_api, monkeypatch) -> int:
current_user = Users(
email="*****@*****.**",
role=CIDCRole.CIMAC_USER.value,
approval_date=datetime.now(),
)
mock_current_user(current_user, monkeypatch)
with cidc_api.app_context():
current_user.insert()
return current_user.id
def test_permissions_broad_perms(clean_db, monkeypatch):
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
other_trial = TrialMetadata(
trial_id="other-trial",
metadata_json={**METADATA, "protocol_identifier": "other-trial"},
)
other_trial.insert()
for ut in ["wes_fastq", "olink"]:
for tid in [trial.trial_id, other_trial.trial_id]:
Permissions(
granted_to_user=user.id,
trial_id=tid,
upload_type=ut,
granted_by_user=user.id,
).insert()
# Can't insert a permission for access to all trials and assays
with pytest.raises(ValueError, match="must have a trial id or upload type"):
Permissions(granted_to_user=user.id, granted_by_user=user.id).insert()
# Inserting a trial-level permission should delete other more specific related perms.
trial_query = clean_db.query(Permissions).filter(
Permissions.trial_id == trial.trial_id
)
assert trial_query.count() == 2
Permissions(
trial_id=trial.trial_id, granted_to_user=user.id, granted_by_user=user.id
).insert()
assert trial_query.count() == 1
perm = trial_query.one()
assert perm.trial_id == trial.trial_id
assert perm.upload_type is None
# Inserting an upload-level permission should delete other more specific related perms.
olink_query = clean_db.query(Permissions).filter(Permissions.upload_type == "olink")
assert olink_query.count() == 1
assert olink_query.one().trial_id == other_trial.trial_id
Permissions(
upload_type="olink", granted_to_user=user.id, granted_by_user=user.id
).insert()
assert olink_query.count() == 1
perm = olink_query.one()
assert perm.trial_id is None
assert perm.upload_type == "olink"
# Getting perms for a particular user-trial-type returns broader perms
perm = Permissions.find_for_user_trial_type(user.id, trial.trial_id, "ihc")
assert perm is not None and perm.upload_type is None
perm = Permissions.find_for_user_trial_type(user.id, "some random trial", "olink")
assert perm is not None and perm.trial_id is None
def setup_user(cidc_api, monkeypatch) -> int:
# this is necessary for adding/removing permissions from this user
# without trying to contact GCP
mock_gcloud_client(monkeypatch)
current_user = Users(
email="*****@*****.**",
role=CIDCRole.CIMAC_USER.value,
approval_date=datetime.now(),
)
mock_current_user(current_user, monkeypatch)
with cidc_api.app_context():
current_user.insert()
return current_user.id
def test_common_insert(clean_db):
"""Test insert, inherited from CommonColumns"""
# Check disabling committing
u1 = Users(email="a")
u1.insert(commit=False)
assert not u1.id
# Insert a new record without disabling committing
u2 = Users(email="b")
u2.insert()
assert u1.id and u1._etag
assert u2.id and u2._etag
assert u1._etag != u2._etag
assert Users.find_by_id(u1.id)
assert Users.find_by_id(u2.id)
def setup_permissions(cidc_api, monkeypatch) -> Tuple[int, int]:
"""
Create two users, one trial, and three permissions in `db`.
Two permissions will belong to the first user, and the third will
belong to the second one. Returns the first and second user ids
as a tuple.
"""
current_user = Users(
id=1,
email="*****@*****.**",
role=CIDCRole.CIMAC_USER.value,
approval_date=datetime.now(),
)
other_user = Users(id=2, email="*****@*****.**")
mock_current_user(current_user, monkeypatch)
with cidc_api.app_context():
# Create users
current_user.insert()
other_user.insert()
# Create trial
TrialMetadata.create(
TRIAL_ID,
{
"protocol_identifier": TRIAL_ID,
"allowed_collection_event_names": [],
"allowed_cohort_names": [],
"participants": [],
},
)
# Create permissions
def create_permission(uid, assay):
Permissions(
granted_by_user=uid,
granted_to_user=uid,
trial_id=TRIAL_ID,
upload_type=assay,
).insert()
create_permission(current_user.id, "ihc")
create_permission(current_user.id, "olink")
create_permission(other_user.id, "olink")
return current_user.id, other_user.id
def test_user_confirm_approval(clean_db, monkeypatch):
"""Ensure that users are notified when their account goes from pending to approved."""
confirm_account_approval = MagicMock()
monkeypatch.setattr(
"cidc_api.shared.emails.confirm_account_approval", confirm_account_approval
)
user = Users(email="*****@*****.**")
user.insert()
# The confirmation email shouldn't be sent for updates unrelated to account approval
user.update(changes={"first_n": "foo"})
confirm_account_approval.assert_not_called()
# The confirmation email should be sent for updates related to account approval
user.update(changes={"approval_date": datetime.now()})
confirm_account_approval.assert_called_once_with(user, send_email=True)
def setup_upload_jobs(cidc_api) -> Tuple[int, int]:
"""
Insert two uploads into the database created by different users
and return their IDs.
"""
with cidc_api.app_context():
other_user = Users(email="*****@*****.**")
other_user.insert()
job1 = UploadJobs(
uploader_email=user_email,
trial_id=trial_id,
status=UploadJobStatus.STARTED.value,
metadata_patch={
"test": {
"upload_placeholder": "baz"
},
"test2": "foo"
},
upload_type="",
gcs_xlsx_uri="",
gcs_file_map={"bip": "baz"},
multifile=False,
)
job2 = UploadJobs(
uploader_email=other_user.email,
trial_id=trial_id,
status=UploadJobStatus.STARTED.value,
metadata_patch={
"array": [{
"upload_placeholder": "baz"
}, {
"test2": "foo"
}]
},
upload_type="",
gcs_xlsx_uri="",
gcs_file_map={"bip": "baz"},
multifile=False,
)
job1.insert()
job2.insert()
return job1.id, job2.id
def setup_users(cidc_api, monkeypatch, registered=True) -> Tuple[int, int]:
"""
Insert two users into the database. If `registered=False`, don't
register the first user.
"""
current_user = Users(id=1, email="*****@*****.**")
other_user = Users(id=2, email="*****@*****.**")
mock_current_user(current_user, monkeypatch)
with cidc_api.app_context():
if registered:
current_user.role = CIDCRole.CIMAC_USER.value
current_user.approval_date = datetime.now()
current_user.insert()
other_user.insert()
return current_user.id, other_user.id
def test_common_delete(clean_db):
"""Test delete, inherited from CommonColumns"""
user1 = Users(email="foo")
user2 = Users(email="bar")
# Try to delete an uninserted record
with pytest.raises(InvalidRequestError):
user1.delete()
user1.insert()
user2.insert()
# Defer a deletion with commit=False
user1.delete(commit=False)
assert Users.find_by_id(user1.id)
# Delete with auto-commit
user2.delete()
assert not Users.find_by_id(user1.id)
assert not Users.find_by_id(user2.id)
def test_common_update(clean_db):
"""Test update, inherited from CommonColumns"""
email = "foo"
user = Users(id=1, email=email)
# Record not found
with pytest.raises(NoResultFound):
user.update()
user.insert()
_updated = user._updated
# Update via setattr and changes
first_n = "hello"
last_n = "goodbye"
user.last_n = last_n
user.update(changes={"first_n": first_n})
user = Users.find_by_id(user.id)
assert user._updated > _updated
assert user.first_n == first_n
assert user.last_n == last_n
_updated = user._updated
_etag = user._etag
# Make sure you can clear a field to null
user.update(changes={"first_n": None})
user = Users.find_by_id(user.id)
assert user._updated > _updated
assert _etag != user._etag
assert user.first_n is None
_updated = user._updated
_etag = user._etag
# Make sure etags don't change if public fields don't change
user.update()
user = Users.find_by_id(user.id)
assert user._updated > _updated
assert _etag == user._etag
def test_permissions_grant_iam_permissions(clean_db, monkeypatch):
"""
Smoke test that Permissions.grant_iam_permissions calls grant_download_access with the right arguments.
"""
refresh_intake_access = MagicMock()
monkeypatch.setattr(
"cidc_api.models.models.refresh_intake_access", refresh_intake_access
)
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**", role=CIDCRole.NETWORK_VIEWER.value)
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
upload_types = ["wes_bam", "ihc", "rna_fastq", "plasma"]
for upload_type in upload_types:
Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type=upload_type,
granted_by_user=user.id,
).insert()
# IAM permissions not granted to network viewers
Permissions.grant_iam_permissions(user=user)
gcloud_client.grant_download_access.assert_not_called()
# IAM permissions should be granted for any other role
user.role = CIDCRole.CIMAC_USER.value
Permissions.grant_iam_permissions(user=user)
for upload_type in upload_types:
assert (
call(user.email, trial.trial_id, upload_type)
in gcloud_client.grant_download_access.call_args_list
)
refresh_intake_access.assert_called_once_with(user.email)
def setup_data(cidc_api, clean_db):
user = Users(email="*****@*****.**", approval_date=datetime.now())
shipment = {
"courier": "FEDEX",
"ship_to": "",
"ship_from": "",
"assay_type": assay_type,
"manifest_id": manifest_id,
"date_shipped": "2020-06-10 00:00:00",
"date_received": "2020-06-11 00:00:00",
"account_number": "",
"assay_priority": "1",
"receiving_party": "MSSM_Rahman",
"tracking_number": "",
"shipping_condition": "Frozen_Dry_Ice",
"quality_of_shipment": "Specimen shipment received in good condition",
}
metadata = {
"protocol_identifier":
trial_id,
"shipments": [
# we get duplicate shipment uploads sometimes
shipment,
shipment,
],
"participants": [{
"cimac_participant_id":
f"CTTTPP{p}",
"participant_id":
"x",
"cohort_name":
"",
"samples": [{
"cimac_id": f"CTTTPP{p}SS.0{s}",
"sample_location": "",
"type_of_primary_container": "Other",
"type_of_sample": "Other",
"collection_event_name": "",
"parent_sample_id": "",
} for s in range(num_samples[p])],
} for p in range(num_participants)],
"allowed_cohort_names": [""],
"allowed_collection_event_names": [""],
}
trial = TrialMetadata(trial_id=trial_id, metadata_json=metadata)
upload_job = UploadJobs(
uploader_email=user.email,
trial_id=trial.trial_id,
upload_type="pbmc",
gcs_xlsx_uri="",
metadata_patch=metadata,
multifile=False,
)
upload_job._set_status_no_validation(UploadJobStatus.MERGE_COMPLETED.value)
with cidc_api.app_context():
user.insert()
trial.insert()
upload_job.insert()
clean_db.refresh(user)
clean_db.refresh(upload_job)
clean_db.refresh(trial)
return user, upload_job, trial
def test_authorize(cidc_api, clean_db):
"""Check that authorization works as expected."""
user = Users(**PAYLOAD)
with cidc_api.app_context():
# Unregistered user should not be authorized to do anything to any resource except "users"
with pytest.raises(Unauthorized, match="not registered"):
auth.authorize(user, [], "some-resource", "some-http-method")
# We can't track accesses for users who aren't registered
assert user._accessed is None
# Unregistered user should not be able to GET users
with pytest.raises(Unauthorized, match="not registered"):
auth.authorize(user, [], "users", "GET")
assert user._accessed is None
# Unregistered user should not be able to GET self
with pytest.raises(Unauthorized, match="not registered"):
auth.authorize(user, [], "self", "GET")
assert user._accessed is None
# Unregistered user should be able to POST users
assert auth.authorize(user, [], "self", "POST")
# Add the user to the db but don't approve yet
user.insert()
# Unapproved user isn't authorized to do anything
with pytest.raises(Unauthorized, match="pending approval"):
auth.authorize(user, [], "self", "POST")
# Check that we tracked this user's last access
assert user._accessed.date() == date.today()
_accessed = user._accessed
# Ensure unapproved user can access their own data
assert auth.authorize(user, [], "self", "GET")
# Give the user a role but don't approve them
user.role = CIDCRole.CIMAC_USER.value
user.update()
# Unapproved user *with an authorized role* still shouldn't be authorized
with pytest.raises(Unauthorized, match="pending approval"):
auth.authorize(user, [CIDCRole.CIMAC_USER.value], "self", "POST")
# Approve the user
user.approval_date = datetime.now()
user.update()
# If user doesn't have required role, they should not be authorized.
with pytest.raises(Unauthorized, match="not authorized to access"):
auth.authorize(user, [CIDCRole.ADMIN.value], "some-resource",
"some-http-method")
# If user has an allowed role, they should be authorized
assert auth.authorize(user, [CIDCRole.CIMAC_USER.value],
"some-resource", "some-http-method")
# If the resource has no role restrictions, they should be authorized
assert auth.authorize(user, [], "some-resource", "some-http-method")
# Disable user
user.disabled = True
user.update()
# If user has an allowed role but is disabled, they should be unauthorized
with pytest.raises(Unauthorized, match="disabled"):
auth.authorize(user, [CIDCRole.CIMAC_USER.value], "some-resource",
"some-http-method")
# Ensure unapproved user can access their own data
assert auth.authorize(user, [], "self", "GET")
# If the resource has no role restrictions, they should be still unauthorized
with pytest.raises(Unauthorized, match="disabled"):
auth.authorize(user, [], "some-resource", "some-http-method")
# Check that user's last access wasn't updated by all activity,
# since it occurred on the same day as previous accesses
assert user._accessed == _accessed
def test_permissions_insert(clean_db, monkeypatch, caplog):
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
_insert = MagicMock()
monkeypatch.setattr(CommonColumns, "insert", _insert)
# if upload_type is invalid
with pytest.raises(ValueError, match="invalid upload type"):
Permissions(upload_type="foo", granted_to_user=user.id, trial_id=trial.trial_id)
# if don't give granted_by_user
perm = Permissions(
granted_to_user=user.id, trial_id=trial.trial_id, upload_type="wes_bam"
)
with pytest.raises(IntegrityError, match="`granted_by_user` user must be given"):
perm.insert()
_insert.assert_not_called()
# if give bad granted_by_user
_insert.reset_mock()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=999999,
)
with pytest.raises(IntegrityError, match="`granted_by_user` user must exist"):
perm.insert()
_insert.assert_not_called()
# if give bad granted_to_user
_insert.reset_mock()
perm = Permissions(
granted_to_user=999999,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=user.id,
)
with pytest.raises(IntegrityError, match="`granted_to_user` user must exist"):
perm.insert()
_insert.assert_not_called()
# This one will work
_insert.reset_mock()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=user.id,
)
with caplog.at_level(logging.DEBUG):
perm.insert()
_insert.assert_called_once()
assert any(
log_record.message.strip()
== f"admin-action: {user.email} gave {user.email} the permission wes_bam on {trial.trial_id}"
for log_record in caplog.records
)
gcloud_client.grant_download_access.assert_called_once()
# If granting a permission to a "network-viewer", no GCS IAM actions are taken
_insert.reset_mock()
gcloud_client.grant_download_access.reset_mock()
user.role = CIDCRole.NETWORK_VIEWER.value
user.update()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="ihc",
granted_by_user=user.id,
)
perm.insert()
_insert.assert_called_once()
gcloud_client.grant_download_access.assert_not_called()