def test_permissions_broad_perms(clean_db, monkeypatch):
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
other_trial = TrialMetadata(
trial_id="other-trial",
metadata_json={**METADATA, "protocol_identifier": "other-trial"},
)
other_trial.insert()
for ut in ["wes_fastq", "olink"]:
for tid in [trial.trial_id, other_trial.trial_id]:
Permissions(
granted_to_user=user.id,
trial_id=tid,
upload_type=ut,
granted_by_user=user.id,
).insert()
# Can't insert a permission for access to all trials and assays
with pytest.raises(ValueError, match="must have a trial id or upload type"):
Permissions(granted_to_user=user.id, granted_by_user=user.id).insert()
# Inserting a trial-level permission should delete other more specific related perms.
trial_query = clean_db.query(Permissions).filter(
Permissions.trial_id == trial.trial_id
)
assert trial_query.count() == 2
Permissions(
trial_id=trial.trial_id, granted_to_user=user.id, granted_by_user=user.id
).insert()
assert trial_query.count() == 1
perm = trial_query.one()
assert perm.trial_id == trial.trial_id
assert perm.upload_type is None
# Inserting an upload-level permission should delete other more specific related perms.
olink_query = clean_db.query(Permissions).filter(Permissions.upload_type == "olink")
assert olink_query.count() == 1
assert olink_query.one().trial_id == other_trial.trial_id
Permissions(
upload_type="olink", granted_to_user=user.id, granted_by_user=user.id
).insert()
assert olink_query.count() == 1
perm = olink_query.one()
assert perm.trial_id is None
assert perm.upload_type == "olink"
# Getting perms for a particular user-trial-type returns broader perms
perm = Permissions.find_for_user_trial_type(user.id, trial.trial_id, "ihc")
assert perm is not None and perm.upload_type is None
perm = Permissions.find_for_user_trial_type(user.id, "some random trial", "olink")
assert perm is not None and perm.trial_id is None
def test_permissions_revoke_all_iam_permissions(clean_db, monkeypatch):
"""
Smoke test that Permissions.revoke_all_iam_permissions calls revoke_download_access the right arguments.
"""
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
upload_types = ["wes_bam", "ihc", "rna_fastq", "plasma"]
for upload_type in upload_types:
Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type=upload_type,
granted_by_user=user.id,
).insert()
Permissions.revoke_all_iam_permissions()
gcloud_client.revoke_download_access.assert_has_calls(
[call(user.email, trial.trial_id, upload_type) for upload_type in upload_types]
)
# not called on admins or nci biobank users
gcloud_client.revoke_download_access.reset_mock()
for role in [CIDCRole.ADMIN.value, CIDCRole.NCI_BIOBANK_USER.value]:
user.role = role
user.update()
Permissions.revoke_all_iam_permissions()
gcloud_client.revoke_download_access.assert_not_called()
def test_trial_metadata_update(clean_db):
"""Test that metadata validation on update works as expected"""
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
# No error on valid `changes` update
trial.update(changes={"metadata_json": {**METADATA, "nct_id": "foo"}})
# No error on valid attribute update
trial.metadata_json = {**METADATA, "nct_id": "bar"}
trial.update()
bad_json = {"metadata_json": {"foo": "bar"}}
# Error on invalid `changes` update
with pytest.raises(ValidationMultiError):
trial.update(changes=bad_json)
# No error on invalid `changes` update if validate_metadata=False
trial.update(changes=bad_json, validate_metadata=False)
# Error on invalid attribute update
trial.metadata_json = bad_json
with pytest.raises(ValidationMultiError):
trial.update()
# No error on invalid attribute update if validate_metadata=False
trial.update(validate_metadata=False)
def setup_trial_and_user(cidc_api, monkeypatch) -> int:
"""
Insert a trial and a cimac-user into the database, and set the user
as the current user.
"""
# this is necessary for adding/removing permissions from this user
# without trying to contact GCP
mock_gcloud_client(monkeypatch)
user = Users(email=user_email,
role=CIDCRole.CIMAC_USER.value,
approval_date=datetime.now())
mock_current_user(user, monkeypatch)
with cidc_api.app_context():
TrialMetadata(
trial_id="test_trial",
metadata_json={
prism.PROTOCOL_ID_FIELD_NAME: trial_id,
"participants": [],
"allowed_cohort_names": ["Arm_Z"],
"allowed_collection_event_names": [],
},
).insert()
user.insert()
return user.id
def test_permissions_delete(clean_db, monkeypatch, caplog):
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=user.id,
)
perm.insert()
# Deleting a record by a user doesn't exist leads to an error
gcloud_client.reset_mocks()
with pytest.raises(NoResultFound, match="no user with id"):
perm.delete(deleted_by=999999)
# Deletion of an existing permission leads to no error
gcloud_client.reset_mocks()
with caplog.at_level(logging.DEBUG):
perm.delete(deleted_by=user.id)
gcloud_client.revoke_download_access.assert_called_once()
gcloud_client.grant_download_access.assert_not_called()
assert any(
log_record.message.strip()
== f"admin-action: {user.email} removed from {user.email} the permission wes_bam on {trial.trial_id}"
for log_record in caplog.records
)
# Deleting an already-deleted record is idempotent
gcloud_client.reset_mocks()
perm.delete(deleted_by=user)
gcloud_client.revoke_download_access.assert_called_once()
gcloud_client.grant_download_access.assert_not_called()
# Deleting a record whose user doesn't exist leads to an error
gcloud_client.reset_mocks()
with pytest.raises(NoResultFound, match="no user with id"):
Permissions(granted_to_user=999999).delete(deleted_by=user)
gcloud_client.revoke_download_access.assert_not_called()
gcloud_client.grant_download_access.assert_not_called()
# If revoking a permission from a "network-viewer", no GCS IAM actions are taken
gcloud_client.revoke_download_access.reset_mock()
user.role = CIDCRole.NETWORK_VIEWER.value
user.update()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="ihc",
granted_by_user=user.id,
)
perm.insert()
perm.delete(deleted_by=user)
gcloud_client.revoke_download_access.assert_not_called()
def setup_db_records(cidc_api):
extra = {"_etag": ETAG}
with cidc_api.app_context():
Users(**users["json"], **extra).insert(compute_etag=False)
TrialMetadata(**trial_metadata["json"], **extra).insert(compute_etag=False)
DownloadableFiles(**downloadable_files["json"], **extra).insert(
compute_etag=False
)
Permissions(**permissions["json"], **extra).insert(compute_etag=False)
UploadJobs(**upload_jobs["json"], **extra).insert(compute_etag=False)
def test_user_get_data_access_report(clean_db, monkeypatch):
"""Test that user data access info is collected as expected"""
mock_gcloud_client(monkeypatch)
admin_user = Users(
email="*****@*****.**",
organization="CIDC",
approval_date=datetime.now(),
role=CIDCRole.ADMIN.value,
)
admin_user.insert()
cimac_user = Users(
email="*****@*****.**",
organization="DFCI",
approval_date=datetime.now(),
role=CIDCRole.CIMAC_USER.value,
)
cimac_user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
upload_types = ["wes_bam", "ihc"]
# Note that admins don't need permissions to view data,
# so we're deliberately issuing unnecessary permissions here.
for user in [admin_user, cimac_user]:
for t in upload_types:
Permissions(
granted_to_user=user.id,
granted_by_user=admin_user.id,
trial_id=trial.trial_id,
upload_type=t,
).insert()
bio = io.BytesIO()
result_df = Users.get_data_access_report(bio)
bio.seek(0)
# Make sure bytes were written to the BytesIO instance
assert bio.getbuffer().nbytes > 0
# Make sure report data has expected info
assert set(result_df.columns) == set(
["email", "role", "organization", "trial_id", "permissions"]
)
for user in [admin_user, cimac_user]:
user_df = result_df[result_df.email == user.email]
assert set([user.role]) == set(user_df.role)
assert set([user.organization]) == set(user_df.organization)
if user == admin_user:
assert set(["*"]) == set(user_df.permissions)
else:
assert set(user_df.permissions).issubset(["wes_bam,ihc", "ihc,wes_bam"])
def test_partial_patch_trial_metadata(clean_db):
"""Update an existing trial_metadata_record"""
# Create the initial trial
clean_db.add(TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA))
clean_db.commit()
# Create patch without all required fields (no "participants")
metadata_patch = {PROTOCOL_ID_FIELD_NAME: TRIAL_ID, "assays": {}}
# patch it - should be no error/exception
TrialMetadata._patch_trial_metadata(TRIAL_ID, metadata_patch)
def test_create_trial_metadata(clean_db):
"""Insert a trial metadata record if one doesn't exist"""
TrialMetadata.create(TRIAL_ID, METADATA)
trial = TrialMetadata.find_by_trial_id(TRIAL_ID)
assert trial
assert trial.metadata_json == METADATA
# Check that you can't insert a trial with invalid metadata
with pytest.raises(ValidationMultiError, match="'buzz' was unexpected"):
TrialMetadata.create("foo", {"buzz": "bazz"})
with pytest.raises(ValidationMultiError, match="'buzz' was unexpected"):
TrialMetadata(trial_id="foo", metadata_json={"buzz": "bazz"}).insert()
def test_trial_metadata_insert(clean_db):
"""Test that metadata validation on insert works as expected"""
# No error with valid metadata
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
# Error with invalid metadata
trial.metadata_json = {"foo": "bar"}
with pytest.raises(ValidationMultiError):
trial.insert()
# No error if validate_metadata=False
trial.insert(validate_metadata=False)
def setup_downloadable_files(cidc_api) -> Tuple[int, int]:
"""Insert two downloadable files into the database."""
metadata_json = {
"protocol_identifier": trial_id_1,
"allowed_collection_event_names": [],
"allowed_cohort_names": [],
"participants": [],
}
trial_1 = TrialMetadata(trial_id=trial_id_1, metadata_json=metadata_json)
trial_2 = TrialMetadata(trial_id=trial_id_2, metadata_json=metadata_json)
def make_file(trial_id, object_url, upload_type,
facet_group) -> DownloadableFiles:
return DownloadableFiles(
trial_id=trial_id,
upload_type=upload_type,
object_url=f"{trial_id}/{object_url}",
facet_group=facet_group,
uploaded_timestamp=datetime.now(),
file_size_bytes=int(51 * 1e6), # 51MB
)
wes_file = make_file(trial_id_1, "wes/.../reads_123.bam", "wes_bam",
"/wes/r1_L.fastq.gz")
cytof_file = make_file(
trial_id_2,
"cytof/.../analysis.zip",
"cytof_10021_9204",
"/cytof_analysis/analysis.zip",
)
with cidc_api.app_context():
trial_1.insert()
trial_2.insert()
wes_file.insert()
cytof_file.insert()
return wes_file.id, cytof_file.id
def create_trial(n, grant_perm=False):
trial_id = f"test-trial-{n}"
metadata_json = {
"protocol_identifier":
trial_id,
"participants": [] if n == 2 else [{
"cimac_participant_id":
"CTTTPP1",
"participant_id":
"x",
"samples": [{
"cimac_id": f"CTTTPP1SS.01",
"sample_location": "",
"type_of_primary_container": "Other",
"type_of_sample": "Other",
"collection_event_name": "",
"parent_sample_id": "",
}],
}],
"allowed_collection_event_names": [""],
"allowed_cohort_names": [],
"assays": {},
"analysis": {},
"shipments": [],
}
trial = TrialMetadata(trial_id=trial_id, metadata_json=metadata_json)
trial.insert()
if grant_perm and user_id:
Permissions(
granted_to_user=user_id,
trial_id=trial.trial_id,
upload_type="olink",
granted_by_user=user_id,
).insert()
Permissions(
granted_to_user=user_id,
trial_id=trial.trial_id,
upload_type="ihc",
granted_by_user=user_id,
).insert()
return trial.id
def test_permissions_grant_iam_permissions(clean_db, monkeypatch):
"""
Smoke test that Permissions.grant_iam_permissions calls grant_download_access with the right arguments.
"""
refresh_intake_access = MagicMock()
monkeypatch.setattr(
"cidc_api.models.models.refresh_intake_access", refresh_intake_access
)
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**", role=CIDCRole.NETWORK_VIEWER.value)
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
upload_types = ["wes_bam", "ihc", "rna_fastq", "plasma"]
for upload_type in upload_types:
Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type=upload_type,
granted_by_user=user.id,
).insert()
# IAM permissions not granted to network viewers
Permissions.grant_iam_permissions(user=user)
gcloud_client.grant_download_access.assert_not_called()
# IAM permissions should be granted for any other role
user.role = CIDCRole.CIMAC_USER.value
Permissions.grant_iam_permissions(user=user)
for upload_type in upload_types:
assert (
call(user.email, trial.trial_id, upload_type)
in gcloud_client.grant_download_access.call_args_list
)
refresh_intake_access.assert_called_once_with(user.email)
def insert_trial(trial_id, num_participants, num_samples):
TrialMetadata(
trial_id=trial_id,
metadata_json={
prism.PROTOCOL_ID_FIELD_NAME:
trial_id,
"allowed_cohort_names": [""],
"allowed_collection_event_names": [""],
"participants": [{
"cimac_participant_id":
f"CTTTPP{p}",
"participant_id":
"x",
"samples": [{
"cimac_id": f"CTTTPP1SS.0{s}",
"sample_location": "",
"type_of_primary_container": "Other",
"type_of_sample": "Other",
"collection_event_name": "",
"parent_sample_id": "",
} for s in range(num_samples[p])],
} for p in range(num_participants)],
},
).insert()
def test_upload_data_files(cidc_api, monkeypatch):
user = Users(email="*****@*****.**")
trial = TrialMetadata(
trial_id="test_trial",
metadata_json={
prism.PROTOCOL_ID_FIELD_NAME: trial_id,
"participants": [],
"allowed_cohort_names": ["Arm_Z"],
"allowed_collection_event_names": [],
},
)
template_type = "foo"
xlsx_file = MagicMock()
md_patch = {}
file_infos = [
finfo(
"localfile1.ext",
"test_trial/url/file1.ext",
"uuid-1",
metadata_availability=None,
allow_empty=None,
),
finfo(
"localfile2.ext",
"test_trial/url/file2.ext",
"uuid-2",
metadata_availability=True,
allow_empty=None,
),
finfo(
"localfile3.ext",
"test_trial/url/file3.ext",
"uuid-3",
metadata_availability=None,
allow_empty=True,
),
finfo(
"localfile4.ext",
"test_trial/url/file4.ext",
"uuid-4",
metadata_availability=True,
allow_empty=True,
),
]
gcloud_client = MagicMock()
gcloud_client.grant_upload_access = MagicMock()
gcloud_client.upload_xlsx_to_gcs = MagicMock()
gcs_blob = MagicMock()
gcs_blob.name = "blob"
gcloud_client.upload_xlsx_to_gcs.return_value = gcs_blob
monkeypatch.setattr("cidc_api.resources.upload_jobs.gcloud_client",
gcloud_client)
create = MagicMock()
job = MagicMock()
job.id = "id"
job._etag = "_etag"
job.token = "token"
create.return_value = job
monkeypatch.setattr("cidc_api.resources.upload_jobs.UploadJobs.create",
create)
with cidc_api.app_context():
response = upload_data_files(user, trial, template_type, xlsx_file,
md_patch, file_infos)
json = response.get_json()
assert "job_id" in json and json["job_id"] == "id"
assert "job_etag" in json and json["job_etag"] == "_etag"
assert "url_mapping" in json
url_mapping = {
k: v.rsplit("/", 1)[0]
for k, v in json["url_mapping"].items()
}
assert url_mapping == {
"localfile1.ext": "test_trial/url/file1.ext",
"localfile2.ext": "test_trial/url/file2.ext",
"localfile3.ext": "test_trial/url/file3.ext",
"localfile4.ext": "test_trial/url/file4.ext",
}
assert "gcs_bucket" in json and json["gcs_bucket"] == "cidc-uploads-staging"
assert "extra_metadata" in json and json["extra_metadata"] == {
"localfile2.ext": "uuid-2",
"localfile4.ext": "uuid-4",
}
assert "gcs_file_map" in json
gcs_file_map = sorted(
[(k.rsplit("/", 1)[0], v) for k, v in json["gcs_file_map"].items()],
key=lambda i: i[0],
)
assert gcs_file_map == [
("test_trial/url/file1.ext", "uuid-1"),
("test_trial/url/file2.ext", "uuid-2"),
("test_trial/url/file3.ext", "uuid-3"),
("test_trial/url/file4.ext", "uuid-4"),
]
assert "optional_files" in json and json["optional_files"] == [
"localfile3.ext",
"localfile4.ext",
]
assert "token" in json and json["token"] == "token"
def setup_data(cidc_api, clean_db):
user = Users(email="*****@*****.**", approval_date=datetime.now())
shipment = {
"courier": "FEDEX",
"ship_to": "",
"ship_from": "",
"assay_type": assay_type,
"manifest_id": manifest_id,
"date_shipped": "2020-06-10 00:00:00",
"date_received": "2020-06-11 00:00:00",
"account_number": "",
"assay_priority": "1",
"receiving_party": "MSSM_Rahman",
"tracking_number": "",
"shipping_condition": "Frozen_Dry_Ice",
"quality_of_shipment": "Specimen shipment received in good condition",
}
metadata = {
"protocol_identifier":
trial_id,
"shipments": [
# we get duplicate shipment uploads sometimes
shipment,
shipment,
],
"participants": [{
"cimac_participant_id":
f"CTTTPP{p}",
"participant_id":
"x",
"cohort_name":
"",
"samples": [{
"cimac_id": f"CTTTPP{p}SS.0{s}",
"sample_location": "",
"type_of_primary_container": "Other",
"type_of_sample": "Other",
"collection_event_name": "",
"parent_sample_id": "",
} for s in range(num_samples[p])],
} for p in range(num_participants)],
"allowed_cohort_names": [""],
"allowed_collection_event_names": [""],
}
trial = TrialMetadata(trial_id=trial_id, metadata_json=metadata)
upload_job = UploadJobs(
uploader_email=user.email,
trial_id=trial.trial_id,
upload_type="pbmc",
gcs_xlsx_uri="",
metadata_patch=metadata,
multifile=False,
)
upload_job._set_status_no_validation(UploadJobStatus.MERGE_COMPLETED.value)
with cidc_api.app_context():
user.insert()
trial.insert()
upload_job.insert()
clean_db.refresh(user)
clean_db.refresh(upload_job)
clean_db.refresh(trial)
return user, upload_job, trial
def test_create_downloadable_file_from_blob(clean_db, monkeypatch):
"""Try to create a downloadable file from a GCS blob"""
fake_blob = MagicMock()
fake_blob.name = "name"
fake_blob.md5_hash = "12345"
fake_blob.crc32c = "54321"
fake_blob.size = 5
fake_blob.time_created = datetime.now()
clean_db.add(
TrialMetadata(
trial_id="id",
metadata_json={
"protocol_identifier": "id",
"allowed_collection_event_names": [],
"allowed_cohort_names": [],
"participants": [],
},
)
)
df = DownloadableFiles.create_from_blob(
"id", "pbmc", "Shipping Manifest", "pbmc/shipping", fake_blob
)
# Mock artifact upload publishing
publisher = MagicMock()
monkeypatch.setattr("cidc_api.models.models.publish_artifact_upload", publisher)
# Check that the file was created
assert 1 == clean_db.query(DownloadableFiles).count()
df_lookup = DownloadableFiles.find_by_id(df.id)
assert df_lookup.object_url == fake_blob.name
assert df_lookup.data_format == "Shipping Manifest"
assert df_lookup.file_size_bytes == fake_blob.size
assert df_lookup.md5_hash == fake_blob.md5_hash
assert df_lookup.crc32c_hash == fake_blob.crc32c
# uploading second time to check non duplicating entries
fake_blob.size = 6
fake_blob.md5_hash = "6"
df = DownloadableFiles.create_from_blob(
"id", "pbmc", "Shipping Manifest", "pbmc/shipping", fake_blob
)
# Check that the file was created
assert 1 == clean_db.query(DownloadableFiles).count()
df_lookup = DownloadableFiles.find_by_id(df.id)
assert df_lookup.file_size_bytes == 6
assert df_lookup.md5_hash == "6"
# Check that no artifact upload event was published
publisher.assert_not_called()
# Check that artifact upload publishes
DownloadableFiles.create_from_blob(
"id",
"pbmc",
"Shipping Manifest",
"pbmc/shipping",
fake_blob,
alert_artifact_upload=True,
)
publisher.assert_called_once_with(fake_blob.name)
def test_trial_metadata_get_summaries(clean_db, monkeypatch):
"""Check that trial data summaries are computed as expected"""
# Add some trials
records = [{"fake": "record"}]
cytof_record_with_output = [{"output_files": {"foo": "bar"}}]
tm1 = {
**METADATA,
# deliberately override METADATA['protocol_identifier']
"protocol_identifier": "tm1",
"participants": [{"samples": [1, 2]}, {"samples": [3]}],
"expected_assays": ["ihc", "olink"],
"assays": {
"wes": [
{"records": records * 6},
{"records": records * 5},
], # 6 + 5 11 = 7 for wes + 4 for wes_tumor_only
"rna": [{"records": records * 2}],
"mif": [
{"records": records * 3},
{"records": records},
{"records": records},
],
"elisa": [{"assay_xlsx": {"number_of_samples": 7}}],
"nanostring": [
{"runs": [{"samples": records * 2}]},
{"runs": [{"samples": records * 1}]},
],
"hande": [{"records": records * 5}],
},
"analysis": {
"wes_analysis": {
"pair_runs": [
# 7 here for wes_assay: t0/1/2, n0/1/2/3
{
"tumor": {"cimac_id": "t0"},
"normal": {"cimac_id": "n0"},
}, # no analysis data
{
"tumor": {"cimac_id": "t1"},
"normal": {"cimac_id": "n1"},
"report": {"report": "foo"},
},
{
"tumor": {"cimac_id": "t1"},
"normal": {"cimac_id": "n2"},
"report": {"report": "foo"},
},
{
"tumor": {"cimac_id": "t2"},
"normal": {"cimac_id": "n3"},
"report": {"report": "foo"},
},
],
# these are excluded, so not adding fake assay data
"excluded_samples": records * 2,
},
"wes_tumor_only_analysis": {
"runs": records * 4, # need 4
# these are excluded, so not adding fake assay data
"excluded_samples": records * 3,
},
},
"clinical_data": {
"records": [
{"clinical_file": {"participants": ["a", "b", "c"]}},
{"clinical_file": {"participants": ["a", "b", "d"]}},
{"clinical_file": {"participants": ["e", "f", "g"]}},
]
},
}
tm2 = {
**METADATA,
# deliberately override METADATA['protocol_identifier']
"protocol_identifier": "tm1",
"participants": [{"samples": []}],
"assays": {
"cytof_10021_9204": [
{
"records": cytof_record_with_output * 2,
"excluded_samples": records * 2,
},
{"records": records * 2},
{"records": records},
],
"cytof_e4412": [
{
"participants": [
{"samples": records},
{"samples": cytof_record_with_output * 5},
{"samples": records * 2},
],
"excluded_samples": records,
}
],
"olink": {
"batches": [
{
"records": [
{"files": {"assay_npx": {"number_of_samples": 2}}},
{"files": {"assay_npx": {"number_of_samples": 3}}},
]
},
{"records": [{"files": {"assay_npx": {"number_of_samples": 3}}}]},
]
},
},
"analysis": {
"rna_analysis": {"level_1": records * 10, "excluded_samples": records * 2},
"tcr_analysis": {
"batches": [
{"records": records * 4, "excluded_samples": records * 3},
{"records": records * 2, "excluded_samples": records * 1},
]
},
},
}
TrialMetadata(trial_id="tm1", metadata_json=tm1).insert(validate_metadata=False)
TrialMetadata(trial_id="tm2", metadata_json=tm2).insert(validate_metadata=False)
# Add some files
for i, (tid, fs) in enumerate([("tm1", 3), ("tm1", 2), ("tm2", 4), ("tm2", 6)]):
DownloadableFiles(
trial_id=tid,
file_size_bytes=fs,
object_url=str(i),
facet_group="",
uploaded_timestamp=datetime.now(),
upload_type="",
).insert()
sorter = lambda s: s["trial_id"]
received = sorted(TrialMetadata.get_summaries(), key=sorter)
expected = sorted(
[
{
"expected_assays": [],
"cytof": 13.0,
"olink": 8.0,
"trial_id": "tm2",
"file_size_bytes": 10,
"total_participants": 1,
"total_samples": 0,
"clinical_participants": 0.0,
"rna": 0.0,
"nanostring": 0.0,
"elisa": 0.0,
"h&e": 0.0,
"mif": 0.0,
"cytof_analysis": 7.0,
"rna_level1_analysis": 10.0,
"tcr_analysis": 6.0,
"wes_analysis": 0.0,
"wes_tumor_only_analysis": 0.0,
"wes": 0.0,
"wes_tumor_only": 0.0,
"excluded_samples": {
"tcr_analysis": records * 4,
"rna_level1_analysis": records * 2,
"cytof_analysis": records * 3,
},
},
{
"expected_assays": ["ihc", "olink"],
"elisa": 7.0,
"cytof": 0.0,
"olink": 0.0,
"trial_id": "tm1",
"file_size_bytes": 5,
"total_participants": 2,
"total_samples": 3,
"clinical_participants": 7.0,
"rna": 2.0,
"nanostring": 3.0,
"h&e": 5.0,
"mif": 5.0,
"cytof_analysis": 0.0,
"rna_level1_analysis": 0.0,
"tcr_analysis": 0.0,
"wes_analysis": 5.0,
"wes_tumor_only_analysis": 4.0,
"wes": 7.0,
"wes_tumor_only": 4.0,
"excluded_samples": {
"wes_analysis": records * 2,
"wes_tumor_only_analysis": records * 3,
},
},
],
key=sorter,
)
assert received == expected
assert all("misc_data" not in entry for entry in received)
def test_permissions_insert(clean_db, monkeypatch, caplog):
gcloud_client = mock_gcloud_client(monkeypatch)
user = Users(email="*****@*****.**")
user.insert()
trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA)
trial.insert()
_insert = MagicMock()
monkeypatch.setattr(CommonColumns, "insert", _insert)
# if upload_type is invalid
with pytest.raises(ValueError, match="invalid upload type"):
Permissions(upload_type="foo", granted_to_user=user.id, trial_id=trial.trial_id)
# if don't give granted_by_user
perm = Permissions(
granted_to_user=user.id, trial_id=trial.trial_id, upload_type="wes_bam"
)
with pytest.raises(IntegrityError, match="`granted_by_user` user must be given"):
perm.insert()
_insert.assert_not_called()
# if give bad granted_by_user
_insert.reset_mock()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=999999,
)
with pytest.raises(IntegrityError, match="`granted_by_user` user must exist"):
perm.insert()
_insert.assert_not_called()
# if give bad granted_to_user
_insert.reset_mock()
perm = Permissions(
granted_to_user=999999,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=user.id,
)
with pytest.raises(IntegrityError, match="`granted_to_user` user must exist"):
perm.insert()
_insert.assert_not_called()
# This one will work
_insert.reset_mock()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="wes_bam",
granted_by_user=user.id,
)
with caplog.at_level(logging.DEBUG):
perm.insert()
_insert.assert_called_once()
assert any(
log_record.message.strip()
== f"admin-action: {user.email} gave {user.email} the permission wes_bam on {trial.trial_id}"
for log_record in caplog.records
)
gcloud_client.grant_download_access.assert_called_once()
# If granting a permission to a "network-viewer", no GCS IAM actions are taken
_insert.reset_mock()
gcloud_client.grant_download_access.reset_mock()
user.role = CIDCRole.NETWORK_VIEWER.value
user.update()
perm = Permissions(
granted_to_user=user.id,
trial_id=trial.trial_id,
upload_type="ihc",
granted_by_user=user.id,
)
perm.insert()
_insert.assert_called_once()
gcloud_client.grant_download_access.assert_not_called()
