def test_get_permission(cidc_api, clean_db, monkeypatch): """Check that getting a single permission by ID works as expected.""" mock_gcloud_client(monkeypatch) current_user_id, other_user_id = setup_permissions(cidc_api, monkeypatch) with cidc_api.app_context(): current_user_perm = Permissions.find_for_user(current_user_id)[0] other_user_perm = Permissions.find_for_user(other_user_id)[0] client = cidc_api.test_client() # Check that getting a permission that doesn't exist yields 404 res = client.get("permissions/123212321") assert res.status_code == 404 # Check that a non-admin getting another user's permission yields 404 res = client.get(f"permissions/{other_user_perm.id}") assert res.status_code == 404 # Check that a non-admin can get their own permission res = client.get(f"permissions/{current_user_perm.id}") assert res.status_code == 200 assert res.json == PermissionSchema().dump(current_user_perm) # Check that an admin can get another user's permission make_admin(current_user_id, cidc_api) res = client.get(f"permissions/{other_user_perm.id}") assert res.status_code == 200 assert res.json == PermissionSchema().dump(other_user_perm)
def test_get_downloadable_file(cidc_api, clean_db, monkeypatch): """Check that getting a single file works as expected.""" user_id = setup_user(cidc_api, monkeypatch) file_id_1, file_id_2 = setup_downloadable_files(cidc_api) client = cidc_api.test_client() # Non-admins get 401s for single files they don't have permision to view res = client.get(f"/downloadable_files/{file_id_1}") assert res.status_code == 401 # Give the user one permission with cidc_api.app_context(): perm = Permissions( granted_to_user=user_id, trial_id=trial_id_1, upload_type=upload_types[0], granted_by_user=user_id, ) perm.insert() # Non-admins can get single files that they have permision to view res = client.get(f"/downloadable_files/{file_id_1}") assert res.status_code == 200 assert res.json["id"] == file_id_1 # Admins can get any file regardless of permissions make_admin(user_id, cidc_api) res = client.get(f"/downloadable_files/{file_id_2}") assert res.status_code == 200 assert res.json["id"] == file_id_2 # Non-existent files yield 404 res = client.get(f"/downloadable_files/123212321") assert res.status_code == 404
def test_permissions_revoke_all_iam_permissions(clean_db, monkeypatch): """ Smoke test that Permissions.revoke_all_iam_permissions calls revoke_download_access the right arguments. """ gcloud_client = mock_gcloud_client(monkeypatch) user = Users(email="*****@*****.**") user.insert() trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA) trial.insert() upload_types = ["wes_bam", "ihc", "rna_fastq", "plasma"] for upload_type in upload_types: Permissions( granted_to_user=user.id, trial_id=trial.trial_id, upload_type=upload_type, granted_by_user=user.id, ).insert() Permissions.revoke_all_iam_permissions() gcloud_client.revoke_download_access.assert_has_calls( [call(user.email, trial.trial_id, upload_type) for upload_type in upload_types] ) # not called on admins or nci biobank users gcloud_client.revoke_download_access.reset_mock() for role in [CIDCRole.ADMIN.value, CIDCRole.NCI_BIOBANK_USER.value]: user.role = role user.update() Permissions.revoke_all_iam_permissions() gcloud_client.revoke_download_access.assert_not_called()
def create_permission(uid, assay): Permissions( granted_by_user=uid, granted_to_user=uid, trial_id=TRIAL_ID, upload_type=assay, ).insert()
def grant_upload_permission(user_id, upload_type, cidc_api): with cidc_api.app_context(): Permissions( granted_by_user=user_id, granted_to_user=user_id, trial_id=trial_id, upload_type=upload_type, ).insert()
def test_get_related_files(cidc_api, clean_db, monkeypatch): """Check that the related_files endpoint calls `get_related_files`""" user_id = setup_user(cidc_api, monkeypatch) file_id_1, file_id_2 = setup_downloadable_files(cidc_api) client = cidc_api.test_client() # Add an additional file that is related to file 1 object_url = "/foo/bar" with cidc_api.app_context(): DownloadableFiles( trial_id=trial_id_1, upload_type="wes", object_url=object_url, facet_group= "/wes/r2_L.fastq.gz", # this is what makes this file "related" uploaded_timestamp=datetime.now(), file_size_bytes=0, ).insert() # Non-admins get 401s when requesting related files they don't have permission to view res = client.get(f"/downloadable_files/{file_id_1}/related_files") assert res.status_code == 401 # Give the user one permission with cidc_api.app_context(): perm = Permissions( granted_to_user=user_id, trial_id=trial_id_1, upload_type=upload_types[0], granted_by_user=user_id, ) perm.insert() # Non-admins can get related files that they have permision to view res = client.get(f"/downloadable_files/{file_id_1}/related_files") assert res.status_code == 200 assert len(res.json["_items"]) == 1 # file 1 has 1 related file assert res.json["_items"][0]["object_url"] == object_url # Admins can get related files without permissions make_admin(user_id, cidc_api) res = client.get(f"/downloadable_files/{file_id_2}/related_files") assert res.status_code == 200 assert len(res.json["_items"]) == 0 # file 2 has 0 related file
def test_get_download_url(cidc_api, clean_db, monkeypatch): """Check that generating a GCS signed URL works as expected""" user_id = setup_user(cidc_api, monkeypatch) file_id, _ = setup_downloadable_files(cidc_api) client = cidc_api.test_client() # A query missing the required parameters should yield 422 res = client.get("/downloadable_files/download_url") assert res.status_code == 422 assert res.json["_error"]["message"]["query"]["id"] == [ "Missing data for required field." ] # A missing file should yield 404 res = client.get("/downloadable_files/download_url?id=123212321") assert res.status_code == 404 # No permission should also yield 404 res = client.get(f"/downloadable_files/download_url?id={file_id}") assert res.status_code == 404 with cidc_api.app_context(): perm = Permissions( granted_to_user=user_id, trial_id=trial_id_1, upload_type=upload_types[0], granted_by_user=user_id, ) perm.insert() test_url = "foo" monkeypatch.setattr("cidc_api.shared.gcloud_client.get_signed_url", lambda *args: test_url) res = client.get(f"/downloadable_files/download_url?id={file_id}") assert res.status_code == 200 assert res.json == test_url # network viewers aren't allowed to get download urls make_role(user_id, CIDCRole.NETWORK_VIEWER.value, cidc_api) res = client.get(f"/downloadable_files/download_url?id={file_id}") assert res.status_code == 401
def setup_db_records(cidc_api): extra = {"_etag": ETAG} with cidc_api.app_context(): Users(**users["json"], **extra).insert(compute_etag=False) TrialMetadata(**trial_metadata["json"], **extra).insert(compute_etag=False) DownloadableFiles(**downloadable_files["json"], **extra).insert( compute_etag=False ) Permissions(**permissions["json"], **extra).insert(compute_etag=False) UploadJobs(**upload_jobs["json"], **extra).insert(compute_etag=False)
def test_user_get_data_access_report(clean_db, monkeypatch): """Test that user data access info is collected as expected""" mock_gcloud_client(monkeypatch) admin_user = Users( email="*****@*****.**", organization="CIDC", approval_date=datetime.now(), role=CIDCRole.ADMIN.value, ) admin_user.insert() cimac_user = Users( email="*****@*****.**", organization="DFCI", approval_date=datetime.now(), role=CIDCRole.CIMAC_USER.value, ) cimac_user.insert() trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA) trial.insert() upload_types = ["wes_bam", "ihc"] # Note that admins don't need permissions to view data, # so we're deliberately issuing unnecessary permissions here. for user in [admin_user, cimac_user]: for t in upload_types: Permissions( granted_to_user=user.id, granted_by_user=admin_user.id, trial_id=trial.trial_id, upload_type=t, ).insert() bio = io.BytesIO() result_df = Users.get_data_access_report(bio) bio.seek(0) # Make sure bytes were written to the BytesIO instance assert bio.getbuffer().nbytes > 0 # Make sure report data has expected info assert set(result_df.columns) == set( ["email", "role", "organization", "trial_id", "permissions"] ) for user in [admin_user, cimac_user]: user_df = result_df[result_df.email == user.email] assert set([user.role]) == set(user_df.role) assert set([user.organization]) == set(user_df.organization) if user == admin_user: assert set(["*"]) == set(user_df.permissions) else: assert set(user_df.permissions).issubset(["wes_bam,ihc", "ihc,wes_bam"])
def create_trial(n, grant_perm=False): trial_id = f"test-trial-{n}" metadata_json = { "protocol_identifier": trial_id, "participants": [] if n == 2 else [{ "cimac_participant_id": "CTTTPP1", "participant_id": "x", "samples": [{ "cimac_id": f"CTTTPP1SS.01", "sample_location": "", "type_of_primary_container": "Other", "type_of_sample": "Other", "collection_event_name": "", "parent_sample_id": "", }], }], "allowed_collection_event_names": [""], "allowed_cohort_names": [], "assays": {}, "analysis": {}, "shipments": [], } trial = TrialMetadata(trial_id=trial_id, metadata_json=metadata_json) trial.insert() if grant_perm and user_id: Permissions( granted_to_user=user_id, trial_id=trial.trial_id, upload_type="olink", granted_by_user=user_id, ).insert() Permissions( granted_to_user=user_id, trial_id=trial.trial_id, upload_type="ihc", granted_by_user=user_id, ).insert() return trial.id
def refresh_download_permissions(*args): """ Extend the expiry date for GCS download permissions belonging to users who accessed the system in the last 2 (or so) days. If we don't do this, users whose accounts are still active might lose GCS download permission prematurely. """ active_today = lambda q: q.filter( # Provide a 3 day window to ensure we don't miss anyone # if, e.g., this function fails to run on a certain day. Users._accessed > datetime.today() - timedelta(days=3) ) with sqlalchemy_session() as session: active_users = Users.list( page_size=Users.count(session=session, filter_=active_today), session=session, filter_=active_today, ) for user in active_users: print(f"Refreshing IAM download permissions for {user.email}") Permissions.grant_iam_permissions(user, session=session)
def test_permissions_grant_iam_permissions(clean_db, monkeypatch): """ Smoke test that Permissions.grant_iam_permissions calls grant_download_access with the right arguments. """ refresh_intake_access = MagicMock() monkeypatch.setattr( "cidc_api.models.models.refresh_intake_access", refresh_intake_access ) gcloud_client = mock_gcloud_client(monkeypatch) user = Users(email="*****@*****.**", role=CIDCRole.NETWORK_VIEWER.value) user.insert() trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA) trial.insert() upload_types = ["wes_bam", "ihc", "rna_fastq", "plasma"] for upload_type in upload_types: Permissions( granted_to_user=user.id, trial_id=trial.trial_id, upload_type=upload_type, granted_by_user=user.id, ).insert() # IAM permissions not granted to network viewers Permissions.grant_iam_permissions(user=user) gcloud_client.grant_download_access.assert_not_called() # IAM permissions should be granted for any other role user.role = CIDCRole.CIMAC_USER.value Permissions.grant_iam_permissions(user=user) for upload_type in upload_types: assert ( call(user.email, trial.trial_id, upload_type) in gcloud_client.grant_download_access.call_args_list ) refresh_intake_access.assert_called_once_with(user.email)
def test_permissions_broad_perms(clean_db, monkeypatch): gcloud_client = mock_gcloud_client(monkeypatch) user = Users(email="*****@*****.**") user.insert() trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA) trial.insert() other_trial = TrialMetadata( trial_id="other-trial", metadata_json={**METADATA, "protocol_identifier": "other-trial"}, ) other_trial.insert() for ut in ["wes_fastq", "olink"]: for tid in [trial.trial_id, other_trial.trial_id]: Permissions( granted_to_user=user.id, trial_id=tid, upload_type=ut, granted_by_user=user.id, ).insert() # Can't insert a permission for access to all trials and assays with pytest.raises(ValueError, match="must have a trial id or upload type"): Permissions(granted_to_user=user.id, granted_by_user=user.id).insert() # Inserting a trial-level permission should delete other more specific related perms. trial_query = clean_db.query(Permissions).filter( Permissions.trial_id == trial.trial_id ) assert trial_query.count() == 2 Permissions( trial_id=trial.trial_id, granted_to_user=user.id, granted_by_user=user.id ).insert() assert trial_query.count() == 1 perm = trial_query.one() assert perm.trial_id == trial.trial_id assert perm.upload_type is None # Inserting an upload-level permission should delete other more specific related perms. olink_query = clean_db.query(Permissions).filter(Permissions.upload_type == "olink") assert olink_query.count() == 1 assert olink_query.one().trial_id == other_trial.trial_id Permissions( upload_type="olink", granted_to_user=user.id, granted_by_user=user.id ).insert() assert olink_query.count() == 1 perm = olink_query.one() assert perm.trial_id is None assert perm.upload_type == "olink" # Getting perms for a particular user-trial-type returns broader perms perm = Permissions.find_for_user_trial_type(user.id, trial.trial_id, "ihc") assert perm is not None and perm.upload_type is None perm = Permissions.find_for_user_trial_type(user.id, "some random trial", "olink") assert perm is not None and perm.trial_id is None
def test_list_downloadable_files(cidc_api, clean_db, monkeypatch): """Check that getting a list of files works as expected""" user_id = setup_user(cidc_api, monkeypatch) file_id_1, file_id_2 = setup_downloadable_files(cidc_api) client = cidc_api.test_client() # Non-admins can't get files they don't have permissions for res = client.get("/downloadable_files") assert res.status_code == 200 assert len(res.json["_items"]) == 0 assert res.json["_meta"]["total"] == 0 # Test a couple different permission configurations that should # produce the same result in this case. for kwargs in [ { "trial_id": trial_id_1, "upload_type": upload_types[0] }, { "upload_type": upload_types[0] }, { "trial_id": trial_id_1 }, ]: # Give the user one permission with cidc_api.app_context(): perm = Permissions(granted_to_user=user_id, granted_by_user=user_id, **kwargs) perm.insert() # Non-admins can view files for which they have permission res = client.get("/downloadable_files") assert res.status_code == 200 assert len(res.json["_items"]) == 1 assert res.json["_meta"]["total"] == 1 assert res.json["_items"][0]["id"] == file_id_1 # Non-admin filter queries exclude files they aren't allowed to view res = client.get( f"/downloadable_files?facets=Assay Type|CyTOF|Analysis Results") assert res.status_code == 200 assert len(res.json["_items"]) == 0 assert res.json["_meta"]["total"] == 0 # Admins and NCI biobank users can view all files regardless of their permissions for role in [CIDCRole.ADMIN.value, CIDCRole.NCI_BIOBANK_USER.value]: make_role(user_id, role, cidc_api) res = client.get("/downloadable_files") assert res.status_code == 200 assert len(res.json["_items"]) == 2 assert res.json["_meta"]["total"] == 2 assert set([f["id"] for f in res.json["_items"] ]) == set([file_id_1, file_id_2]) # Admin filter queries include any files that fit the criteria res = client.get( f"/downloadable_files?facets=Assay Type|CyTOF|Analysis Results") assert res.status_code == 200 assert len(res.json["_items"]) == 1 assert res.json["_meta"]["total"] == 1 assert res.json["_items"][0]["id"] == file_id_2 # Make sure it's possible to sort by file extension res = client.get( f"/downloadable_files?sort_field=file_ext&sort_direction=asc") assert res.status_code == 200 assert [f["file_ext"] for f in res.json["_items"]] == ["bam", "zip"] # Make sure it's possible to sort by data category res = client.get( f"/downloadable_files?sort_field=data_category&sort_direction=asc") assert res.status_code == 200 assert [f["data_category"] for f in res.json["_items"]] == [ "CyTOF|Analysis Results", "WES|Source", ]
def test_get_filter_facets(cidc_api, clean_db, monkeypatch): """Check that getting filter facets works as expected""" user_id = setup_user(cidc_api, monkeypatch) setup_downloadable_files(cidc_api) client = cidc_api.test_client() def check_facet_counts(facets, wes_count=0, cytof_count=0): for facet, subfacets in facets["Assay Type"].items(): if not isinstance(subfacets, list): subfacets = [subfacets] for subfacet in subfacets: if facet == "WES" and subfacet["label"] == "Source": assert subfacet["count"] == wes_count elif facet == "CyTOF" and subfacet[ "label"] == "Analysis Results": assert subfacet["count"] == cytof_count else: assert subfacet["count"] == 0 # Non-admins' facets take into account their permissions with cidc_api.app_context(): perm = Permissions( granted_to_user=user_id, trial_id=trial_id_1, upload_type=upload_types[0], granted_by_user=user_id, ) perm.insert() res = client.get("/downloadable_files/filter_facets") assert res.status_code == 200 check_facet_counts(res.json["facets"], wes_count=1, cytof_count=0) # Admins' facets include all files, regardless of permissions make_admin(user_id, cidc_api) res = client.get("/downloadable_files/filter_facets") assert res.status_code == 200 assert sorted(res.json["trial_ids"], key=lambda x: x["label"]) == sorted( [{ "label": trial_id_1, "count": 1 }, { "label": trial_id_2, "count": 1 }], key=lambda x: x["label"], ) check_facet_counts(res.json["facets"], wes_count=1, cytof_count=1) # Trial facets are governed by data category facets res = client.get( "/downloadable_files/filter_facets?facets=Assay Type|WES|Germline") assert res.status_code == 200 assert res.json["trial_ids"] == [] check_facet_counts(res.json["facets"], wes_count=1, cytof_count=1) # Data category facets are governed by trial facets res = client.get( "/downloadable_files/filter_facets?trial_ids=someothertrial") assert res.status_code == 200 res.json["trial_ids"] == [] check_facet_counts(res.json["facets"], wes_count=0, cytof_count=0)
def test_create_compressed_batch(cidc_api, clean_db, monkeypatch): user_id = setup_user(cidc_api, monkeypatch) file_id_1, file_id_2 = setup_downloadable_files(cidc_api) with cidc_api.app_context(): url_1 = DownloadableFiles.find_by_id(file_id_1).object_url url_2 = DownloadableFiles.find_by_id(file_id_2).object_url client = cidc_api.test_client() url = "/downloadable_files/compressed_batch" # A JSON body containing a file ID list must be provided res = client.post(url) assert res.status_code == 422 # User has no permissions, so no files should be found short_file_list = {"file_ids": [file_id_1, file_id_2]} res = client.post(url, json=short_file_list) assert res.status_code == 404 # Give the user one permission with cidc_api.app_context(): perm = Permissions( granted_to_user=user_id, trial_id=trial_id_1, upload_type=upload_types[0], granted_by_user=user_id, ) perm.insert() # Mock GCS client blob = MagicMock() bucket = MagicMock() bucket.blob.return_value = blob monkeypatch.setattr( "cidc_api.resources.downloadable_files.gcloud_client._get_bucket", lambda _: bucket, ) signed_url = "fake/signed/url" monkeypatch.setattr( "cidc_api.resources.downloadable_files.gcloud_client.get_signed_url", lambda *_: signed_url, ) # User has one permission, s0 the endpoint should try to create # a compressed batch file with the single file the user has # access to in it. res = client.post(url, json=short_file_list) assert res.status_code == 200 assert res.json == signed_url print(bucket.get_blob.call_args_list) bucket.get_blob.assert_called_with(url_1) blob.upload_from_filename.assert_called_once() bucket.reset_mock() blob.reset_mock() make_admin(user_id, cidc_api) # Admin has access to both files, but together they are too large res = client.post(url, json=short_file_list) assert res.status_code == 400 assert "batch too large" in res.json["_error"]["message"] bucket.get_blob.assert_not_called() blob.upload_from_filename.assert_not_called() # Decrease the size of one of the files and try again with cidc_api.app_context(): df = DownloadableFiles.find_by_id(file_id_1) df.file_size_bytes = 1 df.update() res = client.post(url, json=short_file_list) assert res.status_code == 200 assert res.json == signed_url assert call(url_1) in bucket.get_blob.call_args_list assert call(url_2) in bucket.get_blob.call_args_list blob.upload_from_filename.assert_called_once()
def test_get_filelist(cidc_api, clean_db, monkeypatch): """Check that getting a filelist.tsv works as expected""" user_id = setup_user(cidc_api, monkeypatch) file_id_1, file_id_2 = setup_downloadable_files(cidc_api) client = cidc_api.test_client() url = "/downloadable_files/filelist" # A JSON body containing a file ID list must be provided res = client.post(url) assert res.status_code == 422 # User has no permissions, so no files should be found short_file_list = {"file_ids": [file_id_1, file_id_2]} res = client.post(url, json=short_file_list) assert res.status_code == 404 # Give the user one permission with cidc_api.app_context(): perm = Permissions( granted_to_user=user_id, trial_id=trial_id_1, upload_type=upload_types[0], granted_by_user=user_id, ) perm.insert() # User has one permission, so the filelist should contain a single file res = client.post(url, json=short_file_list) assert res.status_code == 200 assert "text/tsv" in res.headers["Content-Type"] assert "filename=filelist.tsv" in res.headers["Content-Disposition"] assert res.data.decode("utf-8") == ( f"gs://{GOOGLE_DATA_BUCKET}/{trial_id_1}/wes/.../reads_123.bam\t{trial_id_1}_wes_..._reads_123.bam\n" ) # Admins don't need permissions to get files make_admin(user_id, cidc_api) res = client.post(url, json=short_file_list) assert res.status_code == 200 assert res.data.decode("utf-8") == ( f"gs://{GOOGLE_DATA_BUCKET}/{trial_id_1}/wes/.../reads_123.bam\t{trial_id_1}_wes_..._reads_123.bam\n" f"gs://{GOOGLE_DATA_BUCKET}/{trial_id_2}/cytof/.../analysis.zip\t{trial_id_2}_cytof_..._analysis.zip\n" ) # Clear inserted file records with cidc_api.app_context(): clean_db.query(DownloadableFiles).delete() # Filelists don't get paginated ids = [] with cidc_api.app_context(): for id in range(1000): df = DownloadableFiles( trial_id=trial_id_1, object_url=str(id), upload_type="", file_size_bytes=0, uploaded_timestamp=datetime.now(), ) df.insert() ids.append(df.id) res = client.post(url, json={"file_ids": ids}) assert res.status_code == 200 # newly inserted files + EOF newline assert len(res.data.decode("utf-8").split("\n")) == len(ids) + 1
def assert_sqla_matching_fields(rec1, rec2): received = rec1.__dict__ expected = rec2.__dict__ received.pop("_sa_instance_state") expected.pop("_sa_instance_state") assert received == expected perm_json = { "upload_type": "olink", "granted_by_user": 1, "granted_to_user": 2, "trial_id": "sometrial", } perm_record = Permissions(**perm_json) def test_unmarshal_request(empty_app: Flask): """Check that unmarshal_request validates and loads request JSON as expected.""" s, p, a = 1, 2, 3 @unmarshal_request(PermissionSchema(), "permission_record", load_sqla=True) def endpoint_sqla(some, positional, args, permission_record): assert some == s and positional == p and args == a return permission_record @unmarshal_request(PermissionSchema(), "permission_record", load_sqla=False) def endpoint_json(some, positional, args, permission_record):
def test_list_trials(cidc_api, clean_db, monkeypatch): """Check that listing trials works as expected""" mock_gcloud_client(monkeypatch) user_id = setup_user(cidc_api, monkeypatch) trial_1, trial_2 = setup_trial_metadata(cidc_api, user_id) client = cidc_api.test_client() # A CIMAC user can list trials that they're allowed to see via # granular permissions res = client.get("/trial_metadata") assert res.status_code == 200 assert len(res.json["_items"]) == 1 assert res.json["_items"][0]["id"] == trial_1 assert "file_bundle" not in res.json["_items"][0] assert "num_participants" not in res.json["_items"][0] assert "num_samples" not in res.json["_items"][0] # A CIMAC user with a cross-trial permission can list all trials with cidc_api.app_context(): Permissions(granted_by_user=user_id, granted_to_user=user_id, upload_type="ihc").insert() res = client.get("/trial_metadata") assert res.status_code == 200 assert len(res.json["_items"]) == 2 # Allowed users can get all trials for role in trial_modifier_roles: make_role(user_id, role, cidc_api) res = client.get("/trial_metadata") assert res.status_code == 200 assert len(res.json["_items"]) == 2 assert res.json["_meta"]["total"] == 2 assert set([t["id"] for t in res.json["_items"]]) == set([trial_1, trial_2]) assert not any("file_bundle" in t for t in res.json["_items"]) # Passing the URL param include_file_bundles=true works on an # as-available basis - if trials have no files associated with them, # they won't have a file bundle in the response res = client.get("/trial_metadata?include_file_bundles=true") assert res.status_code == 200 assert len(res.json["_items"]) == 2 assert "file_bundle" not in res.json["_items"][0] # Add some files... with cidc_api.app_context(): # for trial 1 for id, (type, facet_group) in enumerate([ ("cytof_10021_9204", "/cytof_10021_9204/spike_in.fcs"), ("cytof_10021_9204", "/cytof_10021_9204/source_.fcs"), ( "cytof_10021_9204", "/cytof_analysis/combined_cell_counts_profiling.csv", ), ("wes", "/wes/r1_L.fastq.gz"), ]): DownloadableFiles( id=id, trial_id="test-trial-1", facet_group=facet_group, object_url=f"test-trial-1/{facet_group}", upload_type=type, file_size_bytes=0, uploaded_timestamp=datetime.now(), ).insert() # for trial 2 for id_minus_4, (type, facet_group) in enumerate([ ("participants info", "csv|participants info"), ("mif", "/mif/roi_/cell_seg_data.txt"), ]): DownloadableFiles( id=id_minus_4 + 4, trial_id="test-trial-2", facet_group=facet_group, object_url=f"test-trial-2/{facet_group}", upload_type=type, file_size_bytes=0, uploaded_timestamp=datetime.now(), ).insert() # Listing trials with populated file bundles (also, check that sorting and counting participants/samples works) res = client.get( "/trial_metadata?include_file_bundles=true&include_counts=true&sort_field=trial_id&sort_direction=asc" ) assert res.status_code == 200 assert len(res.json["_items"]) == 2 assert res.json["_items"][0] [trial_json_1, trial_json_2] = res.json["_items"] assert set(trial_json_1["file_bundle"]["CyTOF"]["source"]) == set([0, 1]) assert trial_json_1["file_bundle"]["CyTOF"]["analysis"] == [2] assert trial_json_1["file_bundle"]["WES"]["source"] == [3] assert trial_json_1["num_samples"] == 1 assert trial_json_1["num_participants"] == 1 assert trial_json_2["file_bundle"]["Participants Info"]["clinical"] == [4] assert trial_json_2["file_bundle"]["mIF"]["analysis"] == [5] assert trial_json_2["num_samples"] == 0 assert trial_json_2["num_participants"] == 0 # Filtering by trial id seems to work when file bundles are included res = client.get( "/trial_metadata?include_file_bundles=true&trial_ids=test-trial-1") assert res.status_code == 200 assert len(res.json["_items"]) == 1 assert res.json["_items"][0]["trial_id"] == "test-trial-1" # Pagination seems to work when file bundles are included res = client.get("/trial_metadata?include_file_bundles=true&page_size=1") assert res.status_code == 200 assert len(res.json["_items"]) == 1 # Metadata blobs are pruned as expected res = client.get("/trial_metadata") assert res.status_code == 200 metadata_json = res.json["_items"][0]["metadata_json"] assert metadata_json.get("participants") is None assert metadata_json.get("assays") is None assert metadata_json.get("analysis") is None assert metadata_json.get("shipments") is None
def test_permissions_delete(clean_db, monkeypatch, caplog): gcloud_client = mock_gcloud_client(monkeypatch) user = Users(email="*****@*****.**") user.insert() trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA) trial.insert() perm = Permissions( granted_to_user=user.id, trial_id=trial.trial_id, upload_type="wes_bam", granted_by_user=user.id, ) perm.insert() # Deleting a record by a user doesn't exist leads to an error gcloud_client.reset_mocks() with pytest.raises(NoResultFound, match="no user with id"): perm.delete(deleted_by=999999) # Deletion of an existing permission leads to no error gcloud_client.reset_mocks() with caplog.at_level(logging.DEBUG): perm.delete(deleted_by=user.id) gcloud_client.revoke_download_access.assert_called_once() gcloud_client.grant_download_access.assert_not_called() assert any( log_record.message.strip() == f"admin-action: {user.email} removed from {user.email} the permission wes_bam on {trial.trial_id}" for log_record in caplog.records ) # Deleting an already-deleted record is idempotent gcloud_client.reset_mocks() perm.delete(deleted_by=user) gcloud_client.revoke_download_access.assert_called_once() gcloud_client.grant_download_access.assert_not_called() # Deleting a record whose user doesn't exist leads to an error gcloud_client.reset_mocks() with pytest.raises(NoResultFound, match="no user with id"): Permissions(granted_to_user=999999).delete(deleted_by=user) gcloud_client.revoke_download_access.assert_not_called() gcloud_client.grant_download_access.assert_not_called() # If revoking a permission from a "network-viewer", no GCS IAM actions are taken gcloud_client.revoke_download_access.reset_mock() user.role = CIDCRole.NETWORK_VIEWER.value user.update() perm = Permissions( granted_to_user=user.id, trial_id=trial.trial_id, upload_type="ihc", granted_by_user=user.id, ) perm.insert() perm.delete(deleted_by=user) gcloud_client.revoke_download_access.assert_not_called()
def test_permissions_insert(clean_db, monkeypatch, caplog): gcloud_client = mock_gcloud_client(monkeypatch) user = Users(email="*****@*****.**") user.insert() trial = TrialMetadata(trial_id=TRIAL_ID, metadata_json=METADATA) trial.insert() _insert = MagicMock() monkeypatch.setattr(CommonColumns, "insert", _insert) # if upload_type is invalid with pytest.raises(ValueError, match="invalid upload type"): Permissions(upload_type="foo", granted_to_user=user.id, trial_id=trial.trial_id) # if don't give granted_by_user perm = Permissions( granted_to_user=user.id, trial_id=trial.trial_id, upload_type="wes_bam" ) with pytest.raises(IntegrityError, match="`granted_by_user` user must be given"): perm.insert() _insert.assert_not_called() # if give bad granted_by_user _insert.reset_mock() perm = Permissions( granted_to_user=user.id, trial_id=trial.trial_id, upload_type="wes_bam", granted_by_user=999999, ) with pytest.raises(IntegrityError, match="`granted_by_user` user must exist"): perm.insert() _insert.assert_not_called() # if give bad granted_to_user _insert.reset_mock() perm = Permissions( granted_to_user=999999, trial_id=trial.trial_id, upload_type="wes_bam", granted_by_user=user.id, ) with pytest.raises(IntegrityError, match="`granted_to_user` user must exist"): perm.insert() _insert.assert_not_called() # This one will work _insert.reset_mock() perm = Permissions( granted_to_user=user.id, trial_id=trial.trial_id, upload_type="wes_bam", granted_by_user=user.id, ) with caplog.at_level(logging.DEBUG): perm.insert() _insert.assert_called_once() assert any( log_record.message.strip() == f"admin-action: {user.email} gave {user.email} the permission wes_bam on {trial.trial_id}" for log_record in caplog.records ) gcloud_client.grant_download_access.assert_called_once() # If granting a permission to a "network-viewer", no GCS IAM actions are taken _insert.reset_mock() gcloud_client.grant_download_access.reset_mock() user.role = CIDCRole.NETWORK_VIEWER.value user.update() perm = Permissions( granted_to_user=user.id, trial_id=trial.trial_id, upload_type="ihc", granted_by_user=user.id, ) perm.insert() _insert.assert_called_once() gcloud_client.grant_download_access.assert_not_called()
def test_create_permission(cidc_api, clean_db, monkeypatch): """Check that creating a new permission works as expected.""" gcloud_client = mock_gcloud_client(monkeypatch) current_user_id, other_user_id = setup_permissions(cidc_api, monkeypatch) client = cidc_api.test_client() # Non-admins should be blocked from posting to this endpoint gcloud_client.reset_mocks() res = client.post("permissions") assert res.status_code == 401 assert "not authorized to access this endpoint" in res.json["_error"][ "message"] gcloud_client.grant_download_access.assert_not_called() gcloud_client.revoke_download_access.assert_not_called() make_admin(current_user_id, cidc_api) perm = { "granted_to_user": other_user_id, "trial_id": TRIAL_ID, "upload_type": "ihc", } # When an IAM grant error occurs, the permission db record shouldn't be created gcloud_client.reset_mocks() gcloud_client.grant_download_access.side_effect = Exception("oops") res = client.post("permissions", json=perm) assert "IAM grant failed" in res.json["_error"]["message"] assert res.status_code == 500 with cidc_api.app_context(): assert clean_db.query(Permissions).filter_by(**perm).all() == [] gcloud_client.grant_download_access.side_effect = None # Admins can't create permissions with invalid upload types gcloud_client.reset_mocks() res = client.post("permissions", json={**perm, "upload_type": "foo"}) assert res.status_code == 422 assert "invalid upload type: foo" in res.json["_error"]["message"] # Admins should be able to create new permissions gcloud_client.reset_mocks() res = client.post("permissions", json=perm) assert res.status_code == 201 assert "id" in res.json assert {**res.json, **perm} == res.json with cidc_api.app_context(): assert Permissions.find_by_id(res.json["id"]) gcloud_client.grant_download_access.assert_called_once() gcloud_client.revoke_download_access.assert_not_called() # Re-insertion is not allowed gcloud_client.reset_mocks() res = client.post("permissions", json=perm) assert res.status_code == 400 assert "unique constraint" in res.json["_error"]["message"] gcloud_client.grant_download_access.assert_not_called() gcloud_client.revoke_download_access.assert_not_called() # The permission grantee must exist gcloud_client.reset_mocks() perm["granted_to_user"] = 999999999 # user doesn't exist res = client.post("permissions", json=perm) assert res.status_code == 400 assert "user must exist, but no user found" in res.json["_error"][ "message"] gcloud_client.grant_download_access.assert_not_called() gcloud_client.revoke_download_access.assert_not_called() with cidc_api.app_context(): clean_db.query(Permissions).delete() clean_db.commit() # The permission grantee must have <= GOOGLE_MAX_DOWNLOAD_PERMISSIONS perm["granted_to_user"] = current_user_id inserts_fail_eventually = False upload_types = list(ALL_UPLOAD_TYPES) for i in range(GOOGLE_MAX_DOWNLOAD_PERMISSIONS + 1): gcloud_client.reset_mocks() perm["upload_type"] = upload_types[i] res = client.post("permissions", json=perm) if res.status_code != 201: assert res.status_code == 400 assert ( "greater than or equal to the maximum number of allowed granular permissions" in res.json["_error"]["message"]) gcloud_client.grant_download_access.assert_not_called() gcloud_client.revoke_download_access.assert_not_called() inserts_fail_eventually = True break assert inserts_fail_eventually
def test_delete_permission(cidc_api, clean_db, monkeypatch): """Check that deleting a permission works as expected.""" gcloud_client = mock_gcloud_client(monkeypatch) current_user_id, other_user_id = setup_permissions(cidc_api, monkeypatch) with cidc_api.app_context(): perm = Permissions.find_for_user(current_user_id)[0] client = cidc_api.test_client() # Non-admins are not allowed to delete gcloud_client.reset_mock() res = client.delete(f"permissions/{perm.id}") assert res.status_code == 401 assert "not authorized to access this endpoint" in res.json["_error"][ "message"] gcloud_client.grant_download_access.assert_not_called() gcloud_client.revoke_download_access.assert_not_called() make_admin(current_user_id, cidc_api) # Requester must supply an If-Match header gcloud_client.reset_mock() res = client.delete(f"permissions/{perm.id}") assert res.status_code == 428 gcloud_client.grant_download_access.assert_not_called() gcloud_client.revoke_download_access.assert_not_called() headers = {"If-Match": "foobar"} # Returns NotFound if no record exists gcloud_client.reset_mock() res = client.delete(f"permissions/1232123", headers=headers) assert res.status_code == 404 gcloud_client.grant_download_access.assert_not_called() gcloud_client.revoke_download_access.assert_not_called() # A mismatched ETag leads to a PreconditionFailed error gcloud_client.reset_mock() res = client.delete(f"permissions/{perm.id}", headers=headers) assert res.status_code == 412 gcloud_client.grant_download_access.assert_not_called() gcloud_client.revoke_download_access.assert_not_called() headers["If-Match"] = perm._etag # A well-formed delete request fails if IAM revoke fails gcloud_client.reset_mock() gcloud_client.revoke_download_access.side_effect = Exception("oops") res = client.delete(f"permissions/{perm.id}", headers=headers) assert "IAM revoke failed" in res.json["_error"]["message"] assert res.status_code == 500 with cidc_api.app_context(): assert Permissions.find_by_id(perm.id) is not None gcloud_client.revoke_download_access.side_effect = None # A matching ETag leads to a successful deletion gcloud_client.reset_mock() res = client.delete(f"permissions/{perm.id}", headers=headers) assert res.status_code == 204 with cidc_api.app_context(): assert Permissions.find_by_id(perm.id) is None gcloud_client.grant_download_access.assert_not_called() gcloud_client.revoke_download_access.assert_called_once()