def test_common_insert(clean_db): """Test insert, inherited from CommonColumns""" # Check disabling committing u1 = Users(email="a") u1.insert(commit=False) assert not u1.id # Insert a new record without disabling committing u2 = Users(email="b") u2.insert() assert u1.id and u1._etag assert u2.id and u2._etag assert u1._etag != u2._etag assert Users.find_by_id(u1.id) assert Users.find_by_id(u2.id)
def register_user(user_id, app): """Register the given user as a cimac-user.""" with app.app_context(): user = Users.find_by_id(user_id) user.approval_date = datetime.now() user.role = CIDCRole.CIMAC_USER.value user.update()
def test_poll_upload_merge_status(cidc_api, clean_db, monkeypatch): """ Check pull_upload_merge_status endpoint behavior """ user_id = setup_trial_and_user(cidc_api, monkeypatch) with cidc_api.app_context(): user = Users.find_by_id(user_id) make_cimac_biofx_user(user_id, cidc_api) metadata = {PROTOCOL_ID_FIELD_NAME: trial_id} with cidc_api.app_context(): other_user = Users(email="*****@*****.**") other_user.insert() upload_job = UploadJobs.create( upload_type="wes", uploader_email=user.email, gcs_file_map={}, metadata=metadata, gcs_xlsx_uri="", ) upload_job.insert() upload_job_id = upload_job.id client = cidc_api.test_client() # Upload not found res = client.get( f"/ingestion/poll_upload_merge_status/12345?token={upload_job.token}") assert res.status_code == 404 upload_job_url = ( f"/ingestion/poll_upload_merge_status/{upload_job_id}?token={upload_job.token}" ) # Upload not-yet-ready res = client.get(upload_job_url) assert res.status_code == 200 assert "retry_in" in res.json and res.json["retry_in"] == 5 assert "status" not in res.json test_details = "A human-friendly reason for this " for status in [ UploadJobStatus.MERGE_COMPLETED.value, UploadJobStatus.MERGE_FAILED.value, ]: # Simulate cloud function merge status update with cidc_api.app_context(): upload_job._set_status_no_validation(status) upload_job.status_details = test_details upload_job.update() # Upload ready res = client.get(upload_job_url) assert res.status_code == 200 assert "retry_in" not in res.json assert "status" in res.json and res.json["status"] == status assert ("status_details" in res.json and res.json["status_details"] == test_details)
def test_merge_extra_metadata(cidc_api, clean_db, monkeypatch): """Ensure merging of extra metadata follows the expected execution flow""" user_id = setup_trial_and_user(cidc_api, monkeypatch) with cidc_api.app_context(): user = Users.find_by_id(user_id) make_cimac_biofx_user(user_id, cidc_api) with cidc_api.app_context(): assay_upload = UploadJobs.create( upload_type="assay_with_extra_md", uploader_email=user.email, gcs_file_map={}, metadata={ PROTOCOL_ID_FIELD_NAME: trial_id, "whatever": { "hierarchy": [ { "we just need a": "uuid-1", "to be able": "to merge" }, { "and": "uuid-2" }, ] }, }, gcs_xlsx_uri="", commit=False, ) assay_upload.id = 137 assay_upload.insert() custom_extra_md_parse = MagicMock() custom_extra_md_parse.side_effect = lambda f: { "extra_md": f.read().decode() } monkeypatch.setattr( "cidc_schemas.prism.merger.EXTRA_METADATA_PARSERS", {"assay_with_extra_md": custom_extra_md_parse}, ) form_data = { "job_id": 137, "uuid-1": (io.BytesIO(b"fake file 1"), "fname1"), "uuid-2": (io.BytesIO(b"fake file 2"), "fname2"), } client = cidc_api.test_client() res = client.post("/ingestion/extra-assay-metadata", data=form_data) assert res.status_code == 200 assert custom_extra_md_parse.call_count == 2 fetched_jobs = UploadJobs.list() assert 1 == len(fetched_jobs) au = fetched_jobs[0] assert "extra_md" in au.metadata_patch["whatever"]["hierarchy"][0] assert "extra_md" in au.metadata_patch["whatever"]["hierarchy"][1]
def test_common_delete(clean_db): """Test delete, inherited from CommonColumns""" user1 = Users(email="foo") user2 = Users(email="bar") # Try to delete an uninserted record with pytest.raises(InvalidRequestError): user1.delete() user1.insert() user2.insert() # Defer a deletion with commit=False user1.delete(commit=False) assert Users.find_by_id(user1.id) # Delete with auto-commit user2.delete() assert not Users.find_by_id(user1.id) assert not Users.find_by_id(user2.id)
def test_common_update(clean_db): """Test update, inherited from CommonColumns""" email = "foo" user = Users(id=1, email=email) # Record not found with pytest.raises(NoResultFound): user.update() user.insert() _updated = user._updated # Update via setattr and changes first_n = "hello" last_n = "goodbye" user.last_n = last_n user.update(changes={"first_n": first_n}) user = Users.find_by_id(user.id) assert user._updated > _updated assert user.first_n == first_n assert user.last_n == last_n _updated = user._updated _etag = user._etag # Make sure you can clear a field to null user.update(changes={"first_n": None}) user = Users.find_by_id(user.id) assert user._updated > _updated assert _etag != user._etag assert user.first_n is None _updated = user._updated _etag = user._etag # Make sure etags don't change if public fields don't change user.update() user = Users.find_by_id(user.id) assert user._updated > _updated assert _etag == user._etag
def test_get_self(cidc_api, clean_db, monkeypatch): """Check that get self returns the current user's info.""" user_id, _ = setup_users(cidc_api, monkeypatch, registered=False) with cidc_api.app_context(): user = Users.find_by_id(user_id) client = cidc_api.test_client() res = client.get("users/self") assert res.status_code == 200 assert res.json == UserSchema().dump(user)
def test_create_user(cidc_api, clean_db, monkeypatch): """Check that only admins can create arbitrary users.""" user_id, other_user_id = setup_users(cidc_api, monkeypatch) with cidc_api.app_context(): dup_email = Users.find_by_id(other_user_id).email client = cidc_api.test_client() dup_user_json = {"email": dup_email} new_user_json = {"email": "*****@*****.**"} # Registered users who aren't admins can't create arbitrary users res = client.post("users", json=new_user_json) assert res.status_code == 401 # Users who are admins can create arbitrary users make_admin(user_id, cidc_api) res = client.post("users", json=new_user_json) assert res.status_code == 201 # Even admins can't create users with duplicate emails res = client.post("users", json=dup_user_json) assert res.status_code == 400
def test_get_user(cidc_api, clean_db, monkeypatch): """Check that getting users by ID works as expected.""" user_id, other_user_id = setup_users(cidc_api, monkeypatch, registered=True) client = cidc_api.test_client() # Non-admins can't get themselves or other users by their IDs assert client.get(f"/users/{user_id}").status_code == 401 assert client.get(f"/users/{other_user_id}").status_code == 401 # Admins can get users by their IDs make_admin(user_id, cidc_api) with cidc_api.app_context(): other_user = Users.find_by_id(other_user_id) res = client.get(f"/users/{other_user_id}") assert res.status_code == 200 assert res.json == UserSchema().dump(other_user) # Trying to get a non-existing user yields 404 res = client.get(f"/users/123212321") assert res.status_code == 404
def test_upload_olink(cidc_api, clean_db, monkeypatch): """Ensure the upload endpoint follows the expected execution flow""" user_id = setup_trial_and_user(cidc_api, monkeypatch) with cidc_api.app_context(): user = Users.find_by_id(user_id) make_cimac_biofx_user(user_id, cidc_api) client = cidc_api.test_client() mocks = UploadMocks( monkeypatch, prismify_file_entries=[ finfo(lp, url, "uuid" + str(i), "npx" in url, False) for i, (lp, url) in enumerate(OLINK_TESTDATA) ], ) # No permission to upload yet res = client.post(ASSAY_UPLOAD, data=form_data("olink.xlsx", io.BytesIO(b"1234"), "olink")) assert res.status_code == 401 assert "not authorized to upload olink data" in str( res.json["_error"]["message"]) mocks.clear_all() # Give permission and retry grant_upload_permission(user_id, "olink", cidc_api) res = client.post(ASSAY_UPLOAD, data=form_data("olink.xlsx", io.BytesIO(b"1234"), "olink")) assert res.status_code == 200 assert "url_mapping" in res.json url_mapping = res.json["url_mapping"] # Olink assay has extra_metadata files assert "extra_metadata" in res.json extra_metadata = res.json["extra_metadata"] assert type(extra_metadata) == dict # We expect local_path to map to a gcs object name with gcs_prefix. for local_path, gcs_prefix in OLINK_TESTDATA: gcs_object_name = url_mapping[local_path] assert local_path in url_mapping assert gcs_object_name.startswith(gcs_prefix) assert (local_path not in gcs_object_name ), "PHI from local_path shouldn't end up in gcs urls" # Check that we tried to grant IAM upload access to gcs_object_name mocks.grant_write.assert_called_with(user.email) # Check that we tried to upload the assay metadata excel file mocks.upload_xlsx.assert_called_once() job_id = res.json["job_id"] update_url = f"/upload_jobs/{job_id}" # Report an upload failure res = client.patch( f"{update_url}?token={res.json['token']}", json={"status": UploadJobStatus.UPLOAD_FAILED.value}, headers={"If-Match": res.json["job_etag"]}, ) assert res.status_code == 200 mocks.revoke_write.assert_called_with(user.email) # This was an upload failure, so success shouldn't have been published mocks.publish_success.assert_not_called() # Test upload status validation - since the upload job's current status # is UPLOAD_FAILED, the API shouldn't permit this status to be updated to # UPLOAD_COMPLETED. bad_res = client.patch( f"{update_url}?token={res.json['token']}", json={"status": UploadJobStatus.UPLOAD_COMPLETED.value}, headers={"If-Match": res.json["_etag"]}, ) assert bad_res.status_code == 400 assert ("status upload-failed can't transition to status upload-completed" in bad_res.json["_error"]["message"]) # Reset the upload status and try the request again with cidc_api.app_context(): job = UploadJobs.find_by_id_and_email(job_id, user.email) job._set_status_no_validation(UploadJobStatus.STARTED.value) job.update() _etag = job._etag res = client.patch( f"{update_url}?token={res.json['token']}", json={"status": UploadJobStatus.UPLOAD_COMPLETED.value}, headers={"If-Match": _etag}, ) assert res.status_code == 200 mocks.publish_success.assert_called_with(job_id)
def test_upload_wes(cidc_api, clean_db, monkeypatch): """Ensure the upload endpoint follows the expected execution flow""" user_id = setup_trial_and_user(cidc_api, monkeypatch) make_cimac_biofx_user(user_id, cidc_api) with cidc_api.app_context(): user = Users.find_by_id(user_id) client = cidc_api.test_client() mocks = UploadMocks( monkeypatch, prismify_file_entries=[ finfo("localfile.ext", "test_trial/url/file.ext", "uuid-1", None, False) ], ) # No permission to upload yet res = client.post(ASSAY_UPLOAD, data=form_data("wes.xlsx", io.BytesIO(b"1234"), "wes_fastq")) assert res.status_code == 401 assert "not authorized to upload wes_fastq data" in str( res.json["_error"]["message"]) mocks.clear_all() # Give permission and retry grant_upload_permission(user_id, "wes_fastq", cidc_api) res = client.post(ASSAY_UPLOAD, data=form_data("wes.xlsx", io.BytesIO(b"1234"), "wes_fastq")) assert res.status_code == 200 assert "url_mapping" in res.json url_mapping = res.json["url_mapping"] # WES assay does not have any extra_metadata files, but its (and every assay's) response # should have an extra_metadata field. assert "extra_metadata" in res.json extra_metadata = res.json["extra_metadata"] assert extra_metadata is None # We expect local_path to map to a gcs object name with gcs_prefix local_path = "localfile.ext" gcs_prefix = "test_trial/url/file.ext" gcs_object_name = url_mapping[local_path] assert local_path in url_mapping assert gcs_object_name.startswith(gcs_prefix) assert not gcs_object_name.endswith( local_path), "PHI from local_path shouldn't end up in gcs urls" # Check that we tried to grant IAM upload access to gcs_object_name mocks.grant_write.assert_called_with(user.email) # Check that we tried to upload the assay metadata excel file mocks.upload_xlsx.assert_called_once() job_id = res.json["job_id"] update_url = f"/upload_jobs/{job_id}" # Report an upload failure res = client.patch( f"{update_url}?token={res.json['token']}", json={"status": UploadJobStatus.UPLOAD_FAILED.value}, headers={"If-Match": res.json["job_etag"]}, ) assert res.status_code == 200 mocks.revoke_write.assert_called_with(user.email) # This was an upload failure, so success shouldn't have been published mocks.publish_success.assert_not_called() # Reset the upload status and try the request again with cidc_api.app_context(): job = UploadJobs.find_by_id_and_email(job_id, user.email) job._set_status_no_validation(UploadJobStatus.STARTED.value) job.update() _etag = job._etag # Report an upload success res = client.patch( f"{update_url}?token={res.json['token']}", json={"status": UploadJobStatus.UPLOAD_COMPLETED.value}, headers={"If-Match": _etag}, ) assert res.status_code == 200 mocks.publish_success.assert_called_with(job_id)
def make_cimac_biofx_user(user_id, cidc_api): with cidc_api.app_context(): user = Users.find_by_id(user_id) user.role = CIDCRole.CIMAC_BIOFX_USER.value user.update()
def make_nci_biobank_user(user_id, cidc_api): with cidc_api.app_context(): user = Users.find_by_id(user_id) user.role = CIDCRole.NCI_BIOBANK_USER.value user.update()
def test_update_user(cidc_api, clean_db, monkeypatch): """Check that updating users works as expected.""" user_id, other_user_id = setup_users(cidc_api, monkeypatch, registered=True) with cidc_api.app_context(): user = Users.find_by_id(user_id) other_user = Users.find_by_id(other_user_id) client = cidc_api.test_client() patch = {"role": "cidc-admin"} # Test that non-admins can't modify anyone res = client.patch(f"/users/{user.id}") assert res.status_code == 401 res = client.patch(f"/users/{other_user.id}") assert res.status_code == 401 make_admin(user_id, cidc_api) # A missing ETag blocks an update res = client.patch(f"/users/{other_user.id}") assert res.status_code == 428 # An incorrect ETag blocks an update res = client.patch(f"/users/{other_user.id}", headers={"If-Match": "foo"}) assert res.status_code == 412 # An admin can successfully update a user res = client.patch(f"/users/{other_user.id}", headers={"If-Match": other_user._etag}, json=patch) assert res.status_code == 200 assert res.json["id"] == other_user.id assert res.json["email"] == other_user.email assert res.json["role"] == "cidc-admin" assert res.json["approval_date"] is not None _accessed = res.json["_accessed"] # Reenabling a disabled user updates that user's last access date. mock_permissions = MagicMock() mock_permissions.grant_iam_permissions = MagicMock() monkeypatch.setattr("cidc_api.resources.users.Permissions", mock_permissions) res = client.patch( f"/users/{other_user.id}", headers={"If-Match": res.json["_etag"]}, json={"disabled": True}, ) assert res.status_code == 200 res = client.patch( f"/users/{other_user.id}", headers={"If-Match": res.json["_etag"]}, json={"disabled": False}, ) assert res.status_code == 200 assert res.json["_accessed"] > _accessed mock_permissions.grant_iam_permissions.assert_called() # Trying to update a non-existing user yields 404 res = client.patch(f"/users/123212321", headers={"If-Match": other_user._etag}, json=patch) assert res.status_code == 404
def make_role(user_id, role, app): """Update the user with id `user_id`'s role to `role`.""" with app.app_context(): user = Users.find_by_id(user_id) user.role = role user.update()