def test_common_insert(clean_db):
"""Test insert, inherited from CommonColumns"""
# Check disabling committing
u1 = Users(email="a")
u1.insert(commit=False)
assert not u1.id
# Insert a new record without disabling committing
u2 = Users(email="b")
u2.insert()
assert u1.id and u1._etag
assert u2.id and u2._etag
assert u1._etag != u2._etag
assert Users.find_by_id(u1.id)
assert Users.find_by_id(u2.id)
def register_user(user_id, app):
"""Register the given user as a cimac-user."""
with app.app_context():
user = Users.find_by_id(user_id)
user.approval_date = datetime.now()
user.role = CIDCRole.CIMAC_USER.value
user.update()
def test_poll_upload_merge_status(cidc_api, clean_db, monkeypatch):
"""
Check pull_upload_merge_status endpoint behavior
"""
user_id = setup_trial_and_user(cidc_api, monkeypatch)
with cidc_api.app_context():
user = Users.find_by_id(user_id)
make_cimac_biofx_user(user_id, cidc_api)
metadata = {PROTOCOL_ID_FIELD_NAME: trial_id}
with cidc_api.app_context():
other_user = Users(email="*****@*****.**")
other_user.insert()
upload_job = UploadJobs.create(
upload_type="wes",
uploader_email=user.email,
gcs_file_map={},
metadata=metadata,
gcs_xlsx_uri="",
)
upload_job.insert()
upload_job_id = upload_job.id
client = cidc_api.test_client()
# Upload not found
res = client.get(
f"/ingestion/poll_upload_merge_status/12345?token={upload_job.token}")
assert res.status_code == 404
upload_job_url = (
f"/ingestion/poll_upload_merge_status/{upload_job_id}?token={upload_job.token}"
)
# Upload not-yet-ready
res = client.get(upload_job_url)
assert res.status_code == 200
assert "retry_in" in res.json and res.json["retry_in"] == 5
assert "status" not in res.json
test_details = "A human-friendly reason for this "
for status in [
UploadJobStatus.MERGE_COMPLETED.value,
UploadJobStatus.MERGE_FAILED.value,
]:
# Simulate cloud function merge status update
with cidc_api.app_context():
upload_job._set_status_no_validation(status)
upload_job.status_details = test_details
upload_job.update()
# Upload ready
res = client.get(upload_job_url)
assert res.status_code == 200
assert "retry_in" not in res.json
assert "status" in res.json and res.json["status"] == status
assert ("status_details" in res.json
and res.json["status_details"] == test_details)
def test_merge_extra_metadata(cidc_api, clean_db, monkeypatch):
"""Ensure merging of extra metadata follows the expected execution flow"""
user_id = setup_trial_and_user(cidc_api, monkeypatch)
with cidc_api.app_context():
user = Users.find_by_id(user_id)
make_cimac_biofx_user(user_id, cidc_api)
with cidc_api.app_context():
assay_upload = UploadJobs.create(
upload_type="assay_with_extra_md",
uploader_email=user.email,
gcs_file_map={},
metadata={
PROTOCOL_ID_FIELD_NAME: trial_id,
"whatever": {
"hierarchy": [
{
"we just need a": "uuid-1",
"to be able": "to merge"
},
{
"and": "uuid-2"
},
]
},
},
gcs_xlsx_uri="",
commit=False,
)
assay_upload.id = 137
assay_upload.insert()
custom_extra_md_parse = MagicMock()
custom_extra_md_parse.side_effect = lambda f: {
"extra_md": f.read().decode()
}
monkeypatch.setattr(
"cidc_schemas.prism.merger.EXTRA_METADATA_PARSERS",
{"assay_with_extra_md": custom_extra_md_parse},
)
form_data = {
"job_id": 137,
"uuid-1": (io.BytesIO(b"fake file 1"), "fname1"),
"uuid-2": (io.BytesIO(b"fake file 2"), "fname2"),
}
client = cidc_api.test_client()
res = client.post("/ingestion/extra-assay-metadata", data=form_data)
assert res.status_code == 200
assert custom_extra_md_parse.call_count == 2
fetched_jobs = UploadJobs.list()
assert 1 == len(fetched_jobs)
au = fetched_jobs[0]
assert "extra_md" in au.metadata_patch["whatever"]["hierarchy"][0]
assert "extra_md" in au.metadata_patch["whatever"]["hierarchy"][1]
def test_common_delete(clean_db):
"""Test delete, inherited from CommonColumns"""
user1 = Users(email="foo")
user2 = Users(email="bar")
# Try to delete an uninserted record
with pytest.raises(InvalidRequestError):
user1.delete()
user1.insert()
user2.insert()
# Defer a deletion with commit=False
user1.delete(commit=False)
assert Users.find_by_id(user1.id)
# Delete with auto-commit
user2.delete()
assert not Users.find_by_id(user1.id)
assert not Users.find_by_id(user2.id)
def test_common_update(clean_db):
"""Test update, inherited from CommonColumns"""
email = "foo"
user = Users(id=1, email=email)
# Record not found
with pytest.raises(NoResultFound):
user.update()
user.insert()
_updated = user._updated
# Update via setattr and changes
first_n = "hello"
last_n = "goodbye"
user.last_n = last_n
user.update(changes={"first_n": first_n})
user = Users.find_by_id(user.id)
assert user._updated > _updated
assert user.first_n == first_n
assert user.last_n == last_n
_updated = user._updated
_etag = user._etag
# Make sure you can clear a field to null
user.update(changes={"first_n": None})
user = Users.find_by_id(user.id)
assert user._updated > _updated
assert _etag != user._etag
assert user.first_n is None
_updated = user._updated
_etag = user._etag
# Make sure etags don't change if public fields don't change
user.update()
user = Users.find_by_id(user.id)
assert user._updated > _updated
assert _etag == user._etag
def test_get_self(cidc_api, clean_db, monkeypatch):
"""Check that get self returns the current user's info."""
user_id, _ = setup_users(cidc_api, monkeypatch, registered=False)
with cidc_api.app_context():
user = Users.find_by_id(user_id)
client = cidc_api.test_client()
res = client.get("users/self")
assert res.status_code == 200
assert res.json == UserSchema().dump(user)
def test_create_user(cidc_api, clean_db, monkeypatch):
"""Check that only admins can create arbitrary users."""
user_id, other_user_id = setup_users(cidc_api, monkeypatch)
with cidc_api.app_context():
dup_email = Users.find_by_id(other_user_id).email
client = cidc_api.test_client()
dup_user_json = {"email": dup_email}
new_user_json = {"email": "*****@*****.**"}
# Registered users who aren't admins can't create arbitrary users
res = client.post("users", json=new_user_json)
assert res.status_code == 401
# Users who are admins can create arbitrary users
make_admin(user_id, cidc_api)
res = client.post("users", json=new_user_json)
assert res.status_code == 201
# Even admins can't create users with duplicate emails
res = client.post("users", json=dup_user_json)
assert res.status_code == 400
def test_get_user(cidc_api, clean_db, monkeypatch):
"""Check that getting users by ID works as expected."""
user_id, other_user_id = setup_users(cidc_api,
monkeypatch,
registered=True)
client = cidc_api.test_client()
# Non-admins can't get themselves or other users by their IDs
assert client.get(f"/users/{user_id}").status_code == 401
assert client.get(f"/users/{other_user_id}").status_code == 401
# Admins can get users by their IDs
make_admin(user_id, cidc_api)
with cidc_api.app_context():
other_user = Users.find_by_id(other_user_id)
res = client.get(f"/users/{other_user_id}")
assert res.status_code == 200
assert res.json == UserSchema().dump(other_user)
# Trying to get a non-existing user yields 404
res = client.get(f"/users/123212321")
assert res.status_code == 404
def test_upload_olink(cidc_api, clean_db, monkeypatch):
"""Ensure the upload endpoint follows the expected execution flow"""
user_id = setup_trial_and_user(cidc_api, monkeypatch)
with cidc_api.app_context():
user = Users.find_by_id(user_id)
make_cimac_biofx_user(user_id, cidc_api)
client = cidc_api.test_client()
mocks = UploadMocks(
monkeypatch,
prismify_file_entries=[
finfo(lp, url, "uuid" + str(i), "npx" in url, False)
for i, (lp, url) in enumerate(OLINK_TESTDATA)
],
)
# No permission to upload yet
res = client.post(ASSAY_UPLOAD,
data=form_data("olink.xlsx", io.BytesIO(b"1234"),
"olink"))
assert res.status_code == 401
assert "not authorized to upload olink data" in str(
res.json["_error"]["message"])
mocks.clear_all()
# Give permission and retry
grant_upload_permission(user_id, "olink", cidc_api)
res = client.post(ASSAY_UPLOAD,
data=form_data("olink.xlsx", io.BytesIO(b"1234"),
"olink"))
assert res.status_code == 200
assert "url_mapping" in res.json
url_mapping = res.json["url_mapping"]
# Olink assay has extra_metadata files
assert "extra_metadata" in res.json
extra_metadata = res.json["extra_metadata"]
assert type(extra_metadata) == dict
# We expect local_path to map to a gcs object name with gcs_prefix.
for local_path, gcs_prefix in OLINK_TESTDATA:
gcs_object_name = url_mapping[local_path]
assert local_path in url_mapping
assert gcs_object_name.startswith(gcs_prefix)
assert (local_path not in gcs_object_name
), "PHI from local_path shouldn't end up in gcs urls"
# Check that we tried to grant IAM upload access to gcs_object_name
mocks.grant_write.assert_called_with(user.email)
# Check that we tried to upload the assay metadata excel file
mocks.upload_xlsx.assert_called_once()
job_id = res.json["job_id"]
update_url = f"/upload_jobs/{job_id}"
# Report an upload failure
res = client.patch(
f"{update_url}?token={res.json['token']}",
json={"status": UploadJobStatus.UPLOAD_FAILED.value},
headers={"If-Match": res.json["job_etag"]},
)
assert res.status_code == 200
mocks.revoke_write.assert_called_with(user.email)
# This was an upload failure, so success shouldn't have been published
mocks.publish_success.assert_not_called()
# Test upload status validation - since the upload job's current status
# is UPLOAD_FAILED, the API shouldn't permit this status to be updated to
# UPLOAD_COMPLETED.
bad_res = client.patch(
f"{update_url}?token={res.json['token']}",
json={"status": UploadJobStatus.UPLOAD_COMPLETED.value},
headers={"If-Match": res.json["_etag"]},
)
assert bad_res.status_code == 400
assert ("status upload-failed can't transition to status upload-completed"
in bad_res.json["_error"]["message"])
# Reset the upload status and try the request again
with cidc_api.app_context():
job = UploadJobs.find_by_id_and_email(job_id, user.email)
job._set_status_no_validation(UploadJobStatus.STARTED.value)
job.update()
_etag = job._etag
res = client.patch(
f"{update_url}?token={res.json['token']}",
json={"status": UploadJobStatus.UPLOAD_COMPLETED.value},
headers={"If-Match": _etag},
)
assert res.status_code == 200
mocks.publish_success.assert_called_with(job_id)
def test_upload_wes(cidc_api, clean_db, monkeypatch):
"""Ensure the upload endpoint follows the expected execution flow"""
user_id = setup_trial_and_user(cidc_api, monkeypatch)
make_cimac_biofx_user(user_id, cidc_api)
with cidc_api.app_context():
user = Users.find_by_id(user_id)
client = cidc_api.test_client()
mocks = UploadMocks(
monkeypatch,
prismify_file_entries=[
finfo("localfile.ext", "test_trial/url/file.ext", "uuid-1", None,
False)
],
)
# No permission to upload yet
res = client.post(ASSAY_UPLOAD,
data=form_data("wes.xlsx", io.BytesIO(b"1234"),
"wes_fastq"))
assert res.status_code == 401
assert "not authorized to upload wes_fastq data" in str(
res.json["_error"]["message"])
mocks.clear_all()
# Give permission and retry
grant_upload_permission(user_id, "wes_fastq", cidc_api)
res = client.post(ASSAY_UPLOAD,
data=form_data("wes.xlsx", io.BytesIO(b"1234"),
"wes_fastq"))
assert res.status_code == 200
assert "url_mapping" in res.json
url_mapping = res.json["url_mapping"]
# WES assay does not have any extra_metadata files, but its (and every assay's) response
# should have an extra_metadata field.
assert "extra_metadata" in res.json
extra_metadata = res.json["extra_metadata"]
assert extra_metadata is None
# We expect local_path to map to a gcs object name with gcs_prefix
local_path = "localfile.ext"
gcs_prefix = "test_trial/url/file.ext"
gcs_object_name = url_mapping[local_path]
assert local_path in url_mapping
assert gcs_object_name.startswith(gcs_prefix)
assert not gcs_object_name.endswith(
local_path), "PHI from local_path shouldn't end up in gcs urls"
# Check that we tried to grant IAM upload access to gcs_object_name
mocks.grant_write.assert_called_with(user.email)
# Check that we tried to upload the assay metadata excel file
mocks.upload_xlsx.assert_called_once()
job_id = res.json["job_id"]
update_url = f"/upload_jobs/{job_id}"
# Report an upload failure
res = client.patch(
f"{update_url}?token={res.json['token']}",
json={"status": UploadJobStatus.UPLOAD_FAILED.value},
headers={"If-Match": res.json["job_etag"]},
)
assert res.status_code == 200
mocks.revoke_write.assert_called_with(user.email)
# This was an upload failure, so success shouldn't have been published
mocks.publish_success.assert_not_called()
# Reset the upload status and try the request again
with cidc_api.app_context():
job = UploadJobs.find_by_id_and_email(job_id, user.email)
job._set_status_no_validation(UploadJobStatus.STARTED.value)
job.update()
_etag = job._etag
# Report an upload success
res = client.patch(
f"{update_url}?token={res.json['token']}",
json={"status": UploadJobStatus.UPLOAD_COMPLETED.value},
headers={"If-Match": _etag},
)
assert res.status_code == 200
mocks.publish_success.assert_called_with(job_id)
def make_cimac_biofx_user(user_id, cidc_api):
with cidc_api.app_context():
user = Users.find_by_id(user_id)
user.role = CIDCRole.CIMAC_BIOFX_USER.value
user.update()
def make_nci_biobank_user(user_id, cidc_api):
with cidc_api.app_context():
user = Users.find_by_id(user_id)
user.role = CIDCRole.NCI_BIOBANK_USER.value
user.update()
def test_update_user(cidc_api, clean_db, monkeypatch):
"""Check that updating users works as expected."""
user_id, other_user_id = setup_users(cidc_api,
monkeypatch,
registered=True)
with cidc_api.app_context():
user = Users.find_by_id(user_id)
other_user = Users.find_by_id(other_user_id)
client = cidc_api.test_client()
patch = {"role": "cidc-admin"}
# Test that non-admins can't modify anyone
res = client.patch(f"/users/{user.id}")
assert res.status_code == 401
res = client.patch(f"/users/{other_user.id}")
assert res.status_code == 401
make_admin(user_id, cidc_api)
# A missing ETag blocks an update
res = client.patch(f"/users/{other_user.id}")
assert res.status_code == 428
# An incorrect ETag blocks an update
res = client.patch(f"/users/{other_user.id}", headers={"If-Match": "foo"})
assert res.status_code == 412
# An admin can successfully update a user
res = client.patch(f"/users/{other_user.id}",
headers={"If-Match": other_user._etag},
json=patch)
assert res.status_code == 200
assert res.json["id"] == other_user.id
assert res.json["email"] == other_user.email
assert res.json["role"] == "cidc-admin"
assert res.json["approval_date"] is not None
_accessed = res.json["_accessed"]
# Reenabling a disabled user updates that user's last access date.
mock_permissions = MagicMock()
mock_permissions.grant_iam_permissions = MagicMock()
monkeypatch.setattr("cidc_api.resources.users.Permissions",
mock_permissions)
res = client.patch(
f"/users/{other_user.id}",
headers={"If-Match": res.json["_etag"]},
json={"disabled": True},
)
assert res.status_code == 200
res = client.patch(
f"/users/{other_user.id}",
headers={"If-Match": res.json["_etag"]},
json={"disabled": False},
)
assert res.status_code == 200
assert res.json["_accessed"] > _accessed
mock_permissions.grant_iam_permissions.assert_called()
# Trying to update a non-existing user yields 404
res = client.patch(f"/users/123212321",
headers={"If-Match": other_user._etag},
json=patch)
assert res.status_code == 404
def make_role(user_id, role, app):
"""Update the user with id `user_id`'s role to `role`."""
with app.app_context():
user = Users.find_by_id(user_id)
user.role = role
user.update()
