async def test_delete_job_script(
client,
fill_application_data,
fill_job_script_data,
inject_security_header,
):
"""
Test delete job_script via DELETE.
This test proves that a job_script is successfully deleted via a DELETE request to the /job-scripts/<id>
endpoint. We show this by asserting that the job_script no longer exists in the database after the
request is made and the correct status code is returned (204).
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(),
)
inserted_job_script_id = await database.execute(
query=job_scripts_table.insert(),
values=fill_job_script_data(application_id=inserted_application_id),
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 1
inject_security_header("*****@*****.**", Permissions.JOB_SCRIPTS_EDIT)
response = await client.delete(
f"/jobbergate/job-scripts/{inserted_job_script_id}")
assert response.status_code == status.HTTP_204_NO_CONTENT
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 0
async def test_delete_job_script_bad_permission(
client,
fill_application_data,
fill_job_script_data,
inject_security_header,
):
"""
Test that it is not possible to delete a job_script when the user don't have the permission.
This test proves that it is not possible to delete a job_script if the user don't have the permission.
We show this by assert that a 403 response status code is returned and the job_script still exists in
the database after the request.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(),
)
inserted_job_script_id = await database.execute(
query=job_scripts_table.insert(),
values=fill_job_script_data(application_id=inserted_application_id),
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 1
inject_security_header("*****@*****.**", "INVALID_PERMISSION")
response = await client.delete(
f"/jobbergate/job-scripts/{inserted_job_script_id}")
assert response.status_code == status.HTTP_403_FORBIDDEN
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 1
async def test_get_job_scripts__bad_permission(
client,
fill_application_data,
fill_job_script_data,
inject_security_header,
):
"""
Test GET /job-scripts/ returns 403 since the user don't have the proper permission.
This test proves that GET /job-scripts/ returns the 403 status code when the user making the request
don't have the permission to list. We show this by asserting that the response status code is 403.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
await database.execute(
query=job_scripts_table.insert(),
values=fill_job_script_data(application_id=inserted_application_id),
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 1
inject_security_header("*****@*****.**", "INVALID_PERMISSION")
response = await client.get("/jobbergate/job-scripts/")
assert response.status_code == status.HTTP_403_FORBIDDEN
async def test_update_job_script_bad_permission(
client,
fill_application_data,
fill_job_script_data,
inject_security_header,
):
"""
Test that it is not possible to update a job_script if the user don't have the proper permission.
This test proves that it is not possible to update a job_script if the user don't have permission. We
show this by asserting that the response status code of the request is 403, and that the data stored in
the database for the job_script is not updated.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
await database.execute(
query=job_scripts_table.insert(),
values=fill_job_script_data(job_script_name="target-js",
application_id=inserted_application_id),
)
inject_security_header("*****@*****.**", "INVALID_PERMISSION")
response = await client.put("/jobbergate/job-scripts/1",
data={"job_script_name": "new name"})
assert response.status_code == status.HTTP_403_FORBIDDEN
query = job_scripts_table.select(
job_scripts_table.c.job_script_name == "target-js")
job_script_row = await database.fetch_one(query)
assert job_script_row is not None
assert job_script_row["job_script_name"] == "target-js"
async def test_get_job_script_by_id_bad_permission(
client,
fill_application_data,
fill_job_script_data,
inject_security_header,
):
"""
Test the correct response code is returned when the user don't have the proper permission.
This test proves that GET /job-script/<id> returns the correct response code when the
user don't have the proper permission. We show this by asserting that the status code
returned is what we would expect (403).
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
inserted_job_script_id = await database.execute(
query=job_scripts_table.insert(),
values=fill_job_script_data(application_id=inserted_application_id),
)
inject_security_header("*****@*****.**", "INVALID_PERMISSION")
response = await client.get(
f"/jobbergate/job-scripts/{inserted_job_script_id}")
assert response.status_code == status.HTTP_403_FORBIDDEN
async def test_get_job_scripts__with_all_param(
client,
fill_application_data,
fill_all_job_script_data,
inject_security_header,
):
"""
Test that listing job_scripts, when all=True, contains job_scripts owned by other users.
This test proves that the user making the request can see job_scripts owned by other users.
We show this by creating three job_scripts, one that are owned by the user making the request, and two
owned by another user. Assert that the response to GET /job-scripts/?all=True includes all three
job_scripts.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
await database.execute_many(
query=job_scripts_table.insert(),
values=fill_all_job_script_data(
{
"job_script_name": "script1",
"job_script_owner_email": "*****@*****.**",
"application_id": inserted_application_id,
},
{
"job_script_name": "script2",
"job_script_owner_email": "*****@*****.**",
"application_id": inserted_application_id,
},
{
"job_script_name": "script3",
"job_script_owner_email": "*****@*****.**",
"application_id": inserted_application_id,
},
),
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 3
inject_security_header("*****@*****.**", Permissions.JOB_SCRIPTS_VIEW)
response = await client.get("/jobbergate/job-scripts/?all=True")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert results
assert [d["job_script_name"]
for d in results] == ["script1", "script2", "script3"]
pagination = data.get("pagination")
assert pagination == dict(
total=3,
start=None,
limit=None,
)
async def test_get_job_script__no_params(
client,
fill_application_data,
fill_all_job_script_data,
inject_security_header,
):
"""
Test GET /job-scripts/ returns only job_scripts owned by the user making the request.
This test proves that GET /job-scripts/ returns the correct job_scripts for the user making
the request. We show this by asserting that the job_scripts returned in the response are
only job_scripts owned by the user making the request.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
await database.execute_many(
query=job_scripts_table.insert(),
values=fill_all_job_script_data(
dict(
job_script_name="js1",
job_script_owner_email="*****@*****.**",
application_id=inserted_application_id,
),
dict(
job_script_name="js2",
job_script_owner_email="*****@*****.**",
application_id=inserted_application_id,
),
dict(
job_script_name="js3",
job_script_owner_email="*****@*****.**",
application_id=inserted_application_id,
),
),
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 3
inject_security_header("*****@*****.**", Permissions.JOB_SCRIPTS_VIEW)
response = await client.get("/jobbergate/job-scripts/")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert results
assert [d["job_script_name"] for d in results] == ["js1", "js3"]
pagination = data.get("pagination")
assert pagination == dict(
total=2,
start=None,
limit=None,
)
async def test_update_job_script(
client,
fill_application_data,
fill_job_script_data,
inject_security_header,
time_frame,
):
"""
Test update job_script via PUT.
This test proves that the job_script values are correctly updated following a PUT request to the
/job-scripts/<id> endpoint. We show this by assert the response status code to 201, the response data
corresponds to the updated data, and the data in the database is also updated.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
inserted_job_script_id = await database.execute(
query=job_scripts_table.insert(),
values=fill_job_script_data(application_id=inserted_application_id),
)
inject_security_header("*****@*****.**", Permissions.JOB_SCRIPTS_EDIT)
with time_frame() as window:
response = await client.put(
f"/jobbergate/job-scripts/{inserted_job_script_id}",
json={
"job_script_name": "new name",
"job_script_description": "new description",
"job_script_data_as_string": "new value",
},
)
assert response.status_code == status.HTTP_200_OK
data = response.json()
assert data["job_script_name"] == "new name"
assert data["job_script_description"] == "new description"
assert data["job_script_data_as_string"] == "new value"
assert data["id"] == inserted_job_script_id
query = job_scripts_table.select(
job_scripts_table.c.id == inserted_job_script_id)
job_script = JobScriptResponse.parse_obj(await database.fetch_one(query))
assert job_script is not None
assert job_script.job_script_name == "new name"
assert job_script.job_script_description == "new description"
assert job_script.job_script_data_as_string == "new value"
assert job_script.updated_at in window
async def test_get_job_script_by_id(
client,
fill_application_data,
job_script_data,
fill_job_script_data,
inject_security_header,
):
"""
Test GET /job-scripts/<id>.
This test proves that GET /job-scripts/<id> returns the correct job-script, owned by
the user making the request. We show this by asserting that the job_script data
returned in the response is equal to the job_script data that exists in the database
for the given job_script id.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
inserted_job_script_id = await database.execute(
query=job_scripts_table.insert(),
values=fill_job_script_data(application_id=inserted_application_id),
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 1
inject_security_header("*****@*****.**", Permissions.JOB_SCRIPTS_VIEW)
response = await client.get(
f"/jobbergate/job-scripts/{inserted_job_script_id}")
assert response.status_code == status.HTTP_200_OK
data = response.json()
assert data["id"] == inserted_job_script_id
assert data["job_script_name"] == job_script_data["job_script_name"]
assert data["job_script_data_as_string"] == job_script_data[
"job_script_data_as_string"]
assert data["job_script_owner_email"] == "*****@*****.**"
assert data["application_id"] == inserted_application_id
async def test_delete_job_script__fk_error(
client,
fill_application_data,
fill_job_script_data,
inject_security_header,
):
"""
Test DELETE /job_script/<id> correctly returns a 409 on a foreign-key constraint error.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(),
)
inserted_job_script_id = await database.execute(
query=job_scripts_table.insert(),
values=fill_job_script_data(application_id=inserted_application_id),
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 1
inject_security_header("*****@*****.**", Permissions.JOB_SCRIPTS_EDIT)
with mock.patch(
"jobbergate_api.storage.database.execute",
side_effect=asyncpg.exceptions.ForeignKeyViolationError(f"""
update or delete on table "job_scripts" violates foreign key constraint
"job_submissions_job_script_id_fkey" on table "job_submissions"
DETAIL: Key (id)=({inserted_job_script_id}) is still referenced from table "job_submissions".
"""),
):
response = await client.delete(
f"/jobbergate/job-scripts/{inserted_job_script_id}")
assert response.status_code == status.HTTP_409_CONFLICT
error_data = json.loads(response.text)["detail"]
assert error_data[
"message"] == "Delete failed due to foreign-key constraint"
assert error_data["table"] == "job_submissions"
assert error_data["pk_id"] == f"{inserted_job_script_id}"
async def test_get_job_scripts__with_pagination(
client,
fill_application_data,
fill_job_script_data,
inject_security_header,
):
"""
Test that listing job_scripts works with pagination.
This test proves that the user making the request can see job_scripts paginated.
We show this by creating three job_scripts and assert that the response is correctly paginated.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
await database.execute_many(
query=job_scripts_table.insert(),
values=[
fill_job_script_data(
job_script_name=f"script{i}",
job_script_owner_email="*****@*****.**",
application_id=inserted_application_id,
) for i in range(1, 6)
],
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 5
inject_security_header("*****@*****.**", Permissions.JOB_SCRIPTS_VIEW)
response = await client.get("/jobbergate/job-scripts?start=0&limit=1")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert results
assert [d["job_script_name"] for d in results] == ["script1"]
pagination = data.get("pagination")
assert pagination == dict(total=5, start=0, limit=1)
response = await client.get("/jobbergate/job-scripts?start=1&limit=2")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert results
assert [d["job_script_name"] for d in results] == ["script3", "script4"]
pagination = data.get("pagination")
assert pagination == dict(total=5, start=1, limit=2)
response = await client.get("/jobbergate/job-scripts?start=2&limit=2")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert results
assert [d["job_script_name"] for d in results] == ["script5"]
pagination = data.get("pagination")
assert pagination == dict(total=5, start=2, limit=2)
async def test_get_job_scripts__with_sort_params(
client,
fill_application_data,
inject_security_header,
):
"""
Test that listing job_scripts with sort params returns correctly ordered matches.
This test proves that the user making the request will be shown job_scripts sorted in the correct order
according to the ``sort_field`` and ``sort_ascending`` parameters.
We show this by creating job_scripts and using various sort parameters to order them.
Assert that the response to GET /job_scripts?sort_field=<field>&sort_ascending=<bool> includes correctly
sorted job_script.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
common = dict(
job_script_owner_email="*****@*****.**",
job_script_data_as_string="whatever",
application_id=inserted_application_id,
)
await database.execute_many(
query=job_scripts_table.insert(),
values=[
dict(
job_script_name="Z",
**common,
),
dict(
job_script_name="Y",
**common,
),
dict(
job_script_name="X",
**common,
),
],
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 3
inject_security_header("*****@*****.**", Permissions.JOB_SCRIPTS_VIEW)
response = await client.get("/jobbergate/job-scripts?sort_field=id")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert [d["job_script_name"] for d in results] == ["Z", "Y", "X"]
response = await client.get(
"/jobbergate/job-scripts?sort_field=id&sort_ascending=false")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert [d["job_script_name"] for d in results] == ["X", "Y", "Z"]
response = await client.get(
"/jobbergate/job-scripts?sort_field=job_script_name")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert [d["job_script_name"] for d in results] == ["X", "Y", "Z"]
response = await client.get(
"/jobbergate/job-scripts?all=true&sort_field=job_script_data_as_string"
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "Invalid sorting column requested" in response.text
async def test_get_job_scripts__with_search_param(client,
inject_security_header,
fill_application_data):
"""
Test that listing job scripts, when search=<search terms>, returns matches.
This test proves that the user making the request will be shown job scripts that match the search string.
We show this by creating job scripts and using various search queries to match against them.
Assert that the response to GET /job_scripts?search=<search temrms> includes correct matches.
"""
inserted_application_id = await database.execute(
query=applications_table.insert(),
values=fill_application_data(application_owner_email="*****@*****.**"),
)
common = dict(job_script_data_as_string="whatever",
application_id=inserted_application_id)
await database.execute_many(
query=job_scripts_table.insert(),
values=[
dict(
id=1,
job_script_name="test name one",
job_script_owner_email="*****@*****.**",
**common,
),
dict(
id=2,
job_script_name="test name two",
job_script_owner_email="*****@*****.**",
**common,
),
dict(
id=22,
job_script_name="test name twenty-two",
job_script_owner_email="*****@*****.**",
job_script_description="a long description of this job_script",
**common,
),
],
)
count = await database.fetch_all("SELECT COUNT(*) FROM job_scripts")
assert count[0][0] == 3
inject_security_header("*****@*****.**", Permissions.JOB_SCRIPTS_VIEW)
response = await client.get("/jobbergate/job-scripts?all=true&search=one")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert [d["id"] for d in results] == [1]
response = await client.get("/jobbergate/job-scripts?all=true&search=two")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert [d["id"] for d in results] == [2, 22]
response = await client.get("/jobbergate/job-scripts?all=true&search=long")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert [d["id"] for d in results] == [22]
response = await client.get(
"/jobbergate/job-scripts?all=true&search=name+test")
assert response.status_code == status.HTTP_200_OK
data = response.json()
results = data.get("results")
assert [d["id"] for d in results] == [1, 2, 22]
async def job_script_create(
job_script: JobScriptCreateRequest,
token_payload: TokenPayload = Depends(
guard.lockdown(Permissions.JOB_SCRIPTS_EDIT)),
):
"""
Create a new job script.
Make a post request to this endpoint with the required values to create a new job script.
"""
logger.debug(f"Creating job_script with: {job_script}")
select_query = applications_table.select().where(
applications_table.c.id == job_script.application_id)
raw_application = await database.fetch_one(select_query)
if not raw_application:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=
f"Application with id={job_script.application_id} not found.",
)
application = ApplicationResponse.parse_obj(raw_application)
logger.debug("Fetching application tarfile")
s3_application_tar = get_s3_object_as_tarfile(application.id)
identity_claims = IdentityClaims.from_token_payload(token_payload)
create_dict = dict(
**{
k: v
for (k, v) in job_script.dict(exclude_unset=True).items()
if k != "param_dict"
},
job_script_owner_email=identity_claims.user_email,
)
# Use application_config from the application as a baseline of defaults
print("APP CONFIG: ", application.application_config)
param_dict = safe_load(application.application_config)
# User supplied param dict is optional and may override defaults
param_dict.update(**job_script.param_dict)
logger.debug("Rendering job_script data as string")
job_script_data_as_string = build_job_script_data_as_string(
s3_application_tar, param_dict)
sbatch_params = create_dict.pop("sbatch_params", [])
create_dict["job_script_data_as_string"] = inject_sbatch_params(
job_script_data_as_string, sbatch_params)
logger.debug("Inserting job_script")
try:
insert_query = job_scripts_table.insert().returning(job_scripts_table)
job_script_data = await database.fetch_one(query=insert_query,
values=create_dict)
except INTEGRITY_CHECK_EXCEPTIONS as e:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=str(e))
logger.debug(f"Created job_script={job_script_data}")
return job_script_data