def main(partial_job_ids, cleanup=False):
jobs = get_jobs(partial_job_ids)
for job in jobs:
# If the job has been previously killed we don't want to overwrite the
# timestamps here
if job.state in (State.PENDING, State.RUNNING):
mark_job_as_failed(job, "Killed by admin")
# All these docker commands are idempotent
docker.kill(container_name(job))
if cleanup:
docker.delete_container(container_name(job))
docker.delete_volume(volume_name(job))
def get_log(job):
result = docker.docker(
["container", "logs", local.container_name(job)],
check=True,
text=True,
capture_output=True,
)
return result.stdout + result.stderr
def main(partial_job_id):
job = get_job(partial_job_id)
if not docker.container_exists(container_name(job)):
raise RuntimeError(
"Cannot reset job, associated container does not exist")
job.state = State.RUNNING
job.status_message = "Re-attempting to extract outputs"
job.status_code = None
job.completed_at = None
job.updated_at = int(time.time())
print("\nUpdating job in database:")
print(job)
update(job)
print("\nPOSTing update to job-server")
api_post("jobs", json=[job_to_remote_format(job)])
print("\nDone")
def test_finalize_failed_137(docker_cleanup, test_repo, tmp_work_dir,
volume_api):
ensure_docker_images_present("busybox")
job = JobDefinition(
id="test_finalize_failed",
job_request_id="test_request_id",
study=test_repo.study,
workspace="test",
action="action",
created_at=int(time.time()),
image="ghcr.io/opensafely-core/busybox",
args=["sleep", "101"],
env={},
inputs=["output/input.csv"],
output_spec={
"output/output.*": "high_privacy",
"output/summary.*": "medium_privacy",
},
allow_database_access=False,
)
populate_workspace(job.workspace, "output/input.csv")
api = local.LocalDockerAPI()
status = api.prepare(job)
assert status.state == ExecutorState.PREPARING
status = api.execute(job)
assert status.state == ExecutorState.EXECUTING
# impersonate an admin
docker.kill(local.container_name(job))
wait_for_state(api, job, ExecutorState.EXECUTED)
status = api.finalize(job)
assert status.state == ExecutorState.FINALIZING
# we don't need to wait
assert api.get_status(job).state == ExecutorState.FINALIZED
assert job.id in local.RESULTS
assert local.RESULTS[job.id].exit_code == 137
assert local.RESULTS[job.id].message == "Killed by an OpenSAFELY admin"
def test_execute_success(docker_cleanup, test_repo, tmp_work_dir, volume_api):
ensure_docker_images_present("busybox")
job = JobDefinition(
id="test_execute_success",
job_request_id="test_request_id",
study=test_repo.study,
workspace="test",
action="action",
created_at=int(time.time()),
image="ghcr.io/opensafely-core/busybox",
args=["/usr/bin/true"],
env={},
inputs=["output/input.csv"],
output_spec={},
allow_database_access=False,
cpu_count=1.5,
memory_limit="1G",
)
populate_workspace(job.workspace, "output/input.csv")
api = local.LocalDockerAPI()
# use prepare step as test set up
status = api.prepare(job)
assert status.state == ExecutorState.PREPARING
status = api.execute(job)
assert status.state == ExecutorState.EXECUTING
# could be in either state
assert api.get_status(job).state in (
ExecutorState.EXECUTING,
ExecutorState.EXECUTED,
)
container_data = docker.container_inspect(local.container_name(job),
"HostConfig")
assert container_data["NanoCpus"] == int(1.5 * 1e9)
assert container_data["Memory"] == 2**30 # 1G
def test_cleanup_success(docker_cleanup, test_repo, tmp_work_dir, volume_api):
ensure_docker_images_present("busybox")
job = JobDefinition(
id="test_cleanup_success",
job_request_id="test_request_id",
study=test_repo.study,
workspace="test",
action="action",
created_at=int(time.time()),
image="ghcr.io/opensafely-core/busybox",
args=["/usr/bin/true"],
env={},
inputs=["output/input.csv"],
output_spec={},
allow_database_access=False,
)
populate_workspace(job.workspace, "output/input.csv")
api = local.LocalDockerAPI()
api.prepare(job)
api.execute(job)
container = local.container_name(job)
assert volume_api.volume_exists(job)
assert docker.container_exists(container)
status = api.cleanup(job)
assert status.state == ExecutorState.UNKNOWN
status = api.get_status(job)
assert status.state == ExecutorState.UNKNOWN
assert not volume_api.volume_exists(job)
assert not docker.container_exists(container)