def test_prepare_already_prepared(docker_cleanup, test_repo, volume_api): ensure_docker_images_present("busybox") job = JobDefinition( id="test_prepare_already_prepared", job_request_id="test_request_id", study=test_repo.study, workspace="test", action="action", created_at=int(time.time()), image="ghcr.io/opensafely-core/busybox", args=["/usr/bin/true"], env={}, inputs=["output/input.csv"], output_spec={}, allow_database_access=False, ) # create the volume already volume_api.create_volume(job) api = local.LocalDockerAPI() status = api.prepare(job) assert status.state == ExecutorState.PREPARED
def test_finalize_success(docker_cleanup, test_repo, tmp_work_dir, volume_api): ensure_docker_images_present("busybox") job = JobDefinition( id="test_finalize_success", job_request_id="test_request_id", study=test_repo.study, workspace="test", action="action", created_at=int(time.time()), image="ghcr.io/opensafely-core/busybox", args=[ "touch", "/workspace/output/output.csv", "/workspace/output/summary.csv" ], env={}, inputs=["output/input.csv"], output_spec={ "output/output.*": "high_privacy", "output/summary.*": "medium_privacy", }, allow_database_access=False, ) populate_workspace(job.workspace, "output/input.csv") api = local.LocalDockerAPI() status = api.prepare(job) assert status.state == ExecutorState.PREPARING status = api.execute(job) assert status.state == ExecutorState.EXECUTING wait_for_state(api, job, ExecutorState.EXECUTED) status = api.finalize(job) assert status.state == ExecutorState.FINALIZING # we don't need to wait assert api.get_status(job).state == ExecutorState.FINALIZED assert job.id in local.RESULTS # for test debugging if any asserts fail print(get_log(job)) results = api.get_results(job) assert results.exit_code == 0 assert results.outputs == { "output/output.csv": "high_privacy", "output/summary.csv": "medium_privacy", } assert results.unmatched_patterns == []
def test_finalize_failed_oomkilled(docker_cleanup, test_repo, tmp_work_dir, volume_api): ensure_docker_images_present("busybox") job = JobDefinition( id="test_finalize_failed", job_request_id="test_request_id", study=test_repo.study, workspace="test", action="action", created_at=int(time.time()), image="ghcr.io/opensafely-core/busybox", # Consume memory by writing to the tmpfs at /dev/shm # We write a lot more that our limit, to ensure the OOM killer kicks in # regardless of our tests host's vm.overcommit_memory settings. args=["sh", "-c", "head -c 100m /dev/urandom >/dev/shm/foo"], env={}, inputs=["output/input.csv"], output_spec={ "output/output.*": "high_privacy", "output/summary.*": "medium_privacy", }, allow_database_access=False, memory_limit="6M", # lowest allowable limit ) populate_workspace(job.workspace, "output/input.csv") api = local.LocalDockerAPI() status = api.prepare(job) assert status.state == ExecutorState.PREPARING status = api.execute(job) assert status.state == ExecutorState.EXECUTING wait_for_state(api, job, ExecutorState.EXECUTED) status = api.finalize(job) assert status.state == ExecutorState.FINALIZING # we don't need to wait assert api.get_status(job).state == ExecutorState.FINALIZED assert job.id in local.RESULTS assert local.RESULTS[job.id].exit_code == 137 # Note, 6MB is rounded to 0.01GBM by the formatter assert (local.RESULTS[job.id].message == "Ran out of memory (limit for this job was 0.01GB)")
def test_finalize_failed_137(docker_cleanup, test_repo, tmp_work_dir, volume_api): ensure_docker_images_present("busybox") job = JobDefinition( id="test_finalize_failed", job_request_id="test_request_id", study=test_repo.study, workspace="test", action="action", created_at=int(time.time()), image="ghcr.io/opensafely-core/busybox", args=["sleep", "101"], env={}, inputs=["output/input.csv"], output_spec={ "output/output.*": "high_privacy", "output/summary.*": "medium_privacy", }, allow_database_access=False, ) populate_workspace(job.workspace, "output/input.csv") api = local.LocalDockerAPI() status = api.prepare(job) assert status.state == ExecutorState.PREPARING status = api.execute(job) assert status.state == ExecutorState.EXECUTING # impersonate an admin docker.kill(local.container_name(job)) wait_for_state(api, job, ExecutorState.EXECUTED) status = api.finalize(job) assert status.state == ExecutorState.FINALIZING # we don't need to wait assert api.get_status(job).state == ExecutorState.FINALIZED assert job.id in local.RESULTS assert local.RESULTS[job.id].exit_code == 137 assert local.RESULTS[job.id].message == "Killed by an OpenSAFELY admin"
def test_prepare_success(docker_cleanup, test_repo, tmp_work_dir, volume_api): ensure_docker_images_present("busybox") job = JobDefinition( id="test-id", job_request_id="test_request_id", study=test_repo.study, workspace="test", action="action", created_at=int(time.time()), image="ghcr.io/opensafely-core/busybox", args=["/usr/bin/true"], env={}, inputs=["output/input.csv"], output_spec={ "*": "medium", "**/*": "medium", }, allow_database_access=False, ) populate_workspace(job.workspace, "output/input.csv") api = local.LocalDockerAPI() status = api.prepare(job) assert status.state == ExecutorState.PREPARING # we don't need to wait for this is currently synchronous assert api.get_status(job).state == ExecutorState.PREPARED assert volume_api.volume_exists(job) # check files have been copied expected = set(list_repo_files(test_repo.source) + job.inputs) expected.add(local.TIMESTAMP_REFERENCE_FILE) # glob_volume_files uses find, and its '**/*' regex doesn't find files in # the root dir, which is arguably correct. files = volume_api.glob_volume_files(job) all_files = set(files["*"] + files["**/*"]) assert all_files == expected
def test_execute_success(docker_cleanup, test_repo, tmp_work_dir, volume_api): ensure_docker_images_present("busybox") job = JobDefinition( id="test_execute_success", job_request_id="test_request_id", study=test_repo.study, workspace="test", action="action", created_at=int(time.time()), image="ghcr.io/opensafely-core/busybox", args=["/usr/bin/true"], env={}, inputs=["output/input.csv"], output_spec={}, allow_database_access=False, cpu_count=1.5, memory_limit="1G", ) populate_workspace(job.workspace, "output/input.csv") api = local.LocalDockerAPI() # use prepare step as test set up status = api.prepare(job) assert status.state == ExecutorState.PREPARING status = api.execute(job) assert status.state == ExecutorState.EXECUTING # could be in either state assert api.get_status(job).state in ( ExecutorState.EXECUTING, ExecutorState.EXECUTED, ) container_data = docker.container_inspect(local.container_name(job), "HostConfig") assert container_data["NanoCpus"] == int(1.5 * 1e9) assert container_data["Memory"] == 2**30 # 1G
def test_cleanup_success(docker_cleanup, test_repo, tmp_work_dir, volume_api): ensure_docker_images_present("busybox") job = JobDefinition( id="test_cleanup_success", job_request_id="test_request_id", study=test_repo.study, workspace="test", action="action", created_at=int(time.time()), image="ghcr.io/opensafely-core/busybox", args=["/usr/bin/true"], env={}, inputs=["output/input.csv"], output_spec={}, allow_database_access=False, ) populate_workspace(job.workspace, "output/input.csv") api = local.LocalDockerAPI() api.prepare(job) api.execute(job) container = local.container_name(job) assert volume_api.volume_exists(job) assert docker.container_exists(container) status = api.cleanup(job) assert status.state == ExecutorState.UNKNOWN status = api.get_status(job) assert status.state == ExecutorState.UNKNOWN assert not volume_api.volume_exists(job) assert not docker.container_exists(container)
def test_execute_not_prepared(docker_cleanup, test_repo, tmp_work_dir, volume_api): ensure_docker_images_present("busybox") job = JobDefinition( id="test_execute_not_prepared", job_request_id="test_request_id", study=test_repo.study, workspace="test", action="action", created_at=int(time.time()), image="ghcr.io/opensafely-core/busybox", args=["/usr/bin/true"], env={}, inputs=["output/input.csv"], output_spec={}, allow_database_access=False, ) api = local.LocalDockerAPI() status = api.execute(job) # this will be turned into an error by the loop assert status.state == ExecutorState.UNKNOWN
def test_integration(tmp_work_dir, docker_cleanup, requests_mock, monkeypatch, test_repo): # TODO: add the following parametrize decorator back to this test: # # @pytest.mark.parametrize("extraction_tool", ["cohortextractor", "databuilder"]) # # Databuilder currently supports too few options in dummy data (at the time # of writing we are still building out the "walking skeleton") to be run # alongside cohortextractor in this test, however once it supports a close # enough set of dummy data we can merge them into a single test. extraction_tool = "cohortextractor" if extraction_tool == "cohortextractor": generate_action = "generate_cohort" else: generate_action = "generate_dataset" api = get_executor_api() monkeypatch.setattr("jobrunner.config.JOB_SERVER_ENDPOINT", "http://testserver/api/v2/") # Disable repo URL checking so we can run using a local test repo monkeypatch.setattr("jobrunner.config.ALLOWED_GITHUB_ORGS", None) # Make job execution order deterministic monkeypatch.setattr("jobrunner.config.RANDOMISE_JOB_ORDER", False) if extraction_tool == "cohortextractor": image = "cohortextractor" else: image = "databuilder:v0.36.0" ensure_docker_images_present(image, "python") # Set up a mock job-server with a single job request job_request_1 = { "identifier": 1, "requested_actions": [ f"analyse_data_{extraction_tool}", f"test_reusable_action_{extraction_tool}", f"test_cancellation_{extraction_tool}", ], "cancelled_actions": [], "force_run_dependencies": False, "workspace": { "name": "testing", "repo": str(test_repo.path), "branch": "HEAD", "db": "dummy", }, "sha": test_repo.commit, } requests_mock.get( "http://testserver/api/v2/job-requests/?backend=expectations", json={ "results": [job_request_1], }, ) requests_mock.post("http://testserver/api/v2/jobs/", json={}) # Run sync to grab the JobRequest from the mocked job-server jobrunner.sync.sync() # Check that expected number of pending jobs are created jobs = get_posted_jobs(requests_mock) assert [job["status"] for job in jobs.values()] == ["pending"] * 7 # Execute one tick of the run loop and then sync jobrunner.run.handle_jobs(api) jobrunner.sync.sync() # We should now have one running job and all others waiting on dependencies jobs = get_posted_jobs(requests_mock) assert jobs[generate_action]["status"] == "running" for action in [ f"prepare_data_m_{extraction_tool}", f"prepare_data_f_{extraction_tool}", f"prepare_data_with_quote_in_filename_{extraction_tool}", f"analyse_data_{extraction_tool}", f"test_reusable_action_{extraction_tool}", f"test_cancellation_{extraction_tool}", ]: assert jobs[action]["status_message"].startswith( "Waiting on dependencies") # Update the existing job request to mark a job as cancelled, add a new job # request to be run and then sync job_request_1["cancelled_actions"] = [ f"test_cancellation_{extraction_tool}" ] job_request_2 = { "identifier": 2, "requested_actions": [ "generate_cohort_with_dummy_data", ], "cancelled_actions": [], "force_run_dependencies": False, "workspace": { "name": "testing", "repo": str(test_repo.path), "branch": "HEAD", "db": "dummy", }, "sha": test_repo.commit, } requests_mock.get( "http://testserver/api/v2/job-requests/?backend=expectations", json={ "results": [job_request_1, job_request_2], }, ) jobrunner.sync.sync() # Run the main loop until there are no jobs left and then sync jobrunner.run.main(exit_callback=lambda active_jobs: len(active_jobs) == 0) jobrunner.sync.sync() # All jobs should now have succeeded apart from the cancelled one jobs = get_posted_jobs(requests_mock) assert jobs[generate_action]["status"] == "succeeded" assert jobs[f"prepare_data_m_{extraction_tool}"]["status"] == "succeeded" assert jobs[f"prepare_data_f_{extraction_tool}"]["status"] == "succeeded" assert (jobs[f"prepare_data_with_quote_in_filename_{extraction_tool}"] ["status"] == "succeeded") assert jobs[f"analyse_data_{extraction_tool}"]["status"] == "succeeded" assert jobs[f"test_reusable_action_{extraction_tool}"][ "status"] == "succeeded" assert jobs[f"test_cancellation_{extraction_tool}"]["status"] == "failed" high_privacy_workspace = tmp_work_dir / "high_privacy_workspaces_dir" / "testing" medium_privacy_workspace = (tmp_work_dir / "medium_privacy_workspaces_dir" / "testing") # Check that the manifest contains what we expect. This is a subset of what used to be in the manifest, to support # nicer UX for osrelease. manifest_file = medium_privacy_workspace / "metadata" / "manifest.json" manifest = json.load(manifest_file.open()) assert manifest["workspace"] == "testing" assert manifest["repo"] == str(test_repo.path) if extraction_tool == "cohortextractor": output_name = "input" else: output_name = "dataset" # Check that all the outputs have been produced for highly_sensitive_output in [ f"output/{output_name}.csv", # the cohort/dataset "output/extra/input.csv", # extracted from dummy data f"{extraction_tool}-male.csv", # intermediate analysis f"{extraction_tool}-female.csv", # intermediate analysis f"{extraction_tool}-qu'ote.csv", # checking handling of problematic characters in filenames f"output/{output_name}.backup.csv", # from the reusable action ]: path = high_privacy_workspace / highly_sensitive_output assert path.exists(), highly_sensitive_output for moderately_sensitive_output in [ f"{extraction_tool}-counts.txt", # the study's actual output ]: assert (medium_privacy_workspace / moderately_sensitive_output).exists() # Check that we don't produce outputs for cancelled jobs assert not (high_privacy_workspace / f"{extraction_tool}-somefile.csv").exists()
def test_integration_with_databuilder(tmp_work_dir, docker_cleanup, requests_mock, monkeypatch, test_repo): # TODO: merge this test into test_integration # # Databuilder currently supports too few options in dummy data (at the time # of writing we are still building out the "walking skeleton") to be run # alongside cohortextractor in this test, however once it supports a close # enough set of dummy data we can merge them into a single test. extraction_tool = "databuilder" api = get_executor_api() monkeypatch.setattr("jobrunner.config.JOB_SERVER_ENDPOINT", "http://testserver/api/v2/") # Disable repo URL checking so we can run using a local test repo monkeypatch.setattr("jobrunner.config.ALLOWED_GITHUB_ORGS", None) # Make job execution order deterministic monkeypatch.setattr("jobrunner.config.RANDOMISE_JOB_ORDER", False) ensure_docker_images_present("databuilder:v0.36.0", "python") # Set up a mock job-server with a single job request job_request_1 = { "identifier": 1, "requested_actions": [ f"analyse_data_{extraction_tool}", f"test_cancellation_{extraction_tool}", ], "cancelled_actions": [], "force_run_dependencies": False, "workspace": { "name": "testing", "repo": str(test_repo.path), "branch": "HEAD", "db": "dummy", }, "sha": test_repo.commit, } requests_mock.get( "http://testserver/api/v2/job-requests/?backend=expectations", json={ "results": [job_request_1], }, ) requests_mock.post("http://testserver/api/v2/jobs/", json={}) # Run sync to grab the JobRequest from the mocked job-server jobrunner.sync.sync() # Check that expected number of pending jobs are created jobs = get_posted_jobs(requests_mock) assert [job["status"] for job in jobs.values() ] == ["pending"] * 3, list(jobs.values())[0]["status_message"] # Execute one tick of the run loop and then sync jobrunner.run.handle_jobs(api) jobrunner.sync.sync() # We should now have one running job and all others waiting on dependencies jobs = get_posted_jobs(requests_mock) assert jobs["generate_dataset"]["status"] == "running" for action in [ f"analyse_data_{extraction_tool}", f"test_cancellation_{extraction_tool}", ]: assert jobs[action]["status_message"].startswith( "Waiting on dependencies") # Update the existing job request to mark a job as cancelled, add a new job # request to be run and then sync job_request_1["cancelled_actions"] = [ f"test_cancellation_{extraction_tool}" ] job_request_2 = { "identifier": 2, "requested_actions": [ "derp_action", ], "cancelled_actions": [], "force_run_dependencies": False, "workspace": { "name": "testing", "repo": str(test_repo.path), "branch": "HEAD", "db": "dummy", }, "sha": test_repo.commit, } requests_mock.get( "http://testserver/api/v2/job-requests/?backend=expectations", json={ "results": [job_request_1, job_request_2], }, ) jobrunner.sync.sync() # Run the main loop until there are no jobs left and then sync jobrunner.run.main(exit_callback=lambda active_jobs: len(active_jobs) == 0) jobrunner.sync.sync() # All jobs should now have succeeded apart from the cancelled one jobs = get_posted_jobs(requests_mock) test_cancellation_job = jobs.pop(f"test_cancellation_{extraction_tool}") for action, job in jobs.items(): assert (job["status"] == "succeeded" ), f"{action} failed with: {job['status_message']}" assert test_cancellation_job["status"] == "failed" high_privacy_workspace = tmp_work_dir / "high_privacy_workspaces_dir" / "testing" medium_privacy_workspace = (tmp_work_dir / "medium_privacy_workspaces_dir" / "testing") # Check that the manifest contains what we expect. This is a subset of what used to be in the manifest, to support # nicer UX for osrelease. manifest_file = medium_privacy_workspace / "metadata" / "manifest.json" manifest = json.load(manifest_file.open()) assert manifest["workspace"] == "testing" assert manifest["repo"] == str(test_repo.path) # Check that all the outputs have been produced assert (high_privacy_workspace / "output/dataset.csv").exists() assert (medium_privacy_workspace / "output/count_by_year.csv").exists() # Check that we don't produce outputs for cancelled jobs assert not (high_privacy_workspace / "output/count_by_year_cancelled.csv").exists()