def test_update_excluding_a_field(tmp_work_dir):
job = Job(id="foo123", action="foo", commit="commit-of-glory")
insert(job)
job.action = "bar"
job.commit = "commit-of-doom"
update(job, exclude_fields=["commit"])
j = find_one(Job, id="foo123")
assert j.action == "bar"
assert j.commit == "commit-of-glory"
def job_request_factory(**kwargs):
if "id" not in kwargs:
kwargs["id"] = base64.b32encode(secrets.token_bytes(10)).decode("ascii").lower()
values = deepcopy(JOB_REQUEST_DEFAULTS)
values.update(kwargs)
job_request = JobRequest(**values)
insert(SavedJobRequest(id=job_request.id, original=job_request.original))
return job_request
def job_factory(job_request=None, **kwargs):
if job_request is None:
job_request = job_request_factory()
values = deepcopy(JOB_DEFAULTS)
values.update(kwargs)
values["job_request_id"] = job_request.id
job = Job(**values)
insert(job)
return job
def test_basic_roundtrip(tmp_work_dir):
job = Job(
id="foo123",
job_request_id="bar123",
state=State.RUNNING,
output_spec={"hello": [1, 2, 3]},
)
insert(job)
j = find_one(Job, job_request_id__in=["bar123", "baz123"])
assert job.id == j.id
assert job.output_spec == j.output_spec
def test_select_values(tmp_work_dir):
insert(Job(id="foo123", state=State.PENDING))
insert(Job(id="foo124", state=State.RUNNING))
insert(Job(id="foo125", state=State.FAILED))
values = select_values(Job, "id", state__in=[State.PENDING, State.FAILED])
assert sorted(values) == ["foo123", "foo125"]
values = select_values(Job, "state", id="foo124")
assert values == [State.RUNNING]
def test_local_run_copes_with_detritus_of_earlier_interrupted_run(
extraction_tool, tmp_path
):
# This test simulates the case where an earlier run has been interrupted (for example by the user pressing ctrl-c).
# In particular we put a couple of jobs in unfinished states, which they could never be left in under normal
# operation. The correct behaviour of the local run, which this tests for, is for such unfinished jobs to be marked
# as cancelled on the next run.
project_dir = tmp_path / "project"
shutil.copytree(str(FIXTURE_DIR / "full_project"), project_dir)
config.DATABASE_FILE = project_dir / "metadata" / "db.sqlite"
project = load_pipeline(project_dir / "project.yaml")
database.insert(SavedJobRequest(id="previous-request", original={}))
def job(job_id, action, state):
spec = get_action_specification(
project,
action,
using_dummy_data_backend=config.USING_DUMMY_DATA_BACKEND,
)
return Job(
id=job_id,
job_request_id="previous-request",
state=state,
status_message="",
repo_url=str(project_dir),
workspace=project_dir.name,
database_name="a-database",
action=action,
wait_for_job_ids=[],
requires_outputs_from=spec.needs,
run_command=spec.run,
output_spec=spec.outputs,
created_at=int(time.time()),
updated_at=int(time.time()),
outputs={},
)
# FIXME: consolidate these when databuilder supports more columns in dummy data
if extraction_tool == "cohortextractor":
actions = ["generate_cohort", "prepare_data_m_cohortextractor"]
else:
actions = ["generate_dataset", "analyse_data_databuilder"]
database.insert(job(job_id="123", action=actions[0], state=State.RUNNING))
database.insert(job(job_id="456", action=actions[1], state=State.PENDING))
assert local_run.main(project_dir=project_dir, actions=[actions[1]])
assert database.find_one(Job, id="123").cancelled
assert database.find_one(Job, id="123").state == State.FAILED
assert database.find_one(Job, id="456").cancelled
assert database.find_one(Job, id="456").state == State.FAILED
def insert_into_database(job_request, jobs):
with transaction():
insert(SavedJobRequest(id=job_request.id, original=job_request.original))
for job in jobs:
insert(job)
def test_find_one_fails_if_there_is_more_than_one_result(tmp_work_dir):
insert(Job(id="foo123", workspace="the-workspace"))
insert(Job(id="foo456", workspace="the-workspace"))
with pytest.raises(ValueError):
find_one(Job, workspace="the-workspace")
def test_find_one_returns_a_single_value(tmp_work_dir):
insert(Job(id="foo123", workspace="the-workspace"))
job = find_one(Job, id="foo123")
assert job.workspace == "the-workspace"
def test_update(tmp_work_dir):
job = Job(id="foo123", action="foo")
insert(job)
job.action = "bar"
update(job)
assert find_one(Job, id="foo123").action == "bar"