def generate_job_logs(all_jobs):
"""
Expected output from a `job_logs` request. Note that only completed jobs have logs in this case.
"""
job_logs = generate_bad_jobs()
for job_id in all_jobs:
test_job = get_test_job(job_id)
batch_job = test_job.get("batch_job", JOB_ATTR_DEFAULTS["batch_job"])
batch_id = job_id if batch_job else get_test_job(job_id).get(
"batch_id", None)
if all_jobs[job_id]["status"] == COMPLETED_STATUS:
job_logs[job_id] = {
"job_id": job_id,
"batch_id": batch_id,
"first": 0,
"latest": False,
"max_lines": 50,
"lines": log_gen(5),
}
else:
job_logs[job_id] = {
"batch_id": batch_id,
"error": generate_error(job_id, "no_logs"),
"job_id": job_id,
}
return job_logs
def _generate_job_output(job_id):
state = get_test_job(job_id)
widget_info = state.get("widget_info", None)
state.update({
"batch_id":
state.get("batch_id",
job_id if state.get("batch_job", False) else None),
"job_output":
state.get("job_output", {}),
"child_jobs":
state.get("child_jobs", []),
})
for f in OUTPUT_STATE_EXCLUDED_JOB_STATE_FIELDS:
if f in state:
del state[f]
if state["status"] != COMPLETED_STATUS:
return {
"job_id": job_id,
"jobState": state,
"outputWidgetInfo": widget_info
}
if not widget_info:
widget_info = {}
return {
"job_id": job_id,
"jobState": state,
"outputWidgetInfo": widget_info
}
def generate_job_info(all_jobs):
"""
Expected output from a `job_info` request
"""
job_info = generate_bad_jobs()
for job_id in all_jobs:
test_job = get_test_job(job_id)
job_id = test_job.get("job_id")
app_id = test_job.get("job_input", {}).get("app_id", None)
tag = (test_job.get("job_input", {}).get("narrative_cell_info",
{}).get("tag", "release"))
params = test_job.get("job_input", {}).get("params",
JOB_ATTR_DEFAULTS["params"])
batch_job = test_job.get("batch_job", JOB_ATTR_DEFAULTS["batch_job"])
app_name = "batch" if batch_job else get_test_spec(
tag, app_id)["info"]["name"]
batch_id = (job_id if batch_job else test_job.get(
"batch_id", JOB_ATTR_DEFAULTS["batch_id"]))
job_info[job_id] = {
"app_id": app_id,
"app_name": app_name,
"job_id": job_id,
"job_params": params,
"batch_id": batch_id,
}
return job_info
def create_attrs_from_ee2(job_id):
state = get_test_job(job_id)
job_input = state.get("job_input", {})
narr_cell_info = job_input.get("narrative_cell_info", {})
attrs = dict(
app_id=job_input.get("app_id", JOB_ATTR_DEFAULTS["app_id"]),
app_version=job_input.get("service_ver", JOB_ATTR_DEFAULTS["app_version"]),
batch_id=(
state.get("job_id")
if state.get("batch_job", JOB_ATTR_DEFAULTS["batch_job"])
else state.get("batch_id", JOB_ATTR_DEFAULTS["batch_id"])
),
batch_job=state.get("batch_job", JOB_ATTR_DEFAULTS["batch_job"]),
cell_id=narr_cell_info.get("cell_id", JOB_ATTR_DEFAULTS["cell_id"]),
child_jobs=state.get("child_jobs", JOB_ATTR_DEFAULTS["child_jobs"]),
job_id=state.get("job_id"),
params=job_input.get("params", JOB_ATTR_DEFAULTS["params"]),
retry_ids=state.get("retry_ids", JOB_ATTR_DEFAULTS["retry_ids"]),
retry_parent=state.get("retry_parent", JOB_ATTR_DEFAULTS["retry_parent"]),
run_id=narr_cell_info.get("run_id", JOB_ATTR_DEFAULTS["run_id"]),
tag=narr_cell_info.get("tag", JOB_ATTR_DEFAULTS["tag"]),
user=state.get("user", JOB_ATTR_DEFAULTS["user"]),
)
return attrs
def test_cancel_jobs__job_already_finished(self):
self.assertEqual(get_test_job(JOB_COMPLETED)["status"], "completed")
self.assertEqual(get_test_job(JOB_TERMINATED)["status"], "terminated")
self.assertTrue(self.jm.get_job(JOB_COMPLETED).was_terminal())
self.assertTrue(self.jm.get_job(JOB_TERMINATED).was_terminal())
job_id_list = [JOB_COMPLETED, JOB_TERMINATED]
with mock.patch(
"biokbase.narrative.jobs.jobmanager.JobManager._cancel_job"
) as mock_cancel_job:
canceled_jobs = self.jm.cancel_jobs(job_id_list)
mock_cancel_job.assert_not_called()
self.assertEqual(
{
id: ALL_RESPONSE_DATA[MESSAGE_TYPE["STATUS"]][id]
for id in job_id_list
},
canceled_jobs,
)
def create_state_from_ee2(job_id, exclude_fields=JOB_INIT_EXCLUDED_JOB_STATE_FIELDS):
"""
create the output of job.state() from raw job data
"""
state = get_test_job(job_id)
for attr in exclude_fields:
if attr in state:
del state[attr]
return state
def test_parameters__param_fetch_fail(self):
"""
test failure to retrieve job params data
"""
job_state = get_test_job(JOB_TERMINATED)
del job_state["job_input"]["params"]
job = Job(job_state)
self.assertEqual(job.params, JOB_ATTR_DEFAULTS["params"])
with self.assertRaisesRegex(Exception, "Unable to fetch parameters for job"):
job.parameters()
def test_job_update__invalid_job_id(self):
"""
ensure that an ee2 state with a different job ID cannot be used to update a job
"""
job = create_job_from_ee2(JOB_RUNNING)
expected = create_state_from_ee2(JOB_RUNNING)
self.assertEqual(job.state(), expected)
# try to update it with the job state from a different job
with self.assertRaisesRegex(ValueError, "Job ID mismatch in _update_state"):
job._update_state(get_test_job(JOB_COMPLETED))
def mock_check_job(params):
"""Called from job.state()"""
job_id = params["job_id"]
if job_id == BATCH_PARENT:
return {"child_jobs": new_child_ids}
elif job_id in TEST_JOBS:
return get_test_job(job_id)
elif job_id == JOB_NOT_FOUND:
return {
"job_id": job_id,
"status": generate_error(job_id, "not_found")
}
else:
raise Exception()
def test_parameters(self):
"""
test that a job returns the correct parameters
"""
job_state = get_test_job(JOB_COMPLETED)
job_params = job_state.get("job_input", {}).get("params", None)
self.assertIsNotNone(job_params)
job = Job(job_state)
self.assertIsNotNone(job.params)
with assert_obj_method_called(MockClients, "get_job_params", call_status=False):
params = job.parameters()
self.assertIsNotNone(params)
self.assertEqual(params, job_params)
def test_parameters__param_fetch_ok(self):
"""
test that a job can successfully retrieve parameters from ee2
if they do not exist
"""
job_state = get_test_job(JOB_CREATED)
job_params = job_state.get("job_input", {}).get("params", None)
self.assertIsNotNone(job_params)
# delete the job params from the input
del job_state["job_input"]["params"]
job = Job(job_state)
self.assertEqual(job.params, JOB_ATTR_DEFAULTS["params"])
params = job.parameters()
self.assertEqual(params, job_params)
def test_job_info(self):
job = create_job_from_ee2(JOB_COMPLETED)
job_data = get_test_job(JOB_COMPLETED)
app_id = job_data.get("job_input", {}).get("app_id")
tag = job_data.get("job_input", {}).get("narrative_cell_info", {}).get("tag")
status = job_data["status"]
job_spec = get_test_spec(tag, app_id)
app_name = job_spec.get("info", {}).get("name")
version = job_spec.get("info", {}).get("ver")
info_str = (
f"App name (id): {app_name} ({app_id})\n"
+ f"Version: {version}\n"
+ f"Status: {status}\nInputs:\n------\n"
)
with capture_stdout() as (out, err):
job.info()
self.assertIn(info_str, out.getvalue().strip())
def get_widget_info(job_id):
state = get_test_job(job_id)
if state.get("status") != COMPLETED_STATUS:
return None
job_input = state.get("job_input", {})
app_id = job_input.get("app_id", JOB_ATTR_DEFAULTS["app_id"])
params = job_input.get("params", JOB_ATTR_DEFAULTS["params"])
tag = job_input.get("narrative_cell_info", {}).get("tag", JOB_ATTR_DEFAULTS["tag"])
spec = get_test_spec(tag, app_id)
with mock.patch("biokbase.narrative.app_util.clients.get", get_mock_client):
output_widget, widget_params = map_outputs_from_state(
state,
map_inputs_from_job(params, spec),
spec,
)
return {
"name": output_widget,
"tag": tag,
"params": widget_params,
}
def create_job_from_ee2(job_id, extra_data=None, children=None):
state = get_test_job(job_id)
job = Job(state, extra_data=extra_data, children=children)
return job
def test_show_output_widget__incomplete_state(self):
job = Job(get_test_job(JOB_CREATED))
self.assertRegex(
job.show_output_widget(), "Job is incomplete! It has status 'created'"
)
def test_show_output_widget(self, mock_method):
mock_method.return_value = True
job = Job(get_test_job(JOB_COMPLETED))
self.assertTrue(job.show_output_widget())
mock_method.assert_called_once()
