def generate_job_logs(all_jobs): """ Expected output from a `job_logs` request. Note that only completed jobs have logs in this case. """ job_logs = generate_bad_jobs() for job_id in all_jobs: test_job = get_test_job(job_id) batch_job = test_job.get("batch_job", JOB_ATTR_DEFAULTS["batch_job"]) batch_id = job_id if batch_job else get_test_job(job_id).get( "batch_id", None) if all_jobs[job_id]["status"] == COMPLETED_STATUS: job_logs[job_id] = { "job_id": job_id, "batch_id": batch_id, "first": 0, "latest": False, "max_lines": 50, "lines": log_gen(5), } else: job_logs[job_id] = { "batch_id": batch_id, "error": generate_error(job_id, "no_logs"), "job_id": job_id, } return job_logs
def _generate_job_output(job_id): state = get_test_job(job_id) widget_info = state.get("widget_info", None) state.update({ "batch_id": state.get("batch_id", job_id if state.get("batch_job", False) else None), "job_output": state.get("job_output", {}), "child_jobs": state.get("child_jobs", []), }) for f in OUTPUT_STATE_EXCLUDED_JOB_STATE_FIELDS: if f in state: del state[f] if state["status"] != COMPLETED_STATUS: return { "job_id": job_id, "jobState": state, "outputWidgetInfo": widget_info } if not widget_info: widget_info = {} return { "job_id": job_id, "jobState": state, "outputWidgetInfo": widget_info }
def generate_job_info(all_jobs): """ Expected output from a `job_info` request """ job_info = generate_bad_jobs() for job_id in all_jobs: test_job = get_test_job(job_id) job_id = test_job.get("job_id") app_id = test_job.get("job_input", {}).get("app_id", None) tag = (test_job.get("job_input", {}).get("narrative_cell_info", {}).get("tag", "release")) params = test_job.get("job_input", {}).get("params", JOB_ATTR_DEFAULTS["params"]) batch_job = test_job.get("batch_job", JOB_ATTR_DEFAULTS["batch_job"]) app_name = "batch" if batch_job else get_test_spec( tag, app_id)["info"]["name"] batch_id = (job_id if batch_job else test_job.get( "batch_id", JOB_ATTR_DEFAULTS["batch_id"])) job_info[job_id] = { "app_id": app_id, "app_name": app_name, "job_id": job_id, "job_params": params, "batch_id": batch_id, } return job_info
def create_attrs_from_ee2(job_id): state = get_test_job(job_id) job_input = state.get("job_input", {}) narr_cell_info = job_input.get("narrative_cell_info", {}) attrs = dict( app_id=job_input.get("app_id", JOB_ATTR_DEFAULTS["app_id"]), app_version=job_input.get("service_ver", JOB_ATTR_DEFAULTS["app_version"]), batch_id=( state.get("job_id") if state.get("batch_job", JOB_ATTR_DEFAULTS["batch_job"]) else state.get("batch_id", JOB_ATTR_DEFAULTS["batch_id"]) ), batch_job=state.get("batch_job", JOB_ATTR_DEFAULTS["batch_job"]), cell_id=narr_cell_info.get("cell_id", JOB_ATTR_DEFAULTS["cell_id"]), child_jobs=state.get("child_jobs", JOB_ATTR_DEFAULTS["child_jobs"]), job_id=state.get("job_id"), params=job_input.get("params", JOB_ATTR_DEFAULTS["params"]), retry_ids=state.get("retry_ids", JOB_ATTR_DEFAULTS["retry_ids"]), retry_parent=state.get("retry_parent", JOB_ATTR_DEFAULTS["retry_parent"]), run_id=narr_cell_info.get("run_id", JOB_ATTR_DEFAULTS["run_id"]), tag=narr_cell_info.get("tag", JOB_ATTR_DEFAULTS["tag"]), user=state.get("user", JOB_ATTR_DEFAULTS["user"]), ) return attrs
def test_cancel_jobs__job_already_finished(self): self.assertEqual(get_test_job(JOB_COMPLETED)["status"], "completed") self.assertEqual(get_test_job(JOB_TERMINATED)["status"], "terminated") self.assertTrue(self.jm.get_job(JOB_COMPLETED).was_terminal()) self.assertTrue(self.jm.get_job(JOB_TERMINATED).was_terminal()) job_id_list = [JOB_COMPLETED, JOB_TERMINATED] with mock.patch( "biokbase.narrative.jobs.jobmanager.JobManager._cancel_job" ) as mock_cancel_job: canceled_jobs = self.jm.cancel_jobs(job_id_list) mock_cancel_job.assert_not_called() self.assertEqual( { id: ALL_RESPONSE_DATA[MESSAGE_TYPE["STATUS"]][id] for id in job_id_list }, canceled_jobs, )
def create_state_from_ee2(job_id, exclude_fields=JOB_INIT_EXCLUDED_JOB_STATE_FIELDS): """ create the output of job.state() from raw job data """ state = get_test_job(job_id) for attr in exclude_fields: if attr in state: del state[attr] return state
def test_parameters__param_fetch_fail(self): """ test failure to retrieve job params data """ job_state = get_test_job(JOB_TERMINATED) del job_state["job_input"]["params"] job = Job(job_state) self.assertEqual(job.params, JOB_ATTR_DEFAULTS["params"]) with self.assertRaisesRegex(Exception, "Unable to fetch parameters for job"): job.parameters()
def test_job_update__invalid_job_id(self): """ ensure that an ee2 state with a different job ID cannot be used to update a job """ job = create_job_from_ee2(JOB_RUNNING) expected = create_state_from_ee2(JOB_RUNNING) self.assertEqual(job.state(), expected) # try to update it with the job state from a different job with self.assertRaisesRegex(ValueError, "Job ID mismatch in _update_state"): job._update_state(get_test_job(JOB_COMPLETED))
def mock_check_job(params): """Called from job.state()""" job_id = params["job_id"] if job_id == BATCH_PARENT: return {"child_jobs": new_child_ids} elif job_id in TEST_JOBS: return get_test_job(job_id) elif job_id == JOB_NOT_FOUND: return { "job_id": job_id, "status": generate_error(job_id, "not_found") } else: raise Exception()
def test_parameters(self): """ test that a job returns the correct parameters """ job_state = get_test_job(JOB_COMPLETED) job_params = job_state.get("job_input", {}).get("params", None) self.assertIsNotNone(job_params) job = Job(job_state) self.assertIsNotNone(job.params) with assert_obj_method_called(MockClients, "get_job_params", call_status=False): params = job.parameters() self.assertIsNotNone(params) self.assertEqual(params, job_params)
def test_parameters__param_fetch_ok(self): """ test that a job can successfully retrieve parameters from ee2 if they do not exist """ job_state = get_test_job(JOB_CREATED) job_params = job_state.get("job_input", {}).get("params", None) self.assertIsNotNone(job_params) # delete the job params from the input del job_state["job_input"]["params"] job = Job(job_state) self.assertEqual(job.params, JOB_ATTR_DEFAULTS["params"]) params = job.parameters() self.assertEqual(params, job_params)
def test_job_info(self): job = create_job_from_ee2(JOB_COMPLETED) job_data = get_test_job(JOB_COMPLETED) app_id = job_data.get("job_input", {}).get("app_id") tag = job_data.get("job_input", {}).get("narrative_cell_info", {}).get("tag") status = job_data["status"] job_spec = get_test_spec(tag, app_id) app_name = job_spec.get("info", {}).get("name") version = job_spec.get("info", {}).get("ver") info_str = ( f"App name (id): {app_name} ({app_id})\n" + f"Version: {version}\n" + f"Status: {status}\nInputs:\n------\n" ) with capture_stdout() as (out, err): job.info() self.assertIn(info_str, out.getvalue().strip())
def get_widget_info(job_id): state = get_test_job(job_id) if state.get("status") != COMPLETED_STATUS: return None job_input = state.get("job_input", {}) app_id = job_input.get("app_id", JOB_ATTR_DEFAULTS["app_id"]) params = job_input.get("params", JOB_ATTR_DEFAULTS["params"]) tag = job_input.get("narrative_cell_info", {}).get("tag", JOB_ATTR_DEFAULTS["tag"]) spec = get_test_spec(tag, app_id) with mock.patch("biokbase.narrative.app_util.clients.get", get_mock_client): output_widget, widget_params = map_outputs_from_state( state, map_inputs_from_job(params, spec), spec, ) return { "name": output_widget, "tag": tag, "params": widget_params, }
def create_job_from_ee2(job_id, extra_data=None, children=None): state = get_test_job(job_id) job = Job(state, extra_data=extra_data, children=children) return job
def test_show_output_widget__incomplete_state(self): job = Job(get_test_job(JOB_CREATED)) self.assertRegex( job.show_output_widget(), "Job is incomplete! It has status 'created'" )
def test_show_output_widget(self, mock_method): mock_method.return_value = True job = Job(get_test_job(JOB_COMPLETED)) self.assertTrue(job.show_output_widget()) mock_method.assert_called_once()