def get_beam_job_runs(
refresh: bool = True
) -> List[beam_job_domain.BeamJobRun]:
"""Returns all of the Apache Beam job runs recorded in the datastore.
Args:
refresh: bool. Whether to refresh the jobs' state before returning them.
Returns:
list(BeamJobRun). A list of every job run recorded in the datastore.
"""
beam_job_run_models = list(beam_job_models.BeamJobRunModel.query())
beam_job_runs = [
get_beam_job_run_from_model(m) for m in beam_job_run_models
]
if refresh:
updated_beam_job_run_models = []
for i, beam_job_run_model in enumerate(beam_job_run_models):
if beam_job_runs[i].in_terminal_state:
continue
jobs_manager.refresh_state_of_beam_job_run_model(beam_job_run_model)
beam_job_run_model.update_timestamps(update_last_updated_time=False)
updated_beam_job_run_models.append(beam_job_run_model)
beam_job_runs[i] = get_beam_job_run_from_model(beam_job_run_model)
if updated_beam_job_run_models:
datastore_services.put_multi(updated_beam_job_run_models)
return beam_job_runs
def test_job_with_cancelling_status_but_job_is_running(self) -> None:
self.run_model.latest_job_state = 'CANCELLING'
self.dataflow_job.current_state = dataflow.JobState.JOB_STATE_RUNNING
jobs_manager.refresh_state_of_beam_job_run_model(self.run_model)
self.assertEqual(self.run_model.latest_job_state, 'CANCELLING')
def test_job_with_outdated_status(self) -> None:
self.run_model.latest_job_state = 'PENDING'
self.dataflow_job.current_state = dataflow.JobState.JOB_STATE_RUNNING
jobs_manager.refresh_state_of_beam_job_run_model(self.run_model)
self.assertEqual(self.run_model.latest_job_state, 'RUNNING')
def test_job_with_failed_status(self) -> None:
self.run_model.latest_job_state = 'RUNNING'
self.dataflow_job.current_state = dataflow.JobState.JOB_STATE_FAILED
jobs_manager.refresh_state_of_beam_job_run_model(self.run_model)
self.assertEqual(self.run_model.latest_job_state, 'FAILED')
result = beam_job_services.get_beam_job_run_result(self.run_model.id)
self.assertIn(self.dataflow_job.id, result.stderr)
def test_failed_api_call_logs_the_exception(self) -> None:
self.run_model.latest_job_state = 'PENDING'
self.dataflow_client_mock.get_job.side_effect = Exception('uh-oh')
with self.capture_logging() as logs:
jobs_manager.refresh_state_of_beam_job_run_model(self.run_model)
self.assertGreater(len(logs), 0)
self.assertIn('uh-oh', logs[0])
self.assertEqual(self.run_model.latest_job_state, 'UNKNOWN')
def test_sync_job(self) -> None:
self.run_model.dataflow_job_id = None
jobs_manager.refresh_state_of_beam_job_run_model(self.run_model)
self.assertEqual(self.run_model.latest_job_state, 'UNKNOWN')
