def get_beam_job_runs( refresh: bool = True ) -> List[beam_job_domain.BeamJobRun]: """Returns all of the Apache Beam job runs recorded in the datastore. Args: refresh: bool. Whether to refresh the jobs' state before returning them. Returns: list(BeamJobRun). A list of every job run recorded in the datastore. """ beam_job_run_models = list(beam_job_models.BeamJobRunModel.query()) beam_job_runs = [ get_beam_job_run_from_model(m) for m in beam_job_run_models ] if refresh: updated_beam_job_run_models = [] for i, beam_job_run_model in enumerate(beam_job_run_models): if beam_job_runs[i].in_terminal_state: continue jobs_manager.refresh_state_of_beam_job_run_model(beam_job_run_model) beam_job_run_model.update_timestamps(update_last_updated_time=False) updated_beam_job_run_models.append(beam_job_run_model) beam_job_runs[i] = get_beam_job_run_from_model(beam_job_run_model) if updated_beam_job_run_models: datastore_services.put_multi(updated_beam_job_run_models) return beam_job_runs
def test_job_with_cancelling_status_but_job_is_running(self) -> None: self.run_model.latest_job_state = 'CANCELLING' self.dataflow_job.current_state = dataflow.JobState.JOB_STATE_RUNNING jobs_manager.refresh_state_of_beam_job_run_model(self.run_model) self.assertEqual(self.run_model.latest_job_state, 'CANCELLING')
def test_job_with_outdated_status(self) -> None: self.run_model.latest_job_state = 'PENDING' self.dataflow_job.current_state = dataflow.JobState.JOB_STATE_RUNNING jobs_manager.refresh_state_of_beam_job_run_model(self.run_model) self.assertEqual(self.run_model.latest_job_state, 'RUNNING')
def test_job_with_failed_status(self) -> None: self.run_model.latest_job_state = 'RUNNING' self.dataflow_job.current_state = dataflow.JobState.JOB_STATE_FAILED jobs_manager.refresh_state_of_beam_job_run_model(self.run_model) self.assertEqual(self.run_model.latest_job_state, 'FAILED') result = beam_job_services.get_beam_job_run_result(self.run_model.id) self.assertIn(self.dataflow_job.id, result.stderr)
def test_failed_api_call_logs_the_exception(self) -> None: self.run_model.latest_job_state = 'PENDING' self.dataflow_client_mock.get_job.side_effect = Exception('uh-oh') with self.capture_logging() as logs: jobs_manager.refresh_state_of_beam_job_run_model(self.run_model) self.assertGreater(len(logs), 0) self.assertIn('uh-oh', logs[0]) self.assertEqual(self.run_model.latest_job_state, 'UNKNOWN')
def test_sync_job(self) -> None: self.run_model.dataflow_job_id = None jobs_manager.refresh_state_of_beam_job_run_model(self.run_model) self.assertEqual(self.run_model.latest_job_state, 'UNKNOWN')