def test_launch_recovery_actionruns_for_job_runs(self, mock_filter):
mock_actions = (
[
mock.Mock(action_runner=NoActionRunnerFactory(),
spec=SSHActionRun),
mock.Mock(
action_runner=SubprocessActionRunnerFactory(
status_path='/tmp/foo', exec_path=('/tmp/foo')),
spec=SSHActionRun,
),
],
[
mock.Mock(action_runner=NoActionRunnerFactory(),
spec=MesosActionRun),
],
)
mock_filter.return_value = mock_actions
mock_action_runner = mock.Mock(autospec=True)
mock_job_run = mock.Mock()
launch_recovery_actionruns_for_job_runs([mock_job_run],
mock_action_runner)
ssh_runs = mock_actions[0]
for run in ssh_runs:
assert run.recover.call_count == 1
mesos_run = mock_actions[1][0]
assert mesos_run.recover.call_count == 1
def test_launch_recovery_actionruns_for_job_runs(self, mock_filter,
mock_recover_action_run):
mock_actions = (
[
mock.Mock(action_runner=NoActionRunnerFactory(),
spec=SSHActionRun),
mock.Mock(
action_runner=SubprocessActionRunnerFactory(
status_path='/tmp/foo', exec_path=('/tmp/foo')),
spec=SSHActionRun,
),
],
[
mock.Mock(action_runner=NoActionRunnerFactory(),
spec=MesosActionRun),
],
)
mock_filter.return_value = mock_actions
mock_action_runner = mock.Mock(autospec=True)
mock_job_run = mock.Mock()
launch_recovery_actionruns_for_job_runs([mock_job_run],
mock_action_runner)
ssh_runs = mock_actions[0]
calls = [
call(ssh_runs[0], mock_action_runner),
call(ssh_runs[1], ssh_runs[1].action_runner)
]
mock_recover_action_run.assert_has_calls(calls, any_order=True)
mesos_run = mock_actions[1][0]
assert mesos_run.recover.call_count == 1
def test_launch_recovery_actionruns_for_job_runs(self):
with mock.patch('tron.core.recovery.filter_action_runs_needing_recovery', autospec=True) as mock_filter, \
mock.patch('tron.core.recovery.recover_action_run', autospec=True) as mock_recover_action_run:
mock_actions = [
mock.Mock(action_runner=NoActionRunnerFactory(), spec=SSHActionRun),
mock.Mock(
action_runner=SubprocessActionRunnerFactory(
status_path='/tmp/foo', exec_path=('/tmp/foo')
),
spec=SSHActionRun,
),
mock.Mock(action_runner=NoActionRunnerFactory(), spec=MesosActionRun),
]
mock_filter.return_value = mock_actions
mock_action_runner = mock.Mock(autospec=True)
mock_job_run = mock.Mock()
launch_recovery_actionruns_for_job_runs([mock_job_run],
mock_action_runner)
ssh_runs = mock_actions[:2]
calls = [
call(ssh_runs[0], mock_action_runner),
call(ssh_runs[1], ssh_runs[1].action_runner)
]
mock_recover_action_run.assert_has_calls(calls, any_order=True)
mesos_run = mock_actions[2]
assert mesos_run.recover.call_count == 1
def test_launch_recovery_actionruns_for_job_runs(self, mock_filter, mock_recover_action_run):
mock_actions = (
[
mock.Mock(
action_runner=NoActionRunnerFactory(), spec=SSHActionRun
),
mock.Mock(
action_runner=SubprocessActionRunnerFactory(
status_path='/tmp/foo', exec_path=('/tmp/foo')
),
spec=SSHActionRun,
),
],
[
mock.Mock(
action_runner=NoActionRunnerFactory(), spec=MesosActionRun
),
],
)
mock_filter.return_value = mock_actions
mock_action_runner = mock.Mock(autospec=True)
mock_job_run = mock.Mock()
launch_recovery_actionruns_for_job_runs([mock_job_run],
mock_action_runner)
ssh_runs = mock_actions[0]
calls = [
call(ssh_runs[0], mock_action_runner),
call(ssh_runs[1], ssh_runs[1].action_runner)
]
mock_recover_action_run.assert_has_calls(calls, any_order=True)
mesos_run = mock_actions[1][0]
assert mesos_run.recover.call_count == 1
def test_launch_recovery_actionruns_empty_job_run(self, mock_filter):
"""_action_runs=None shouldn't prevent other job runs from being recovered"""
empty_job_run = mock.Mock(_action_runs=None)
other_job_run = mock.Mock(_action_runs=[mock.Mock()])
mock_action_runner = mock.Mock()
mock_filter.return_value = ([], [])
launch_recovery_actionruns_for_job_runs(
[empty_job_run, other_job_run],
mock_action_runner,
)
mock_filter.assert_called_with(other_job_run._action_runs)
def test_launch_recovery_actionruns_empty_job_run(self, mock_filter):
"""_action_runs=None shouldn't prevent other job runs from being recovered"""
empty_job_run = mock.Mock(_action_runs=None)
other_job_run = mock.Mock(_action_runs=[mock.Mock()])
mock_action_runner = mock.Mock()
mock_filter.return_value = ([], [])
launch_recovery_actionruns_for_job_runs(
[empty_job_run, other_job_run],
mock_action_runner,
)
mock_filter.assert_called_with(other_job_run._action_runs)
def restore_state(self, job_state_data, config_action_runner):
"""Restore the job state and schedule any JobRuns."""
job_runs = self.job.get_job_runs_from_state(job_state_data)
for run in job_runs:
self.job.watch(run)
self.job.runs.runs.extend(job_runs)
log.info(f'{self} restored')
recovery.launch_recovery_actionruns_for_job_runs(
job_runs=job_runs, master_action_runner=config_action_runner)
scheduled = self.job.runs.get_scheduled()
# for those that were already scheduled, we reschedule them to run.
for job_run in scheduled:
self._set_callback(job_run)
# Ensure we have at least 1 scheduled run
self.schedule()
def restore_state(self, job_state_data, config_action_runner):
"""Restore the job state and schedule any JobRuns."""
job_runs = self.job.get_job_runs_from_state(job_state_data)
for run in job_runs:
self.job.watch(run)
self.job.runs.runs.extend(job_runs)
log.info(f'{self} restored')
recovery.launch_recovery_actionruns_for_job_runs(
job_runs=job_runs, master_action_runner=config_action_runner
)
scheduled = self.job.runs.get_scheduled()
# for those that were already scheduled, we reschedule them to run.
for job_run in scheduled:
self._set_callback(job_run)
# Ensure we have at least 1 scheduled run
self.schedule()
