def testRunUnexpectedStopFromRegularVm_WithPreemptibles(
self, mock_dsub_call):
self._argv.extend([
'--jobs_to_run',
'postprocess_variants',
'--preemptible',
'--max_preemptible_tries',
'1',
'--max_non_preemptible_tries',
'2',
])
preemptible_exception = dsub_errors.JobExecutionError(
'error', ['Error in job - code 10: 14: VM stopped unexpectedly'])
unexpected_stop_exception = dsub_errors.JobExecutionError(
'error', ['Error in job - code 10: 13: VM stopped unexpectedly'])
mock_dsub_call.side_effect = [
preemptible_exception, unexpected_stop_exception,
unexpected_stop_exception
]
try:
gcp_deepvariant_runner.run(self._argv)
self.fail(
'Too many unexpected stops from regular VMs should throw exception.'
)
except RuntimeError:
pass
mock_dsub_call.assert_has_calls([
mock.call(mock.HASALLOF('postprocess_variants', '--preemptible')),
mock.call(mock.HASALLOF('postprocess_variants')),
mock.call(mock.HASALLOF('postprocess_variants'))
])
def testRunWithPreemptibles_NonPreemptibleFailure(self, mock_dsub_call):
self._argv.extend([
'--jobs_to_run',
'postprocess_variants',
'--preemptible',
'--max_preemptible_tries',
'2',
])
preemptible_exception = dsub_errors.JobExecutionError(
'error', ['Error in job - code 10: 13: VM stopped unexpectedly'])
# Note that the error code changed to '130' from '13'.
non_preemptible_exception = dsub_errors.JobExecutionError(
'error', ['Error in job - code 10: 130: VM stopped unexpectedly'])
mock_dsub_call.side_effect = [
preemptible_exception, non_preemptible_exception
]
try:
gcp_deepvariant_runner.run(self._argv)
self.fail('Non-preemptible failures should throw exception.')
except RuntimeError:
pass
# Two preemptible tries should have still happened.
mock_dsub_call.assert_has_calls([
mock.call(mock.HASALLOF('postprocess_variants', '--preemptible')),
mock.call(mock.HASALLOF('postprocess_variants', '--preemptible'))
])
def testRunUnexpectedStopFromRegularVm(self, mock_dsub_call):
self._argv.extend([
'--jobs_to_run',
'postprocess_variants',
'--max_non_preemptible_tries',
'3',
])
unexpected_stop_exception = dsub_errors.JobExecutionError(
'error', ['Error in job - code 10: 13: VM stopped unexpectedly'])
mock_dsub_call.side_effect = [
unexpected_stop_exception, unexpected_stop_exception, None
]
gcp_deepvariant_runner.run(self._argv)
mock_dsub_call.assert_has_calls([
mock.call(mock.HASALLOF('postprocess_variants')),
mock.call(mock.HASALLOF('postprocess_variants')),
mock.call(mock.HASALLOF('postprocess_variants'))
])
def testRunWithPreemptibles(self, mock_dsub_call):
self._argv.extend([
'--jobs_to_run',
'postprocess_variants',
'--preemptible',
'--max_preemptible_tries',
'2',
])
preemptible_exception = dsub_errors.JobExecutionError(
'error', ['Error in job - code 10: 14: VM stopped unexpectedly'])
mock_dsub_call.side_effect = [
preemptible_exception, preemptible_exception, None
]
gcp_deepvariant_runner.run(self._argv)
mock_dsub_call.assert_has_calls([
mock.call(mock.HASALLOF('postprocess_variants', '--preemptible')),
mock.call(mock.HASALLOF('postprocess_variants', '--preemptible')),
mock.call(mock.HASALLOF('postprocess_variants'))
])