def run_job(self,
uuid,
job_parms=None,
timeout=CONF.pypowervm_job_request_timeout,
sensitive=False,
synchronous=True):
"""Invokes and polls a job.
Adds job parameters to the job element if specified and calls the
create_job method. It then monitors the job for completion and sends a
JobRequestFailed exception if it did not complete successfully.
:param uuid: uuid of the target
:param job_parms: list of JobParamters to add
:param timeout: maximum number of seconds for job to complete
:param sensitive: If True, mask the Job payload in the logs.
:param synchronous: If True (the default), wait for the Job to complete
or time out. If False, return as soon as the Job
starts. Note that this may still involve polling
(if the Job is waiting in queue to start), and may
still time out (if the Job hasn't started within
the requested timeout.)
:raise JobRequestFailed: if the job did not complete successfully.
:raise JobRequestTimedOut: if the job timed out.
"""
if job_parms:
self.add_job_parameters_to_existing(*job_parms)
try:
self.entry = self.adapter.create_job(
self.entry.element,
self._get_val_str(_JOB_GROUP_NAME),
uuid,
sensitive=sensitive).entry
except pvmex.Error as exc:
LOG.exception(exc)
raise pvmex.JobRequestFailed(operation_name=self.op, error=exc)
timed_out = self._monitor_job(timeout=timeout,
sensitive=sensitive,
synchronous=synchronous)
if timed_out:
try:
self.cancel_job()
except pvmex.JobRequestFailed as e:
LOG.warn(six.text_type(e))
exc = pvmex.JobRequestTimedOut(operation_name=self.op,
seconds=timeout)
LOG.error(exc.args[0])
raise exc
if not synchronous:
# _monitor_job spawned a subthread that will delete_job when done.
return
self.delete_job()
if self.job_status != JobStatus.COMPLETED_OK:
exc = pvmex.JobRequestFailed(operation_name=self.op,
error=self.get_job_message(''))
LOG.error(exc.args[0])
raise exc
def test_force_immed_no_retry(self, mock_vios, mock_run_job):
"""With force_immediate=NO_RETRY, errors don't retry."""
mock_vios.adapter = self.adpt
mock_vios.rmc_state = pvm_bp.RMCState.ACTIVE
for exc in (pexc.JobRequestFailed(error='e', operation_name='op'),
pexc.JobRequestTimedOut(operation_name='op', seconds=60)):
mock_run_job.side_effect = exc
self.assertRaises(pexc.VMPowerOffFailure,
power.power_off,
mock_vios,
'huuid',
force_immediate=power.Force.NO_RETRY)
self.assertEqual(1, mock_run_job.call_count)
mock_run_job.reset_mock()
def test_power_off_timeout_retry_vios(self, mock_vios, mock_job_p,
mock_run_job):
"""Validate that when first power off times out, re-run."""
mock_vios.adapter = self.adpt
mock_vios.rmc_state = pvm_bp.RMCState.ACTIVE
mock_run_job.side_effect = pexc.JobRequestTimedOut(
operation_name='PowerOff', seconds=60)
# Invoke the run, should power off graceful, fail, then force off
# and fail again.
self.assertRaises(pexc.VMPowerOffFailure,
power._power_on_off, mock_vios, 'PowerOff', '1111')
# It should have been called twice, once for the elegant power
# off, and another for the immediate power off
self.assertEqual(2, mock_run_job.call_count)
def etimeout():
"""Returns a JobRequestTimedOut exception."""
return pexc.JobRequestTimedOut(operation_name='foo', seconds=1800)
def test_power_off_timeout_retry(self, mock_lpar, mock_job_p,
mock_run_job):
"""Validate that when first power off times out, re-run."""
mock_lpar.adapter = self.adpt
mock_lpar.rmc_state = pvm_bp.RMCState.ACTIVE
mock_run_job.side_effect = pexc.JobRequestTimedOut(
operation_name='PowerOff', seconds=60)
# Invoke the run, should power off graceful, fail, then force off
# and fail again.
self.assertRaises(pexc.VMPowerOffFailure, power._power_on_off,
mock_lpar, 'PowerOff', '1111')
# It should have been called twice, once for the elegant power
# off, and another for the immediate power off
self.assertEqual(2, mock_run_job.call_count)
mock_job_p.assert_has_calls([
mock.call('operation', 'osshutdown'),
mock.call('immediate', 'true'),
mock.call('operation', 'shutdown'),
mock.call('immediate', 'true')
])
# Try a timedout only for the 2nd and 3rd job
mock_run_job.reset_mock()
mock_job_p.reset_mock()
rmc_error = power._OSSHUTDOWN_RMC_ERRS[0]
mock_run_job.side_effect = [
pexc.JobRequestFailed(error='PowerOff', operation_name=rmc_error),
pexc.JobRequestTimedOut(operation_name='PowerOff', seconds=60),
pexc.JobRequestTimedOut(operation_name='PowerOff', seconds=60)
]
self.assertRaises(pexc.VMPowerOffFailure, power._power_on_off,
mock_lpar, 'PowerOff', '1111')
# It should have been called three times,
# once for the immediate os shutdown, once for vsp normal,
# and another time for vsp hard
self.assertEqual(3, mock_run_job.call_count)
mock_job_p.assert_has_calls([
mock.call('operation', 'osshutdown'),
mock.call('immediate', 'true'),
mock.call('operation', 'shutdown'),
mock.call('operation', 'shutdown'),
mock.call('immediate', 'true')
])
# Try IBMi
mock_run_job.reset_mock()
mock_job_p.reset_mock()
mock_lpar.rmc_state = pvm_bp.RMCState.INACTIVE
mock_lpar.env = pvm_bp.LPARType.OS400
mock_lpar.ref_code = '00000000'
mock_run_job.side_effect = pexc.JobRequestTimedOut(
operation_name='PowerOff', seconds=60)
self.assertRaises(pexc.VMPowerOffFailure, power._power_on_off,
mock_lpar, 'PowerOff', '1111')
# It should have been called four times,
# once for the normal os shutdown,
# once for the immediate os shutdown, once for vsp normal,
# and another time for vsp hard
self.assertEqual(4, mock_run_job.call_count)
mock_job_p.assert_has_calls([
mock.call('operation', 'osshutdown'),
mock.call('operation', 'osshutdown'),
mock.call('immediate', 'true'),
mock.call('operation', 'shutdown'),
mock.call('operation', 'shutdown'),
mock.call('immediate', 'true')
])