def _stale_resource_needs_retry(self, cnxt, rsrc, prev_template_id):
"""Determine whether a resource needs retrying after failure to lock.
Return True if we need to retry the check operation because of a
failure to acquire the lock. This can be either because the engine
holding the lock is no longer working, or because no other engine had
locked the resource and the data was just out of date.
In the former case, the lock will be stolen and the resource status
changed to FAILED.
"""
fields = {'current_template_id', 'engine_id'}
rs_obj = resource_objects.Resource.get_obj(cnxt,
rsrc.id,
refresh=True,
fields=fields)
if rs_obj.engine_id not in (None, self.engine_id):
if not listener_client.EngineListenerClient(
rs_obj.engine_id).is_alive(cnxt):
# steal the lock.
rs_obj.update_and_save({'engine_id': None})
# set the resource state as failed
status_reason = ('Worker went down '
'during resource %s' % rsrc.action)
rsrc.state_set(rsrc.action,
rsrc.FAILED,
six.text_type(status_reason))
return True
elif (rs_obj.engine_id is None and
rs_obj.current_template_id == prev_template_id):
LOG.debug('Resource id=%d stale; retrying check', rsrc.id)
return True
LOG.debug('Resource id=%d modified by another traversal', rsrc.id)
return False
def _try_steal_engine_lock(self, cnxt, resource_id):
rs_obj = resource_objects.Resource.get_obj(cnxt, resource_id)
if rs_obj.engine_id not in (None, self.engine_id):
if not listener_client.EngineListenerClient(
rs_obj.engine_id).is_alive(cnxt):
# steal the lock.
rs_obj.update_and_save({'engine_id': None})
return True
return False
def test_engine_alive_ok(self, rpc_client_method):
mock_rpc_client = rpc_client_method.return_value
mock_prepare_method = mock_rpc_client.prepare
mock_prepare_client = mock_prepare_method.return_value
mock_cnxt = mock.Mock()
listener_client = rpc_client.EngineListenerClient('engine-007')
rpc_client_method.assert_called_once_with(
version=rpc_client.EngineListenerClient.BASE_RPC_API_VERSION,
topic=rpc_api.LISTENER_TOPIC,
server='engine-007',
)
mock_prepare_method.assert_called_once_with(timeout=2)
self.assertEqual(mock_prepare_client, listener_client._client,
"Failed to create RPC client")
ret = listener_client.is_alive(mock_cnxt)
self.assertTrue(ret)
mock_prepare_client.call.assert_called_once_with(
mock_cnxt, 'listening')
def engine_alive(context, engine_id):
return listener_client.EngineListenerClient(engine_id).is_alive(
context)
