def test_try_thread_lock_context_mgr_no_exception(self):
stack_lock_object.StackLock.create = mock.Mock(return_value=None)
stack_lock_object.StackLock.release = mock.Mock(return_value=None)
slock = stack_lock.StackLock(self.context, self.stack_id,
self.engine_id)
with slock.try_thread_lock():
self.assertEqual(1, stack_lock_object.StackLock.create.call_count)
assert not stack_lock_object.StackLock.release.called
def delete_stack(self, cnxt, stack_identity):
"""
The delete_stack method deletes a given stack.
:param cnxt: RPC context.
:param stack_identity: Name of the stack you want to delete.
"""
def remote_stop(lock_engine_id):
rpc = proxy.RpcProxy(lock_engine_id, "1.0")
msg = rpc.make_msg("stop_stack", stack_identity=stack_identity)
timeout = cfg.CONF.engine_life_check_timeout
try:
rpc.call(cnxt, msg, topic=lock_engine_id, timeout=timeout)
except rpc_common.Timeout:
return False
st = self._get_stack(cnxt, stack_identity)
logger.info(_('Deleting stack %s') % st.name)
stack = parser.Stack.load(cnxt, stack=st)
lock = stack_lock.StackLock(cnxt, stack, self.engine_id)
acquire_result = lock.try_acquire()
if acquire_result is None:
self.thread_group_mgr.start_with_acquired_lock(stack, lock,
stack.delete)
return
elif acquire_result == self.engine_id: # Current engine has the lock
self.thread_group_mgr.stop(stack.id)
# If the lock isn't released here, then the call to
# start_with_lock below will raise an ActionInProgress
# exception. Ideally, we wouldn't be calling another
# release() here, since it should be called as soon as the
# ThreadGroup is stopped. But apparently there's a race
# between release() the next call to lock.acquire().
db_api.stack_lock_release(stack.id, self.engine_id)
else: # Another engine has the lock
other_engine_id = acquire_result
stop_result = remote_stop(other_engine_id)
if stop_result is None:
logger.debug(_("Successfully stopped remote task on engine %s")
% other_engine_id)
else:
raise exception.StopActionFailed(stack_name=stack.name,
engine_id=other_engine_id)
# There may be additional resources that we don't know about
# if an update was in-progress when the stack was stopped, so
# reload the stack from the database.
st = self._get_stack(cnxt, stack_identity)
stack = parser.Stack.load(cnxt, stack=st)
self.thread_group_mgr.start_with_lock(cnxt, stack, self.engine_id,
stack.delete)
return None
def test_failed_acquire_existing_lock_current_engine(self):
mock_create = self.patchobject(stack_lock_object.StackLock,
'create',
return_value=self.engine_id)
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.assertRaises(exception.ActionInProgress, slock.acquire)
mock_create.assert_called_once_with(self.stack.id, self.engine_id)
def test_successful_acquire_new_lock(self):
mock_create = self.patchobject(stack_lock_object.StackLock,
'create',
return_value=None)
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
slock.acquire()
mock_create.assert_called_once_with(self.stack.id, self.engine_id)
def test_successful_acquire_new_lock(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, self.engine_id).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
slock.acquire()
self.m.VerifyAll()
def test_failed_acquire_existing_lock_current_engine(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn(self.engine_id)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.assertRaises(rpc_common.ClientException, slock.acquire)
self.m.VerifyAll()
def test_try_thread_lock_context_mgr_existing_lock(self):
db_api.stack_lock_create = mock.Mock(return_value=1234)
db_api.stack_lock_release = mock.Mock(return_value=None)
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
try:
with slock.try_thread_lock(self.stack.id):
db_api.stack_lock_create.assert_called_once()
raise Exception
except:
assert not db_api.stack_lock_release.called
def test_try_thread_lock_context_mgr_exception(self):
db_api.stack_lock_create = mock.Mock(return_value=None)
db_api.stack_lock_release = mock.Mock(return_value=None)
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
try:
with slock.try_thread_lock(self.stack.id):
self.assertEqual(1, db_api.stack_lock_create.call_count)
raise Exception
except:
self.assertEqual(1, db_api.stack_lock_release.call_count)
def test_engine_alive_timeout(self):
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
mget_client = self.patchobject(stack_lock.rpc_messaging,
'get_rpc_client')
mclient = mget_client.return_value
mclient_ctx = mclient.prepare.return_value
mclient_ctx.call.side_effect = messaging.MessagingTimeout('too slow')
ret = slock.engine_alive(self.context, self.engine_id)
self.assertIs(False, ret)
mclient.prepare.assert_called_once_with(timeout=2)
mclient_ctx.call.assert_called_once_with(self.context, 'listening')
def test_try_thread_lock_context_mgr_existing_lock(self):
db_api.stack_lock_create = mock.Mock(return_value=1234)
db_api.stack_lock_release = mock.Mock(return_value=None)
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
def check_thread_lock():
with slock.try_thread_lock(self.stack.id):
self.assertEqual(1, db_api.stack_lock_create.call_count)
raise self.TestThreadLockException
self.assertRaises(self.TestThreadLockException, check_thread_lock)
assert not db_api.stack_lock_release.called
def test_thread_lock_context_mgr_exception_acquire_fail(self):
db_api.stack_lock_create = mock.Mock(return_value=self.engine_id)
db_api.stack_lock_release = mock.Mock()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
def check_thread_lock():
with slock.thread_lock(self.stack.id):
self.assertEqual(1, db_api.stack_lock_create.call_count)
raise exception.ActionInProgress
self.assertRaises(exception.ActionInProgress, check_thread_lock)
assert not db_api.stack_lock_release.called
def test_failed_acquire_existing_lock_engine_alive(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=True)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()
def test_engine_alive_ok(self):
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
mget_client = self.patchobject(stack_lock.rpc_messaging,
'get_rpc_client')
mclient = mget_client.return_value
mclient_ctx = mclient.prepare.return_value
mclient_ctx.call.return_value = True
ret = slock.engine_alive(self.context, self.engine_id)
self.assertTrue(ret)
mclient.prepare.assert_called_once_with(timeout=2)
mclient_ctx.call.assert_called_once_with(self.context, 'listening')
def test_try_thread_lock_context_mgr_existing_lock(self):
stack_lock_object.StackLock.create = mock.Mock(return_value=1234)
stack_lock_object.StackLock.release = mock.Mock(return_value=None)
slock = stack_lock.StackLock(self.context, self.stack_id,
self.engine_id)
def check_thread_lock():
with slock.try_thread_lock():
self.assertEqual(1,
stack_lock_object.StackLock.create.call_count)
raise self.TestThreadLockException
self.assertRaises(self.TestThreadLockException, check_thread_lock)
self.assertFalse(stack_lock_object.StackLock.release.called)
def test_thread_lock_context_mgr_exception_acquire_success(self):
stack_lock_object.StackLock.create = mock.Mock(return_value=None)
stack_lock_object.StackLock.release = mock.Mock(return_value=None)
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
def check_thread_lock():
with slock.thread_lock(self.stack.id):
self.assertEqual(1,
stack_lock_object.StackLock.create.call_count)
raise self.TestThreadLockException
self.assertRaises(self.TestThreadLockException, check_thread_lock)
self.assertEqual(1, stack_lock_object.StackLock.release.call_count)
def test_thread_lock_context_mgr_exception_acquire_fail(self):
stack_lock_object.StackLock.create = mock.Mock(
return_value=self.engine_id)
stack_lock_object.StackLock.release = mock.Mock()
slock = stack_lock.StackLock(self.context, self.stack_id,
self.engine_id)
def check_thread_lock():
with slock.thread_lock():
self.assertEqual(1,
stack_lock_object.StackLock.create.call_count)
raise exception.ActionInProgress
self.assertRaises(exception.ActionInProgress, check_thread_lock)
self.assertFalse(stack_lock_object.StackLock.release.called)
def test_failed_acquire_existing_lock_engine_alive(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(messaging.rpc.client._CallContext, "call")
messaging.rpc.client._CallContext.call(self.context,
"listening").AndReturn(True)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()
def test_successful_acquire_existing_lock_engine_dead(self):
mock_create = self.patchobject(stack_lock_object.StackLock,
'create',
return_value='fake-engine-id')
mock_steal = self.patchobject(stack_lock_object.StackLock,
'steal',
return_value=None)
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=False)
slock.acquire()
mock_create.assert_called_once_with(self.stack.id, self.engine_id)
mock_steal.assert_called_once_with(self.stack.id, 'fake-engine-id',
self.engine_id)
def test_failed_acquire_existing_lock_engine_dead(self):
mock_create = self.patchobject(stack_lock_object.StackLock,
'create',
return_value='fake-engine-id')
mock_steal = self.patchobject(stack_lock_object.StackLock,
'steal',
return_value='fake-engine-id2')
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=False)
self.assertRaises(exception.ActionInProgress, slock.acquire)
mock_create.assert_called_once_with(self.stack.id, self.engine_id)
mock_steal.assert_called_once_with(self.stack.id, 'fake-engine-id',
self.engine_id)
def test_successful_acquire_existing_lock_engine_dead(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=False)
slock.acquire()
self.m.VerifyAll()
def test_failed_acquire_existing_lock_engine_alive(self):
mock_create = self.patchobject(stack_lock_object.StackLock,
'create',
return_value='fake-engine-id')
slock = stack_lock.StackLock(self.context, self.stack_id,
self.engine_id)
self.patchobject(service_utils, 'engine_alive', return_value=True)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.mock_get_by_id.assert_called_once_with(self.context,
self.stack_id,
tenant_safe=False,
show_deleted=True)
mock_create.assert_called_once_with(self.stack_id, self.engine_id)
def test_successful_acquire_with_retry(self):
mock_create = self.patchobject(stack_lock_object.StackLock,
'create',
return_value='fake-engine-id')
mock_steal = self.patchobject(stack_lock_object.StackLock,
'steal',
side_effect=[True, None])
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=False)
slock.acquire()
mock_create.assert_has_calls(
[mock.call(self.stack.id, self.engine_id)] * 2)
mock_steal.assert_has_calls(
[mock.call(self.stack.id, 'fake-engine-id', self.engine_id)] * 2)
def test_failed_acquire_existing_lock_engine_alive(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
self.m.StubOutWithMock(proxy.RpcProxy, "call")
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndReturn(True)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()
def test_failed_acquire_one_retry_only(self):
mock_create = self.patchobject(stack_lock_object.StackLock,
'create',
return_value='fake-engine-id')
mock_steal = self.patchobject(stack_lock_object.StackLock,
'steal',
return_value=True)
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=False)
self.assertRaises(exception.ActionInProgress, slock.acquire)
mock_create.assert_has_calls(
[mock.call(self.stack.id, self.engine_id)] * 2)
mock_steal.assert_has_calls(
[mock.call(self.stack.id, 'fake-engine-id', self.engine_id)] * 2)
def abandon_stack(self, cnxt, stack_identity):
"""
The abandon_stack method abandons a given stack.
:param cnxt: RPC context.
:param stack_identity: Name of the stack you want to abandon.
"""
st = self._get_stack(cnxt, stack_identity)
LOG.info(_('abandoning stack %s') % st.name)
stack = parser.Stack.load(cnxt, stack=st)
lock = stack_lock.StackLock(cnxt, stack, self.engine_id)
with lock.thread_lock(stack.id):
# Get stack details before deleting it.
stack_info = stack.prepare_abandon()
self.thread_group_mgr.start_with_acquired_lock(
stack, lock, stack.delete)
return stack_info
def start_with_lock(self, cnxt, stack, engine_id, func, *args, **kwargs):
"""
Try to acquire a stack lock and, if successful, run the given
method in a sub-thread. Release the lock when the thread
finishes.
:param cnxt: RPC context
:param stack: Stack to be operated on
:type stack: heat.engine.parser.Stack
:param engine_id: The UUID of the engine acquiring the lock
:param func: Callable to be invoked in sub-thread
:type func: function or instancemethod
:param args: Args to be passed to func
:param kwargs: Keyword-args to be passed to func.
"""
lock = stack_lock.StackLock(cnxt, stack, engine_id)
with lock.thread_lock(stack.id):
self.start_with_acquired_lock(stack, lock, func, *args, **kwargs)
def test_successful_acquire_existing_lock_engine_dead(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(messaging.rpc.client._CallContext, "call")
messaging.rpc.client._CallContext.call(
self.context, "listening").AndRaise(messaging.MessagingTimeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
slock.acquire()
self.m.VerifyAll()
def test_successful_acquire_existing_lock_engine_dead(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
self.m.StubOutWithMock(proxy.RpcProxy, "call")
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
slock.acquire()
self.m.VerifyAll()
def test_failed_acquire_one_retry_only(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
self.m.StubOutWithMock(proxy.RpcProxy, "call")
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context,
rpc.make_msg("listening"),
timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(True)
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context,
rpc.make_msg("listening"),
timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(True)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.assertRaises(rpc_common.ClientException, slock.acquire)
self.m.VerifyAll()