def test_failed_acquire_one_retry_only(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(
self.stack.id, self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(messaging.rpc.client._CallContext, "call")
messaging.rpc.client._CallContext.call(
self.context, "listening").AndRaise(messaging.MessagingTimeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(
self.stack.id, "fake-engine-id", self.engine_id).AndReturn(True)
db_api.stack_lock_create(
self.stack.id, self.engine_id).AndReturn("fake-engine-id")
messaging.rpc.client._CallContext.call(
self.context, "listening").AndRaise(messaging.MessagingTimeout)
db_api.stack_lock_steal(
self.stack.id, "fake-engine-id", self.engine_id).AndReturn(True)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()
def test_failed_acquire_one_retry_only(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
self.m.StubOutWithMock(proxy.RpcProxy, "call")
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(True)
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(True)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()
def test_successful_acquire_with_retry(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(messaging.rpc.client._CallContext, "call")
messaging.rpc.client._CallContext.call(
self.context, "listening").AndRaise(messaging.MessagingTimeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn(True)
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
messaging.rpc.client._CallContext.call(
self.context, "listening").AndRaise(messaging.MessagingTimeout)
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
slock.acquire()
self.m.VerifyAll()
def test_successful_acquire_with_retry(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(
self.stack.id, self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(messaging.rpc.client._CallContext, "call")
messaging.rpc.client._CallContext.call(
self.context, "listening").AndRaise(messaging.MessagingTimeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(
self.stack.id, "fake-engine-id", self.engine_id).AndReturn(True)
db_api.stack_lock_create(
self.stack.id, self.engine_id).AndReturn("fake-engine-id")
messaging.rpc.client._CallContext.call(
self.context, "listening").AndRaise(messaging.MessagingTimeout)
db_api.stack_lock_steal(
self.stack.id, "fake-engine-id", self.engine_id).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
slock.acquire()
self.m.VerifyAll()
def test_successful_acquire_with_retry(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
self.m.StubOutWithMock(proxy.RpcProxy, "call")
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(True)
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
slock.acquire()
self.m.VerifyAll()
def test_failed_acquire_one_retry_only(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
self.m.StubOutWithMock(proxy.RpcProxy, "call")
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(True)
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(True)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()
def test_failed_acquire_one_retry_only(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(messaging.rpc.client._CallContext, "call")
messaging.rpc.client._CallContext.call(
self.context, "listening").AndRaise(messaging.MessagingTimeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn(True)
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
messaging.rpc.client._CallContext.call(
self.context, "listening").AndRaise(messaging.MessagingTimeout)
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn(True)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()
def test_successful_acquire_with_retry(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
self.m.StubOutWithMock(proxy.RpcProxy, "call")
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(True)
db_api.stack_lock_create(self.stack.id, self.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).\
AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
slock.acquire()
self.m.VerifyAll()
def test_successful_acquire_existing_lock_engine_dead(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=False)
slock.acquire()
self.m.VerifyAll()
def test_successful_acquire_existing_lock_engine_dead(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(
self.stack.id, self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=False)
slock.acquire()
self.m.VerifyAll()
def test_failed_acquire_existing_lock_engine_dead(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id,
self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
self.engine_id).AndReturn("fake-engine-id2")
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=False)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()
def test_successful_acquire_steal_lock_updated(self):
existing_lock = self.m.CreateMockAnything()
existing_lock.updated_at = datetime.datetime(2012, 10, 16, 18, 35, 18)
current_time = datetime.datetime(2012, 10, 16, 18, 36, 29)
self.m.StubOutWithMock(db_api, "current_timestamp")
db_api.current_timestamp().AndReturn(current_time)
self.m.StubOutWithMock(db_api, "stack_lock_get")
db_api.stack_lock_get(mox.IgnoreArg(), mox.IgnoreArg())\
.AndReturn(existing_lock)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(None)
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack)
slock.acquire()
self.m.VerifyAll()
def test_failed_acquire_existing_lock_engine_dead(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(
self.stack.id, self.engine_id).AndReturn("fake-engine-id")
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(
self.stack.id, "fake-engine-id",
self.engine_id).AndReturn("fake-engine-id2")
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack, self.engine_id)
self.patchobject(slock, 'engine_alive', return_value=False)
self.assertRaises(exception.ActionInProgress, slock.acquire)
self.m.VerifyAll()
def test_failed_acquire_existing_lock_engine_dead(self):
self.m.StubOutWithMock(db_api, "stack_lock_create")
db_api.stack_lock_create(self.stack.id, stack_lock.engine_id).\
AndReturn("fake-engine-id")
topic = self.stack.id
self.m.StubOutWithMock(proxy.RpcProxy, "call")
rpc = proxy.RpcProxy(topic, "1.0")
rpc.call(self.context, rpc.make_msg("listening"), timeout=2,
topic="fake-engine-id").AndRaise(rpc_common.Timeout)
self.m.StubOutWithMock(db_api, "stack_lock_steal")
db_api.stack_lock_steal(self.stack.id, "fake-engine-id",
stack_lock.engine_id).\
AndReturn("fake-engine-id2")
self.m.ReplayAll()
slock = stack_lock.StackLock(self.context, self.stack)
self.assertRaises(rpc_common.ClientException, slock.acquire)
self.m.VerifyAll()
def acquire(self, retry=True):
"""
Acquire a lock on the stack.
:param retry: When True, retry if lock was released while stealing.
:type retry: boolean
"""
lock_engine_id = db_api.stack_lock_create(self.stack.id,
self.engine_id)
if lock_engine_id is None:
LOG.debug("Engine %(engine)s acquired lock on stack "
"%(stack)s" % {'engine': self.engine_id,
'stack': self.stack.id})
return
if lock_engine_id == self.engine_id or \
self.engine_alive(self.context, lock_engine_id):
LOG.debug("Lock on stack %(stack)s is owned by engine "
"%(engine)s" % {'stack': self.stack.id,
'engine': lock_engine_id})
raise exception.ActionInProgress(stack_name=self.stack.name,
action=self.stack.action)
else:
LOG.info(_("Stale lock detected on stack %(stack)s. Engine "
"%(engine)s will attempt to steal the lock")
% {'stack': self.stack.id, 'engine': self.engine_id})
result = db_api.stack_lock_steal(self.stack.id, lock_engine_id,
self.engine_id)
if result is None:
LOG.info(_("Engine %(engine)s successfully stole the lock "
"on stack %(stack)s")
% {'engine': self.engine_id,
'stack': self.stack.id})
return
elif result is True:
if retry:
LOG.info(_("The lock on stack %(stack)s was released "
"while engine %(engine)s was stealing it. "
"Trying again") % {'stack': self.stack.id,
'engine': self.engine_id})
return self.acquire(retry=False)
else:
new_lock_engine_id = result
LOG.info(_("Failed to steal lock on stack %(stack)s. "
"Engine %(engine)s stole the lock first")
% {'stack': self.stack.id,
'engine': new_lock_engine_id})
raise exception.ActionInProgress(
stack_name=self.stack.name, action=self.stack.action)
def acquire(self, retry=True):
"""
Acquire a lock on the stack.
:param retry: When True, retry if lock was released while stealing.
:type retry: boolean
"""
lock_engine_id = db_api.stack_lock_create(self.stack.id,
self.engine_id)
if lock_engine_id is None:
logger.debug(_("Engine %(engine)s acquired lock on stack "
"%(stack)s") % {'engine': self.engine_id,
'stack': self.stack.id})
return
if lock_engine_id == self.engine_id or \
self.engine_alive(self.context, lock_engine_id):
logger.debug(_("Lock on stack %(stack)s is owned by engine "
"%(engine)s") % {'stack': self.stack.id,
'engine': lock_engine_id})
raise exception.ActionInProgress(stack_name=self.stack.name,
action=self.stack.action)
else:
logger.info(_("Stale lock detected on stack %(stack)s. Engine "
"%(engine)s will attempt to steal the lock")
% {'stack': self.stack.id, 'engine': self.engine_id})
result = db_api.stack_lock_steal(self.stack.id, lock_engine_id,
self.engine_id)
if result is None:
logger.info(_("Engine %(engine)s successfully stole the lock "
"on stack %(stack)s")
% {'engine': self.engine_id,
'stack': self.stack.id})
return
elif result is True:
if retry:
logger.info(_("The lock on stack %(stack)s was released "
"while engine %(engine)s was stealing it. "
"Trying again") % {'stack': self.stack.id,
'engine': self.engine_id})
return self.acquire(retry=False)
else:
new_lock_engine_id = result
logger.info(_("Failed to steal lock on stack %(stack)s. "
"Engine %(engine)s stole the lock first")
% {'stack': self.stack.id,
'engine': new_lock_engine_id})
raise exception.ActionInProgress(
stack_name=self.stack.name, action=self.stack.action)
def steal(cls, stack_id, old_engine_id, new_engine_id):
return db_api.stack_lock_steal(stack_id, old_engine_id, new_engine_id)
def steal(cls, stack_id, old_engine_id, new_engine_id):
return db_api.stack_lock_steal(stack_id,
old_engine_id,
new_engine_id)