def test_acquire_retry_never_acquired(self):
"""BasicLock.acquire will retry max_retry times and then give up."""
lock_uuid = uuid.uuid1()
clock = task.Clock()
lock = BasicLock(self.client,
self.table_name,
lock_uuid,
max_retry=1,
reactor=clock)
responses = [
defer.fail(BusyLockError('', '')),
defer.fail(BusyLockError('', ''))
]
def _new_verify_lock(response):
return responses.pop(0)
lock._verify_lock = _new_verify_lock
def _side_effect(*args, **kwargs):
return defer.succeed([])
self.client.execute.side_effect = _side_effect
d = lock.acquire()
clock.advance(20)
result = self.failureResultOf(d)
self.assertTrue(result.check(BusyLockError))
self.assertEqual(self.client.execute.call_count, 4)
def test_acquire(self):
"""Lock acquire should write and then read back its write."""
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid)
def _side_effect(*args, **kwargs):
return defer.succeed([{
'lockId': lock._lock_id,
'claimId': lock._claim_id
}])
self.client.execute.side_effect = _side_effect
d = lock.acquire()
self.assertEqual(self.assertFired(d), True)
expected = [
mock.call(
'INSERT INTO lock ("lockId","claimId") VALUES (:lockId,:claimId) USING TTL 300;',
{
'lockId': lock._lock_id,
'claimId': lock._claim_id
}, 2),
mock.call(
'SELECT * FROM lock WHERE "lockId"=:lockId ORDER BY "claimId";',
{'lockId': lock._lock_id}, 2)
]
self.assertEqual(self.client.execute.call_args_list, expected)
def test_acquire_retries_on_NoLockClaimsError(self):
"""
acquire retries when _verify_lock fails with a NoLockClaimsError.
"""
lock_uuid = uuid.uuid1()
clock = task.Clock()
lock = BasicLock(self.client,
self.table_name,
lock_uuid,
max_retry=1,
reactor=clock)
responses = [
defer.fail(NoLockClaimsError('', '')),
defer.succeed(True)
]
def _new_verify_lock(response):
return responses.pop(0)
lock._verify_lock = _new_verify_lock
def _side_effect(*args, **kwargs):
return defer.succeed([])
self.client.execute.side_effect = _side_effect
d = lock.acquire()
clock.advance(20)
self.assertEqual(self.assertFired(d), True)
self.assertEqual(self.client.execute.call_count, 4)
def test_acquire_logs(self, uuid1):
"""
When lock is acquired, it logs with time taken to acquire the log. Different claim ids
message is also logged. Intermittent 'release lock' messages are not logged
"""
lock_uuid = 'lock_uuid'
log = mock.MagicMock(spec=['msg'])
clock = task.Clock()
lock = BasicLock(self.client, self.table_name, lock_uuid, max_retry=1,
retry_wait=3, reactor=clock, log=log)
self.responses = [
None, # _write_lock
[{'lockId': lock._lock_id, 'claimId': 'wait for it..'}], # _read_lock
None, # delete for release lock
None, # _write_lock again
[{'lockId': lock._lock_id, 'claimId': lock._claim_id}] # _read_lock
]
d = lock.acquire()
clock.advance(5)
self.assertEqual(self.assertFired(d), True)
log.msg.assert_has_calls(
[mock.call('Got different claimId: wait for it..', diff_claim_id='wait for it..',
lock_id=lock_uuid, claim_id='claim_uuid'),
mock.call('Acquired lock in 5.0 seconds', lock_acquire_time=5.0,
lock_id=lock_uuid, claim_id='claim_uuid')])
def test_acquire_retry_never_acquired(self):
"""BasicLock.acquire will retry max_retry times and then give up."""
lock_uuid = uuid.uuid1()
clock = task.Clock()
lock = BasicLock(self.client, self.table_name, lock_uuid, max_retry=1, reactor=clock)
responses = [
defer.fail(BusyLockError('', '')),
defer.fail(BusyLockError('', ''))
]
def _new_verify_lock(response):
return responses.pop(0)
lock._verify_lock = _new_verify_lock
def _side_effect(*args, **kwargs):
return defer.succeed([])
self.client.execute.side_effect = _side_effect
d = lock.acquire()
clock.advance(20)
result = self.failureResultOf(d)
self.assertTrue(result.check(BusyLockError))
self.assertEqual(self.client.execute.call_count, 4)
def test_lock_acquire_anyfailure_logged(self, uuid1):
"""
If lock acquisition fails due to any error, it is logged along with time taken
"""
lock_uuid = 'lock_uuid'
log = mock.MagicMock(spec=['msg'])
clock = task.Clock()
lock = BasicLock(self.client, self.table_name, lock_uuid, max_retry=1,
retry_wait=3, reactor=clock, log=log)
self.responses = [
None, # _write_lock
[{'lockId': lock._lock_id, 'claimId': 'wait for it..'}], # _read_lock
None, # delete for release lock
None, # _write_lock again
]
def _execute(*args, **kwargs):
if not self.responses:
return defer.fail(ValueError('hmm'))
return defer.succeed(self.responses.pop(0))
self.client.execute.side_effect = _execute
d = lock.acquire()
clock.advance(3)
f = self.failureResultOf(d, ValueError)
log.msg.assert_called_with('Could not acquire lock in 3.0 seconds due to ' + str(f),
lock_acquire_fail_time=3.0, reason=f, lock_id=lock_uuid,
claim_id='claim_uuid')
def test_acquire_retries_on_NoLockClaimsError(self):
"""
acquire retries when _verify_lock fails with a NoLockClaimsError.
"""
lock_uuid = uuid.uuid1()
clock = task.Clock()
lock = BasicLock(self.client, self.table_name, lock_uuid, max_retry=1, reactor=clock)
responses = [
defer.fail(NoLockClaimsError('', '')),
defer.succeed(True)
]
def _new_verify_lock(response):
return responses.pop(0)
lock._verify_lock = _new_verify_lock
def _side_effect(*args, **kwargs):
return defer.succeed([])
self.client.execute.side_effect = _side_effect
d = lock.acquire()
clock.advance(20)
self.assertEqual(self.assertFired(d), True)
self.assertEqual(self.client.execute.call_count, 4)
def test_acquire_retry_not_lock_error(self):
"""If an error occurs that is not lock related, it is propagated."""
lock_uuid = uuid.uuid1()
clock = task.Clock()
lock = BasicLock(self.client,
self.table_name,
lock_uuid,
max_retry=1,
reactor=clock)
responses = [
defer.fail(
NameError('Keep your foot off the blasted samoflange.')),
]
def _new_verify_lock(response):
return responses.pop(0)
lock._verify_lock = _new_verify_lock
def _side_effect(*args, **kwargs):
return defer.succeed([])
self.client.execute.side_effect = _side_effect
d = lock.acquire()
result = self.failureResultOf(d)
self.assertTrue(result.check(NameError))
def test__verify_lock(self):
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid)
d = lock._verify_lock([{'lockId': lock._lock_id, 'claimId': lock._claim_id}])
result = self.assertFired(d)
self.assertEqual(result, True)
def test__verify_lock_no_rows(self):
"""
_verify_lock fails with an error when response contains no rows.
"""
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid)
d = lock._verify_lock([])
result = self.failureResultOf(d)
self.assertTrue(result.check(NoLockClaimsError))
def test__verify_lock_no_rows(self):
"""
_verify_lock fails with an error when response contains no rows.
"""
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid)
d = lock._verify_lock([])
result = self.failureResultOf(d)
self.assertTrue(result.check(NoLockClaimsError))
def test__verify_lock(self):
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid)
d = lock._verify_lock([{
'lockId': lock._lock_id,
'claimId': lock._claim_id
}])
result = self.assertFired(d)
self.assertEqual(result, True)
def test__read_lock(self):
lock_uuid = uuid.uuid1()
expected = [
'SELECT * FROM lock WHERE "lockId"=:lockId ORDER BY "claimId";',
{'lockId': lock_uuid}, 2]
lock = BasicLock(self.client, self.table_name, lock_uuid)
d = lock._read_lock(None)
self.assertEqual(self.assertFired(d), 1)
self.client.execute.assert_called_once_with(*expected)
def test__write_lock(self):
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid, 1000)
expected = [
'INSERT INTO lock ("lockId","claimId") VALUES (:lockId,:claimId) USING TTL 1000;',
{'lockId': lock_uuid, 'claimId': lock._claim_id}, 2]
d = lock._write_lock()
self.assertEqual(self.assertFired(d), 1)
self.client.execute.assert_called_once_with(*expected)
def test_release(self):
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid)
expected = [
'DELETE FROM lock WHERE "lockId"=:lockId AND "claimId"=:claimId;',
{'lockId': lock_uuid, 'claimId': lock._claim_id}, 2]
d = lock.release()
self.assertFired(d)
self.client.execute.assert_called_once_with(*expected)
def test__read_lock(self):
lock_uuid = uuid.uuid1()
expected = [
'SELECT * FROM lock WHERE "lockId"=:lockId ORDER BY "claimId";', {
'lockId': lock_uuid
}, 2
]
lock = BasicLock(self.client, self.table_name, lock_uuid)
d = lock._read_lock(None)
self.assertEqual(self.assertFired(d), 1)
self.client.execute.assert_called_once_with(*expected)
def test_drop_schema(self):
"""BasicLock.drop_schema deletes the table/columnfamily."""
expected = ['DROP TABLE lock', {}, 2]
d = BasicLock.drop_schema(self.client, 'lock')
self.successResultOf(d)
self.client.execute.assert_called_once_with(*expected)
def test_release(self):
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid)
expected = [
'DELETE FROM lock WHERE "lockId"=:lockId AND "claimId"=:claimId;',
{
'lockId': lock_uuid,
'claimId': lock._claim_id
}, 2
]
d = lock.release()
self.assertFired(d)
self.client.execute.assert_called_once_with(*expected)
def modify_state(self, modifier_callable, *args, **kwargs):
"""
see :meth:`otter.models.interface.IScalingGroup.modify_state`
"""
def _write_state(new_state):
assert (new_state.tenant_id == self.tenant_id
and new_state.group_id == self.uuid)
params = {
'tenantId': new_state.tenant_id,
'groupId': new_state.group_id,
'active': serialize_json_data(new_state.active, 1),
'pending': serialize_json_data(new_state.pending, 1),
'paused': new_state.paused,
'groupTouched': new_state.group_touched,
'policyTouched': serialize_json_data(new_state.policy_touched,
1)
}
return self.connection.execute(
_cql_insert_group_state.format(cf=self.group_table), params,
get_consistency_level('update', 'state'))
def _modify_state():
d = self.view_state()
d.addCallback(
lambda state: modifier_callable(self, state, *args, **kwargs))
return d.addCallback(_write_state)
lock = BasicLock(self.connection,
LOCK_TABLE_NAME,
self.uuid,
max_retry=5,
retry_wait=random.uniform(3, 5))
return with_lock(lock, _modify_state)
def test__write_lock(self):
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid, 1000)
expected = [
'INSERT INTO lock ("lockId","claimId") VALUES (:lockId,:claimId) USING TTL 1000;',
{
'lockId': lock_uuid,
'claimId': lock._claim_id
}, 2
]
d = lock._write_lock()
self.assertEqual(self.assertFired(d), 1)
self.client.execute.assert_called_once_with(*expected)
def test__verify_lock_release(self):
lock_uuid = uuid.uuid1()
def _side_effect(*args, **kwargs):
return defer.succeed(None)
self.client.execute.side_effect = _side_effect
lock = BasicLock(self.client, self.table_name, lock_uuid)
expected = [
'DELETE FROM lock WHERE "lockId"=:lockId AND "claimId"=:claimId;',
{'lockId': lock_uuid, 'claimId': lock._claim_id}, 2]
d = lock._verify_lock([{'lockId': lock._lock_id, 'claimId': ''}])
result = self.failureResultOf(d)
self.assertTrue(result.check(BusyLockError))
self.client.execute.assert_called_once_with(*expected)
def test_ensure_schema(self):
"""BasicLock.ensure_schema creates the table/columnfamily."""
expected = [
'CREATE TABLE lock ("lockId" ascii, "claimId" timeuuid, PRIMARY KEY("lockId", "claimId"));',
{}, 2]
d = BasicLock.ensure_schema(self.client, 'lock')
self.successResultOf(d)
self.client.execute.assert_called_once_with(*expected)
def test_drop_schema(self):
"""BasicLock.drop_schema deletes the table/columnfamily."""
expected = [
'DROP TABLE lock',
{}, 2]
d = BasicLock.drop_schema(self.client, 'lock')
self.successResultOf(d)
self.client.execute.assert_called_once_with(*expected)
def test_ensure_schema(self):
"""BasicLock.ensure_schema creates the table/columnfamily."""
expected = [
'CREATE TABLE lock ("lockId" ascii, "claimId" timeuuid, PRIMARY KEY("lockId", "claimId"));',
{}, 2
]
d = BasicLock.ensure_schema(self.client, 'lock')
self.successResultOf(d)
self.client.execute.assert_called_once_with(*expected)
def test_acquire_logs(self, uuid1):
"""
When lock is acquired, it logs with time taken to acquire the log. Different claim ids
message is also logged. Intermittent 'release lock' messages are not logged
"""
lock_uuid = 'lock_uuid'
log = mock.MagicMock(spec=['msg'])
clock = task.Clock()
lock = BasicLock(self.client,
self.table_name,
lock_uuid,
max_retry=1,
retry_wait=3,
reactor=clock,
log=log)
self.responses = [
None, # _write_lock
[{
'lockId': lock._lock_id,
'claimId': 'wait for it..'
}], # _read_lock
None, # delete for release lock
None, # _write_lock again
[{
'lockId': lock._lock_id,
'claimId': lock._claim_id
}] # _read_lock
]
d = lock.acquire()
clock.advance(5)
self.assertEqual(self.assertFired(d), True)
log.msg.assert_has_calls([
mock.call('Got different claimId: wait for it..',
diff_claim_id='wait for it..',
lock_id=lock_uuid,
claim_id='claim_uuid'),
mock.call('Acquired lock in 5.0 seconds',
lock_acquire_time=5.0,
lock_id=lock_uuid,
claim_id='claim_uuid')
])
def test_ensure_schema_already_created(self):
"""
BasicLock.ensure_schema doesn't explode on InvalidRequestException,
meaning the table already exists.
"""
def _side_effect(*args, **kwargs):
return defer.fail(InvalidRequestException())
self.client.execute.side_effect = _side_effect
d = BasicLock.ensure_schema(self.client, 'lock')
self.successResultOf(d)
def test_ensure_schema_already_created(self):
"""
BasicLock.ensure_schema doesn't explode on InvalidRequestException,
meaning the table already exists.
"""
def _side_effect(*args, **kwargs):
return defer.fail(InvalidRequestException())
self.client.execute.side_effect = _side_effect
d = BasicLock.ensure_schema(self.client, 'lock')
self.successResultOf(d)
def test_acquire(self):
"""Lock acquire should write and then read back its write."""
lock_uuid = uuid.uuid1()
lock = BasicLock(self.client, self.table_name, lock_uuid)
def _side_effect(*args, **kwargs):
return defer.succeed([{'lockId': lock._lock_id,
'claimId': lock._claim_id}])
self.client.execute.side_effect = _side_effect
d = lock.acquire()
self.assertEqual(self.assertFired(d), True)
expected = [
mock.call('INSERT INTO lock ("lockId","claimId") VALUES (:lockId,:claimId) USING TTL 300;',
{'lockId': lock._lock_id, 'claimId': lock._claim_id}, 2),
mock.call('SELECT * FROM lock WHERE "lockId"=:lockId ORDER BY "claimId";',
{'lockId': lock._lock_id}, 2)]
self.assertEqual(self.client.execute.call_args_list, expected)
def test_release_logs(self, uuid1):
"""
When lock is released, it logs with time the lock was held
"""
lock_uuid = 'lock_uuid'
log = mock.MagicMock(spec=['msg'])
clock = task.Clock()
lock = BasicLock(self.client,
self.table_name,
lock_uuid,
max_retry=1,
retry_wait=3,
reactor=clock,
log=log)
self.responses = [
None, # _write_lock
[{
'lockId': lock._lock_id,
'claimId': lock._claim_id
}], # _read_lock
None # delete for release lock
]
lock.acquire()
clock.advance(34)
lock.release()
log.msg.assert_called_with('Released lock. Was held for 34.0 seconds',
lock_held_time=34.0,
lock_id=lock_uuid,
claim_id='claim_uuid',
result=None)
def test_lock_acquire_anyfailure_logged(self, uuid1):
"""
If lock acquisition fails due to any error, it is logged along with time taken
"""
lock_uuid = 'lock_uuid'
log = mock.MagicMock(spec=['msg'])
clock = task.Clock()
lock = BasicLock(self.client,
self.table_name,
lock_uuid,
max_retry=1,
retry_wait=3,
reactor=clock,
log=log)
self.responses = [
None, # _write_lock
[{
'lockId': lock._lock_id,
'claimId': 'wait for it..'
}], # _read_lock
None, # delete for release lock
None, # _write_lock again
]
def _execute(*args, **kwargs):
if not self.responses:
return defer.fail(ValueError('hmm'))
return defer.succeed(self.responses.pop(0))
self.client.execute.side_effect = _execute
d = lock.acquire()
clock.advance(3)
f = self.failureResultOf(d, ValueError)
log.msg.assert_called_with(
'Could not acquire lock in 3.0 seconds due to ' + str(f),
lock_acquire_fail_time=3.0,
reason=f,
lock_id=lock_uuid,
claim_id='claim_uuid')
def test_release_logs(self, uuid1):
"""
When lock is released, it logs with time the lock was held
"""
lock_uuid = 'lock_uuid'
log = mock.MagicMock(spec=['msg'])
clock = task.Clock()
lock = BasicLock(self.client, self.table_name, lock_uuid, max_retry=1,
retry_wait=3, reactor=clock, log=log)
self.responses = [
None, # _write_lock
[{'lockId': lock._lock_id, 'claimId': lock._claim_id}], # _read_lock
None # delete for release lock
]
lock.acquire()
clock.advance(34)
lock.release()
log.msg.assert_called_with('Released lock. Was held for 34.0 seconds',
lock_held_time=34.0, lock_id=lock_uuid,
claim_id='claim_uuid', result=None)
def test__verify_lock_release(self):
lock_uuid = uuid.uuid1()
def _side_effect(*args, **kwargs):
return defer.succeed(None)
self.client.execute.side_effect = _side_effect
lock = BasicLock(self.client, self.table_name, lock_uuid)
expected = [
'DELETE FROM lock WHERE "lockId"=:lockId AND "claimId"=:claimId;',
{
'lockId': lock_uuid,
'claimId': lock._claim_id
}, 2
]
d = lock._verify_lock([{'lockId': lock._lock_id, 'claimId': ''}])
result = self.failureResultOf(d)
self.assertTrue(result.check(BusyLockError))
self.client.execute.assert_called_once_with(*expected)
def test_lock_acquire_failure_logged(self, uuid1):
"""
If lock acquisition fails due to BusyLockError, it is logged along with time taken
"""
lock_uuid = 'lock_uuid'
log = mock.MagicMock(spec=['msg'])
clock = task.Clock()
lock = BasicLock(self.client,
self.table_name,
lock_uuid,
max_retry=1,
retry_wait=3,
reactor=clock,
log=log)
self.responses = [
None, # _write_lock
[{
'lockId': lock._lock_id,
'claimId': 'wait for it..'
}], # _read_lock
None, # delete for release lock
None, # _write_lock again
[{
'lockId': lock._lock_id,
'claimId': 'nope'
}], # _read_lock
None # delete again for release lock
]
d = lock.acquire()
clock.advance(3)
f = self.failureResultOf(d, BusyLockError)
log.msg.assert_called_with(
'Could not acquire lock in 3.0 seconds due to ' + str(f),
lock_acquire_fail_time=3.0,
reason=f,
lock_id=lock_uuid,
claim_id='claim_uuid')
def test_acquire_retry_not_lock_error(self):
"""If an error occurs that is not lock related, it is propagated."""
lock_uuid = uuid.uuid1()
clock = task.Clock()
lock = BasicLock(self.client, self.table_name, lock_uuid, max_retry=1, reactor=clock)
responses = [
defer.fail(NameError('Keep your foot off the blasted samoflange.')),
]
def _new_verify_lock(response):
return responses.pop(0)
lock._verify_lock = _new_verify_lock
def _side_effect(*args, **kwargs):
return defer.succeed([])
self.client.execute.side_effect = _side_effect
d = lock.acquire()
result = self.failureResultOf(d)
self.assertTrue(result.check(NameError))
def test_lock_acquire_failure_logged(self, uuid1):
"""
If lock acquisition fails due to BusyLockError, it is logged along with time taken
"""
lock_uuid = 'lock_uuid'
log = mock.MagicMock(spec=['msg'])
clock = task.Clock()
lock = BasicLock(self.client, self.table_name, lock_uuid, max_retry=1,
retry_wait=3, reactor=clock, log=log)
self.responses = [
None, # _write_lock
[{'lockId': lock._lock_id, 'claimId': 'wait for it..'}], # _read_lock
None, # delete for release lock
None, # _write_lock again
[{'lockId': lock._lock_id, 'claimId': 'nope'}], # _read_lock
None # delete again for release lock
]
d = lock.acquire()
clock.advance(3)
f = self.failureResultOf(d, BusyLockError)
log.msg.assert_called_with('Could not acquire lock in 3.0 seconds due to ' + str(f),
lock_acquire_fail_time=3.0, reason=f, lock_id=lock_uuid,
claim_id='claim_uuid')
def __init__(self, batchsize, interval, slv_client, clock=None):
"""
Initializes the scheduler service with batch size and interval
:param int batchsize: number of events to fetch on each iteration
:param int interval: time between each iteration
:param slv_client: a :class:`silverberg.client.CQLClient` or
:class:`silverberg.cluster.RoundRobinCassandraCluster` instance used to get lock
:param clock: An instance of IReactorTime provider that defaults to reactor if not provided
"""
from otter.models.cass import LOCK_TABLE_NAME
self.lock = BasicLock(slv_client, LOCK_TABLE_NAME, 'schedule', max_retry=0)
TimerService.__init__(self, interval, self.check_for_events, batchsize)
self.clock = clock
self.log = otter_log.bind(system='otter.scheduler')
def delete_group(self):
"""
see :meth:`otter.models.interface.IScalingGroup.delete_group`
"""
# Events can only be deleted by policy id, since that and trigger are
# the only parts of the compound key
def _delete_everything(policies):
params = {'tenantId': self.tenant_id, 'groupId': self.uuid}
queries = [
_cql_delete_all_in_group.format(cf=table)
for table in (self.group_table, self.policies_table,
self.webhooks_table)
]
if len(policies) > 0:
events_query, events_params = _delete_many_query_and_params(
self.event_table, '"policyId"', policies.keys())
queries.append(events_query)
params.update(events_params)
b = Batch(queries,
params,
consistency=get_consistency_level('delete', 'group'))
return b.execute(self.connection)
def _maybe_delete(state):
if len(state.active) + len(state.pending) > 0:
raise GroupNotEmptyError(self.tenant_id, self.uuid)
d = self._naive_list_policies()
d.addCallback(_delete_everything)
return d
def _delete_group():
d = self.view_state()
d.addCallback(_maybe_delete)
return d
lock = BasicLock(self.connection,
LOCK_TABLE_NAME,
self.uuid,
max_retry=5,
retry_wait=random.uniform(3, 5))
return with_lock(lock, _delete_group)