def test_add_collision_when_missing(self, mock_logger):
correlation_id = uuid.uuid4().hex
# init aws memcache connection
connection = aws.get_connection(
self.mock_settings.PRIMARY_CACHE_SOURCE)
def inline(x, c, ac):
y = c.original_gets(x)
ac.add('lease-' + correlation_id, '99:99:99:99')
return y
try:
connection.original_gets = connection.gets
another_connection = memcache.Client(['localhost:11211'],
cache_cas=True)
connection.gets = lambda x: inline(x, connection,
another_connection)
acquired = aws.acquire_lease(correlation_id, 1, 1, primary=True)
self.assertFalse(acquired)
self.assertEqual(
mock.call.warn(
"Cannot acquire memcache lease: unexpectedly lost 'memcache.add' race"
), mock_logger.mock_calls[-1])
finally:
connection.gets = connection.original_gets
del connection.original_gets
def test_watch_collision_when_expired(self, mock_logger):
correlation_id = uuid.uuid4().hex
# init aws redis connection
connection = aws.get_connection(
self.mock_settings.PRIMARY_CACHE_SOURCE)
connection.setex('lease-' + correlation_id, 3600, '-1:-1:0:99')
def inline(x, c, ac):
r = aws.original_deserialize_lease_value(x)
ac.setex('lease-' + correlation_id, 3600, '99:99:99:99')
return r
try:
aws.original_deserialize_lease_value = aws._deserialize_lease_value
another_connection = redis.StrictRedis(host='localhost',
port=6379,
db=0)
aws._deserialize_lease_value = lambda x: inline(
x, connection, another_connection)
acquired = aws.acquire_lease(correlation_id, 1, 1, primary=True)
self.assertFalse(acquired)
self.assertEqual(
mock.call.warn(
"Cannot acquire redis lease: unexpectedly lost 'pipe.watch' race"
), mock_logger.mock_calls[-1])
finally:
aws._deserialize_lease_value = aws._deserialize_lease_value
del aws.original_deserialize_lease_value
def dispatch(self, event, obj):
"""
Acquires an exclusive lease for the machine's correlation_id, and executes
all the machinery of the framework and all the user code.
:param event: a str event.
:param obj: a dict.
"""
fence_token = None
try:
# attempt to acquire the lease and execute the state transition
fence_token = acquire_lease(self.correlation_id,
self.steps,
self.retries,
primary=self.lease_primary)
if fence_token == 0:
self._queue_error(
ERRORS.CACHE, 'System error acquiring primary=%s lease.' %
self.lease_primary)
self.lease_primary = not self.lease_primary
fence_token = acquire_lease(self.correlation_id,
self.steps,
self.retries,
primary=self.lease_primary)
if not fence_token:
# could not get the lease. something is going wrong
self._queue_error(ERRORS.CACHE,
'Could not acquire lease. Retrying.')
self._retry(obj)
else:
# lease acquired, execute the state transition
self._dispatch_and_retry(event, obj)
finally:
released = release_lease(self.correlation_id,
self.steps,
self.retries,
fence_token,
primary=self.lease_primary)
if not released:
self._queue_error(ERRORS.CACHE, 'Could not release lease.')
def smoke(self):
correlation_id = uuid.uuid4().hex
# can acquire lease
acquired = aws.acquire_lease(correlation_id, 1, 1, primary=True)
self.assertTrue(acquired is 1)
# cannot re-acquire own lease (not re-entrant)
acquired = aws.acquire_lease(correlation_id, 1, 1, primary=True)
self.assertTrue(acquired is False)
# cannot acquire someone else's lease
acquired = aws.acquire_lease(correlation_id, 1, 2, primary=True)
self.assertTrue(acquired is False)
# cannot release someone else's lease
released = aws.release_lease(correlation_id, 1, 2, 1, primary=True)
self.assertTrue(released is False)
# cannot release own lease (different fence token)
released = aws.release_lease(correlation_id, 1, 1, 99, primary=True)
self.assertTrue(released is False)
# can release own lease
released = aws.release_lease(correlation_id, 1, 1, 1, primary=True)
self.assertTrue(released is True)
# someone else can acquire new lease with short timeout
acquired = aws.acquire_lease(correlation_id,
1,
2,
primary=True,
timeout=1)
self.assertTrue(acquired is 2)
time.sleep(2)
# someone else can acquire new lease when previous times out
acquired = aws.acquire_lease(correlation_id, 1, 2, primary=True)
self.assertTrue(acquired is 3)
def dispatch(self, event, obj):
"""
Acquires an exclusive lease for the machine's correlation_id, and executes
all the machinery of the framework and all the user code.
In Marting Kleppmann's blog entry "How to do distributed locking"
(http://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html) he points out
that the following code is broken (in a distributed system):
// THIS CODE IS BROKEN
function writeData(filename, data) {
var lock = lockService.acquireLock(filename);
if (!lock) {
throw 'Failed to acquire lock';
}
// GARBAGE COLLECTION
try {
var file = storage.readFile(filename);
var updated = updateContents(file, data);
storage.writeFile(filename, updated);
} finally {
lock.release();
}
}
It is broken under the following circumstances:
time 0 - lease acquired by process 1
time 1 - huge garbage collection pause in process 1 at "// GARBAGE COLLECTION"
time 2 - lease expires
time 3 - lease acquired by process 2
time 4 - writeData executed in process 2
time 5 - garbage collection done in process 1
time 6 - writeData loses race in process 1
In order to fix it, a monotonically increasing "fence token" must be supplied by
the locking/leasing system and the storage system must be able to use the fence token
to detect late writes.
The framework makes the current fence token available in obj[OBJ.FENCE_TOKEN] so
that developers writing Action code can implement the above advice.
:param event: a str event.
:param obj: a dict.
"""
fence_token = None
try:
# attempt to acquire the lease and execute the state transition
fence_token = acquire_lease(self.correlation_id,
self.steps,
self.retries,
primary=self.lease_primary)
# 0 indicates system error, False indicates lease acquisition failure
if fence_token == 0:
self._queue_error(
ERRORS.CACHE, 'System error acquiring primary=%s lease.' %
self.lease_primary)
self.lease_primary = not self.lease_primary
fence_token = acquire_lease(self.correlation_id,
self.steps,
self.retries,
primary=self.lease_primary)
if not fence_token:
# could not get the lease. something is going wrong
self._queue_error(ERRORS.CACHE,
'Could not acquire lease. Retrying.')
self._retry(obj)
else:
# lease acquired, execute the state transition
# NOTE: idempotency
#
# In the happy path, each state transition is associated with a
# correlation_id and a monotonically increasing step value. So,
# a typical machine executes with the following values for
# step, retry count, and fence token
#
# | correlation_id | steps | retries | fence |
# +----------------+-------+---------+-------+
# | abc123 | 0 | 0 | 1 |
# | abc123 | 1 | 0 | 2 |
# | abc123 | 2 | 0 | 3 |
# | abc123 | 3 | 0 | 4 |
#
# In the event of system or user-code error, the framework may
# retry a given step multiple times
#
# | correlation_id | steps | retries | fence |
# +----------------+-------+---------+-------+
# | abc123 | 0 | 0 | 1 |
# | abc123 | 1 | 0 | 2 |
# | abc123 | 1 | 1 | 3 | # retry
# | abc123 | 1 | 2 | 4 | # retry
# | abc123 | 2 | 0 | 5 |
# | abc123 | 3 | 0 | 6 |
#
# so user-code should ensure it is idempotent (details below).
#
# The most interesting case arises when the AWS Lambda function
# is re-executed with a duplicate message, poentially out-of-order.
# This can happen for any number of reasons, since AWS Lambda ensure
# at-least-once delivery of messages. In this case, one may see messages
# like the following
#
# | correlation_id | steps | retries | fence |
# +----------------+-------+---------+-------+
# | abc123 | 0 | 0 | 1 |
# | abc123 | 1 | 0 | 2 |
# | abc123 | 1 | 2 | 3 | # out-of-order retry
# | abc123 | 1 | 1 | 4 | # out-of-order retry
# | abc123 | 1 | 2 | 5 | # duplicate message
# | abc123 | 2 | 0 | 6 |
# | abc123 | 3 | 0 | 7 |
#
# User code can use (correlation_id, steps) as an idempotency token
# for any resource unique to a specific state machine, and
# (correlation_id, steps, fence) as an idempotency token for any
# global resource.
#
# In the latter case, the global storage/resource system needs to
# understand fence tokens.
# make the fence token available
if isinstance(fence_token, (int, long)):
obj[OBJ.FENCE_TOKEN] = fence_token
self._dispatch_and_retry(event, obj)
finally:
released = release_lease(self.correlation_id,
self.steps,
self.retries,
fence_token,
primary=self.lease_primary)
if not released:
self._queue_error(ERRORS.CACHE, 'Could not release lease.')
