def testTimeoutAndDelays(self):
"""Test retry scenario in which the RetryPolicy timeout is exceeded."""
retry_policy = retry.RetryPolicy(timeout=.6, min_delay=.05, max_delay=.2, check_result=lambda res, err: True)
retry.CallWithRetryAsync(retry_policy, self._AsyncFunc, callback=self._OnCompleted)
self.wait()
self.assertLogMatches('Retrying.*Retrying.*Retrying',
'Expected at least 3 retries in the log')
def testMaxTries(self):
"""Test retry scenario in which the RetryPolicy max_tries is exceeded."""
retry_policy = retry.RetryPolicy(max_tries=10, check_result=lambda res, err: True)
retry.CallWithRetryAsync(retry_policy, self._AsyncFunc, callback=self._OnCompleted)
self.wait()
self.assertLogMatches('Retrying.*Retrying.*Retrying.*Retrying.*Retrying.*Retrying.*Retrying.*Retrying.*Retrying',
'Expected 9 retries in the log')
def testWithStackContext2(self):
"""Ensure Retry doesn't interfere with asynchronous function that throws immediately."""
try:
with stack_context.ExceptionStackContext(self._OnError):
retry.CallWithRetryAsync(retry.RetryPolicy(), self._AsyncFuncRaisesError,
callback=self._OnCompleted)
self.assert_(False, 'Expected exception to be raised')
except:
self.wait()
def testRetryWithException2(self):
"""Retry on exceptions raised by async function after stack transfer."""
def CallAfterStackTransfer(dict, callback):
func = functools.partial(self._AsyncFuncRaisesErrorOnce, dict, callback)
self.io_loop.add_callback(func)
retry_policy = retry.RetryPolicy(max_tries=3, check_exception=lambda typ, val, tb: True)
retry.CallWithRetryAsync(retry_policy, CallAfterStackTransfer, dict(), callback=self.stop)
self.wait()
def testRetryPolicyApi(self):
"""Test RetryPolicy __init__ API."""
self.assertRaises(OverflowError, functools.partial(retry.RetryPolicy, timeout=1234123412341234))
retry.RetryPolicy(timeout=timedelta(milliseconds=500))
self.assertEqual(retry.RetryPolicy(timeout=10).timeout.total_seconds(), 10)
self.assertEqual(retry.RetryPolicy(timeout= -1.5).timeout.total_seconds(), -1.5)
retry.RetryPolicy(min_delay=timedelta(days=500))
self.assertEqual(retry.RetryPolicy(min_delay=10).min_delay.total_seconds(), 10)
self.assertEqual(retry.RetryPolicy(min_delay= -1.5).min_delay.total_seconds(), -1.5)
retry.RetryPolicy(max_delay=timedelta(hours=500))
self.assertEqual(retry.RetryPolicy(max_delay=10).max_delay.total_seconds(), 10)
self.assertEqual(retry.RetryPolicy(max_delay= -1.5).max_delay.total_seconds(), -1.5)
def testWithStackContext3(self):
"""Ensure Retry doesn't interfere with asynchronous callback that throws."""
try:
with stack_context.ExceptionStackContext(self._OnError):
retry.CallWithRetryAsync(retry.RetryPolicy(check_exception=lambda typ, val, tb: True), self._AsyncFunc,
callback=self._OnCompletedRaisesError)
self.wait()
self.assert_(False, 'Expected exception to be raised')
except Exception as e:
self.assert_('_OnCompletedRaisesError' in e.message, e)
def testWithStackContext1(self):
"""Ensure Retry preserves StackContext."""
self.__in_context = False
@contextlib.contextmanager
def _MyContext():
try:
self.__in_context = True
yield
finally:
self.__in_context = False
def _OnCompletedCheckContext(result, error):
self.assertTrue(self.__in_context)
self.stop()
with stack_context.StackContext(_MyContext):
retry_policy = retry.RetryPolicy(max_tries=2, check_result=lambda res, err: err)
retry.CallWithRetryAsync(retry_policy, self._AsyncFuncFailOnce, callback=_OnCompletedCheckContext)
self.wait()
""" __author__ = '[email protected] (Marc Berhault)' import logging import os import tempfile from tornado import gen from viewfinder.backend.base import retry from viewfinder.backend.storage import file_object_store, s3_object_store # Retry policy for uploading files to S3 (merge logs and registry). kS3UploadRetryPolicy = retry.RetryPolicy(max_tries=5, timeout=300, min_delay=1, max_delay=30, check_exception=retry.RetryPolicy.AlwaysRetryOnException) class LocalLogMerge(object): """Class used to build a single merged log file locally.""" def __init__(self, logs_store, id_list, s3_base): self._logs_store = logs_store self._s3_filename = os.path.join(s3_base, *id_list) fd, self._working_filename = tempfile.mkstemp(suffix='.' + '.'.join(id_list)) self._output = os.fdopen(fd, 'w') self._buffer = [] self._needs_separator = False @gen.engine def FetchExistingFromS3(self, callback): """If S3 already has a file for this day/instance, fetch it and write its contents to the
def testWithBarrier(self):
"""Ensure Retry doesn't interfere with barriers."""
retry_policy = retry.RetryPolicy(max_tries=2, check_result=lambda res, err: err)
with util.MonoBarrier(self._OnCompleted) as b:
retry.CallWithRetryAsync(retry_policy, self._AsyncFuncFailOnce, callback=b.Callback())
self.wait()
def CallWithRetry(): retry_policy = retry.RetryPolicy(max_tries=3, check_exception=lambda typ, val, tb: True) retry.CallWithRetryAsync(retry_policy, self._AsyncFuncRaisesErrorOnce, dict(), callback=self.stop)
class Version(object):
"""The version base class. Provides a guarantee that rank order is
correct (that careless code updating doesn't confuse the ordering of
versions).
When authoring a version migrator, do not assume that the version
is correct. There are cases where it can "get behind". So, for
example, while the version on the row = 5, the actual row data
corresponds to version 8. However, the reverse *cannot* happen;
if the version = 8, then the actual row data cannot correspond
to version 5. So if a row version shows the row is at the latest
version, then there is no need to migrate.
"""
_DELETED_MIGRATOR_COUNT = 16
"""If older migrators are deleted, update this."""
_version_classes = set()
_rank_ordering = []
_mutate_items = True
_allow_s3_queries = True
_migrate_retry_policy = retry.RetryPolicy(
max_tries=5, min_delay=1, check_exception=lambda type, value, tb: True)
"""Retry migration of an item up to 5 times before giving up."""
def __init__(self):
self.rank = len(
Version._rank_ordering) + 1 + Version._DELETED_MIGRATOR_COUNT
if Version._rank_ordering:
assert self.rank > Version._rank_ordering[-1], \
'rank is out of order (! %d > %d)' % (self.rank, Version._rank_ordering[-1])
assert self.__class__ not in Version._version_classes, \
'class %s has already been added to version set' % self.__class__.__name__
Version._rank_ordering.append(self.rank)
Version._version_classes.add(self.__class__)
@classmethod
def SetMutateItems(cls, mutate):
"""Set to affect mutations on the database. If False, the planned
modifications to each item are verbosely logged but not persisted.
"""
Version._mutate_items = mutate
@classmethod
def SetAllowS3Queries(cls, allow):
"""Allow S3 queries. If False, upgrades that involve querying S3 will
skip it, but may perform other work.
eg: CreateMD5Hashes and FillFileSizes both use S3 queries as a fallback
when the desired fields are not found the Photo.client_data.
"""
Version._allow_s3_queries = allow
@classmethod
def GetCurrentVersion(cls):
"""Returns the maximum version. New objects have item._version set
to this value.
"""
if Version._rank_ordering:
return Version._rank_ordering[-1]
else:
return 0
@classmethod
def MaybeMigrate(cls, client, original_item, versions, callback):
"""Migrates the data in one table row ('item') by advancing
'item's version via successive data migrations. If 'item' does not
have a version yet, all data migrations are applied. If item's
version is current, does nothing. Return the migrated object if
mutations are enabled, or the original object if not. Take care
if the migration changes the primary key; the caller might fetch
an object using one primary key, but get back an object with a
different migrated primary key!
"""
def _Migrate(start_rank, mutate_item):
last_rank = 0
for version in versions:
if version.rank < start_rank:
last_rank = version.rank
continue
assert version.rank > last_rank, \
'tags listed out of order (! %d > %d)' % (version.rank, last_rank)
last_rank = version.rank
item_version = mutate_item._version or 0
if item_version < version.rank:
logging.debug('upgrading item from %s to version %s' %
(type(mutate_item)._table.name,
version.__class__.__name__))
mutate_item._version = version.rank
# If Transform fails, retry several times before giving up.
transform_callback = partial(_Migrate, last_rank + 1)
retry.CallWithRetryAsync(Version._migrate_retry_policy,
version.Transform,
client,
mutate_item,
callback=transform_callback)
return
callback(mutate_item if Version._mutate_items else original_item)
_Migrate(
0,
original_item if Version._mutate_items else original_item._Clone())
def _LogUpdate(self, item):
"""Log the changes to the object."""
mods = [
'%s => %r' % (n, getattr(item, n)) for n in item.GetColNames()
if item._IsModified(n)
]
if mods:
logging.info(
'%s (%r): %s' %
(type(item)._table.name, item.GetKey(), ', '.join(mods)))
def Transform(self, client, item, callback):
"""Implement in each subclass to effect the required data migration.
'callback' should be invoked on completion with the update object.
If no async processing is required, it should be invoked directly.
"""
raise NotImplementedError()