def test_no_exponential_backoff(self):
"""
If ``False`` is passed for the ``backoff`` parameter, the effect is
always retried with the same delay.
"""
divisors = [0, 0, 0, 1]
def tester():
x = divisors.pop(0)
return 1 / x
seq = [
(Delay(5), lambda ignore: None),
(Delay(5), lambda ignore: None),
(Delay(5), lambda ignore: None),
]
retrier = retry_effect_with_timeout(
Effect(Func(tester)),
timeout=1,
retry_wait=timedelta(seconds=5),
backoff=False,
)
result = perform_sequence(seq, retrier)
self.assertEqual(result, 1)
def test_acquire_timeout(self):
"""
acquire_eff creates child node and keeps checking if it is smallest and
eventually gives up by raising `LockTimeout`. It deletes child node
before returning.
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0)), (Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0.12)), (Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0.4)),
(DeleteNode(path="/testlock/prefix0000000001",
version=-1), noop)]
self.assertRaises(LockTimeout, perform_sequence, seq,
self.lock.acquire_eff(True, 0.3))
def test_timeout(self):
"""
If the timeout expires, the retry effect fails with the exception from
the final time the wrapped effect is performed.
"""
expected_intents = [
(Delay(1), lambda ignore: None),
(Delay(2), lambda ignore: None),
]
exceptions = [
Exception("Wrong (1)"),
Exception("Wrong (2)"),
CustomException(),
]
def tester():
raise exceptions.pop(0)
retrier = retry_effect_with_timeout(
Effect(Func(tester)),
timeout=3,
time=self.get_time([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]),
)
self.assertRaises(CustomException, perform_sequence, expected_intents,
retrier)
async def test_delay(self, event_loop):
"""
Delay intents will cause time to pass with reactor.callLater, and
result in None.
"""
now = event_loop.time()
def tick():
nonlocal now
now += 1
def time():
return now
with patch.object(event_loop, 'time', new_callable=lambda: time):
called = []
eff = Effect(Delay(4)).on(called.append)
fut = asyncio_perform(make_asyncio_dispatcher(), eff)
for _ in range(5):
assert not fut.done()
assert not called
event_loop.call_soon(tick)
await asyncio.sleep(1)
assert fut.done()
assert called == [None]
async def test_parallel_with_error(self, dispatcher):
"""
'parallel' results in a list of results of the given effects, in the
same order that they were passed to parallel.
"""
@do
def fail():
yield Effect(Delay(0.01))
raise RuntimeError('My error')
future = asyncio_perform(
dispatcher, parallel([
Effect(Delay(1)),
Effect(Delay(1)),
fail(),
]))
with pytest.raises(FirstError):
await future
def simple_intents():
return [
Authenticate(None, None, None),
InvalidateToken(None, None),
Request(method='GET', url='http://example.com/'),
Retry(effect=Effect(Constant(None)), should_retry=lambda e: False),
Delay(0),
Constant(None),
ReadReference(ref=Reference(None)),
]
def test_should_retry(self):
"""
When called and can_retry returns True, a Delay based on next_interval
is executed and the ultimate result is True.
"""
sdar = ShouldDelayAndRetry(can_retry=lambda f: True,
next_interval=lambda f: 1.5)
eff = sdar(get_exc_info())
next_eff = _perform_func_intent(eff)
self.assertEqual(next_eff.intent, Delay(delay=1.5))
self.assertEqual(resolve_effect(next_eff, None), True)
def test_acquire_blocking_success(self):
"""
acquire_eff creates child, realizes its not the smallest. Tries again
every 0.01 seconds until it succeeds
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0)), (Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0.2)), (Delay(0.1), noop),
(GetChildren("/testlock"), const(["prefix0000000001"]))]
self.assertTrue(perform_sequence(seq, self.lock.acquire_eff(True, 1)))
def should_retry(e):
if time() >= end_time:
return Effect(Constant(False))
else:
retry_delay = should_retry.wait_secs.total_seconds()
effect = Effect(Delay(retry_delay)).on(
success=lambda x: Effect(Constant(True)))
if backoff:
should_retry.wait_secs *= 2
return effect
def test_exponential_backoff(self):
"""
Retry the effect multiple times with exponential backoff between
retries.
"""
divisors = [0, 0, 0, 1]
def tester():
x = divisors.pop(0)
return 1 / x
seq = [
(Delay(1), lambda ignore: None),
(Delay(2), lambda ignore: None),
(Delay(4), lambda ignore: None),
]
retrier = retry_effect_with_timeout(
Effect(Func(tester)), timeout=10, time=self.get_time(),
)
result = perform_sequence(seq, retrier)
self.assertEqual(result, 1)
def test_acquire_blocking_no_timeout(self):
"""
When acquire_eff is called without timeout, it creates child, realizes
its not the smallest, tries again every 0.1 seconds without checking
time and succeeds if its the smallest node
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0)), (Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Delay(0.1), noop),
(GetChildren("/testlock"), const(["prefix0000000001"]))]
self.assertTrue(
perform_sequence(seq, self.lock.acquire_eff(True, None)))
async def test_parallel(self, dispatcher):
"""
'parallel' results in a list of results of the given effects, in the
same order that they were passed to parallel.
"""
d = await asyncio_perform(
dispatcher,
parallel([
Effect(Constant('a')),
Effect(
Delay(0.01)).on(success=lambda _: Effect(Constant('...'))),
Effect(Constant('b'))
]))
assert d == ['a', '...', 'b']
def _acquire_loop(self, blocking, timeout):
acquired = yield self.is_acquired_eff()
if acquired or not blocking:
yield do_return(acquired)
start = yield Effect(Func(time.time))
while True:
yield Effect(Delay(self._interval))
if (yield self.is_acquired_eff()):
yield do_return(True)
if timeout is not None:
now = yield Effect(Func(time.time))
if now - start > timeout:
raise LockTimeout(
"Failed to acquire lock on {} in {} seconds".format(
self.path, now - start))
def test_one_retry(self):
"""
Retry the effect if it fails once.
"""
divisors = [0, 1]
def tester():
x = divisors.pop(0)
return 1 / x
seq = [
(Delay(1), lambda ignore: None),
]
retrier = retry_effect_with_timeout(Effect(Func(tester)), 10,
time=self.get_time())
result = perform_sequence(seq, retrier)
self.assertEqual(result, 1 / 1)
def test_timeout_measured_from_perform(self):
"""
The timeout is measured from the time the effect is performed (not from
the time it is created).
"""
timeout = 3.0
time = self.get_time([0.0] + list(timeout + i for i in range(10)))
exceptions = [Exception("One problem")]
result = object()
def tester():
if exceptions:
raise exceptions.pop()
return result
retrier = retry_effect_with_timeout(
Effect(Func(tester)),
timeout=3,
time=time,
)
# The retry effect has been created. Advance time a little bit before
# performing it.
time()
expected_intents = [
# The first call raises an exception and should be retried even
# though (as a side-effect of the `time` call above) the timeout,
# as measured from when `retry_effect_with_timeout` was called, has
# already elapsed.
#
# There's no second intent because the second call to the function
# succeeds.
(Delay(1), lambda ignore: None),
]
self.assertThat(
perform_sequence(expected_intents, retrier), Is(result)
)
def should_retry(exc_info):
# This is the wrong time to compute end_time. It's a lot simpler to do
# this than to hook into the effect being wrapped and record the time
# it starts to run. Perhaps implementing that would be a nice thing to
# do later.
#
# Anyway, make note of when we want to be finished if we haven't yet
# done so.
if State.end_time is None:
State.end_time = time() + timeout
if time() >= State.end_time:
return Effect(Constant(False))
else:
retry_delay = State.wait_time.total_seconds()
effect = Effect(Delay(retry_delay)).on(
success=lambda x: Effect(Constant(True)))
if backoff:
State.wait_time *= 2
return effect
def doit():
if self.can_retry(failure):
interval = self.next_interval(failure)
return Effect(Delay(interval)).on(lambda r: True)
else:
return False
def fail():
yield Effect(Delay(0.01))
raise RuntimeError('My error')
