def test_nested_msg(self):
"""
message is logged when nested inside other effects
"""
eff = Effect(Constant("foo")).on(lambda _: msg("yo", a='b')).on(
lambda _: Effect(Constant("goo")))
self.assertEqual(sync_perform(self.disp, eff), "goo")
self.log.msg.assert_called_once_with("yo", f1='v', a='b')
def test_nested_err(self):
"""
error is logged when nested inside other effects
"""
f = object()
eff = Effect(Constant("foo")).on(lambda _: err(f, "yo", a='b')).on(
lambda _: Effect(Constant("goo")))
self.assertEqual(sync_perform(self.disp, eff), "goo")
self.log.err.assert_called_once_with(f, "yo", f1='v', a='b')
def simple_intents():
return [
Authenticate(None, None, None),
InvalidateToken(None, None),
Request(method='GET', url='http://example.com/'),
Retry(effect=Effect(Constant(None)), should_retry=lambda e: False),
Delay(0),
Constant(None),
ReadReference(ref=Reference(None)),
]
def test_boundfields(self):
"""
When an effect is wrapped `BoundFields` then any logging effect
inside is performed with fields setup in `BoundFields`
"""
f = object()
eff = Effect(Constant("foo")).on(lambda _: err(f, "yo", a='b')).on(
lambda _: msg("foo", m='d')).on(lambda _: Effect(Constant("goo")))
eff = with_log(eff, bf='new')
self.assertEqual(sync_perform(self.disp, eff), "goo")
self.log.msg.assert_called_once_with("foo", f1='v', bf='new', m='d')
self.log.err.assert_called_once_with(f, "yo", f1='v', bf='new', a='b')
def should_retry(e):
if time() >= end_time:
return Effect(Constant(False))
else:
retry_delay = should_retry.wait_secs.total_seconds()
effect = Effect(Delay(retry_delay)).on(
success=lambda x: Effect(Constant(True)))
if backoff:
should_retry.wait_secs *= 2
return effect
async def test_parallel(self, dispatcher):
"""
'parallel' results in a list of results of the given effects, in the
same order that they were passed to parallel.
"""
d = await asyncio_perform(
dispatcher,
parallel([
Effect(Constant('a')),
Effect(
Delay(0.01)).on(success=lambda _: Effect(Constant('...'))),
Effect(Constant('b'))
]))
assert d == ['a', '...', 'b']
def test_nested_boundfields(self):
"""
BoundFields effects can be nested and the log effects internally
will expand with all bound fields
"""
eff = Effect(Constant("foo")).on(lambda _: msg("foo", m='d')).on(
lambda _: Effect(Constant("goo")))
e = Effect(Constant("abc")).on(lambda _: with_log(eff, i='a')).on(
lambda _: Effect(Constant("def")))
self.assertEqual(sync_perform(self.disp, with_log(e, o='f')), "def")
self.log.msg.assert_called_once_with('foo',
i='a',
f1='v',
m='d',
o='f')
def prefetch_github(owner, repo, prefetch=True, rev=None):
if isinstance(rev, str) and is_sha1_hash(rev):
actual_rev = rev
else:
list_remote = yield Effect(GetListRemote(owner=owner, repo=repo))
if rev is None:
actual_rev = list_remote.branch(list_remote.symref("HEAD"))
else:
actual_rev = (list_remote.full_ref_name(rev)
or list_remote.branch(rev)
or list_remote.tag(f"{rev}^{{}}")
or list_remote.tag(rev))
if actual_rev is None:
yield Effect(
AbortWithErrorMessage(
message=revision_not_found_errormessage(
owner=owner, repo=repo, revision=rev)))
return
calculated_hash = yield Effect(
CalculateSha256Sum(owner=owner, repo=repo, revision=actual_rev))
if not calculated_hash:
raise click.ClickException(
("Internal Error: Calculate hash value for sources "
"in github repo {owner}/{repo}.").format(owner=owner, repo=repo))
if prefetch:
yield Effect(
TryPrefetch(owner=owner,
repo=repo,
sha256=calculated_hash,
rev=actual_rev))
return Effect(Constant({"rev": actual_rev, "sha256": calculated_hash}))
def test_acquire_timeout(self):
"""
acquire_eff creates child node and keeps checking if it is smallest and
eventually gives up by raising `LockTimeout`. It deletes child node
before returning.
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0)), (Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0.12)), (Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0.4)),
(DeleteNode(path="/testlock/prefix0000000001",
version=-1), noop)]
self.assertRaises(LockTimeout, perform_sequence, seq,
self.lock.acquire_eff(True, 0.3))
def fold_effect(f, initial, effects):
"""
Fold over the results of effects, left-to-right.
This is like :func:`functools.reduce`, but instead of acting on plain
values, it acts on the results of effects.
The function ``f`` will be called with the accumulator (starting with
``initial``) and a result of an effect repeatedly for each effect. The
result of the previous call will be passed as the accumulator to the next
call.
For example, the following code evaluates to an Effect of 6::
fold_effect(operator.add, 0, [Effect(Constant(1)),
Effect(Constant(2)),
Effect(Constant(3))])
If no elements were in the list, Effect would result in 0.
:param callable f: function of ``(accumulator, element) -> accumulator``
:param initial: The value to be passed as the accumulator to the first
invocation of ``f``.
:param effects: sequence of Effects.
"""
def failed(acc, e):
raise FoldError(acc, e)
def folder(acc, element):
return acc.on(
lambda r: element.on(lambda r2: f(r, r2), error=lambda e: failed(r, e))
)
return reduce(folder, effects, Effect(Constant(initial)))
def test_perform_retry(self):
"""
When the specified effect is successful, its result is propagated.
"""
retry = Retry(effect=Effect(Constant('foo')),
should_retry=lambda e: 1 / 0)
result = sync_perform(self.dispatcher, Effect(retry))
self.assertEqual(result, 'foo')
def test_tenant_scope(self):
"""The :obj:`TenantScope` performer passes through to child effects."""
# This is not testing much, but at least that it calls
# perform_tenant_scope in a vaguely working manner. There are
# more specific TenantScope performer tests in otter.test.test_http
dispatcher = get_full_dispatcher(*([None] * 8))
scope = TenantScope(Effect(Constant('foo')), 1)
eff = Effect(scope)
self.assertEqual(sync_perform(dispatcher, eff), 'foo')
def test_immediate_success(self):
"""
If the wrapped effect succeeds at first, no delay or retry is done and
the retry effect's result is the wrapped effect's result.
"""
effect = Effect(Constant(1000))
retrier = retry_effect_with_timeout(effect, 10, time=self.get_time())
result = perform_sequence([], retrier)
self.assertEqual(result, 1000)
def acquire_eff(self, blocking, timeout):
assert (self._behavior.acquired is LockBehavior.NOT_STARTED
or (not self._behavior.acquired))
assert (blocking, timeout) == self._behavior.acquire_call[:2]
ret = self._behavior.acquire_call[-1]
if isinstance(ret, Exception):
self._behavior.acquired = False
return Effect(Error(ret))
else:
self._behavior.acquired = ret
return Effect(Constant(ret))
def test_perform_throttle(self):
"""
The bracket given to :obj:`_Throttle` is used to call the nested
performer.
"""
def bracket(f, *args, **kwargs):
return f(*args, **kwargs).addCallback(lambda r: ('bracketed', r))
throttle = _Throttle(bracket=bracket, effect=Effect(Constant('foo')))
dispatcher = ComposedDispatcher(
[TypeDispatcher({_Throttle: _perform_throttle}), base_dispatcher])
result = sync_perform(dispatcher, Effect(throttle))
self.assertEqual(result, ('bracketed', 'foo'))
def should_retry(exc_info):
# This is the wrong time to compute end_time. It's a lot simpler to do
# this than to hook into the effect being wrapped and record the time
# it starts to run. Perhaps implementing that would be a nice thing to
# do later.
#
# Anyway, make note of when we want to be finished if we haven't yet
# done so.
if State.end_time is None:
State.end_time = time() + timeout
if time() >= State.end_time:
return Effect(Constant(False))
else:
retry_delay = State.wait_time.total_seconds()
effect = Effect(Delay(retry_delay)).on(
success=lambda x: Effect(Constant(True)))
if backoff:
State.wait_time *= 2
return effect
def test_uses_step_request(self):
"""Steps are converted to requests."""
steps = [
TestStep(Effect(Constant((StepResult.SUCCESS, 'foo')))),
TestStep(Effect(Error(RuntimeError('uh oh'))))
]
effect = steps_to_effect(steps)
self.assertIs(type(effect.intent), ParallelEffects)
expected_exc_info = matches(MatchesException(RuntimeError('uh oh')))
self.assertEqual(
sync_perform(test_dispatcher(), effect),
[(StepResult.SUCCESS, 'foo'),
(StepResult.RETRY, [ErrorReason.Exception(expected_exc_info)])])
def test_perform_srvreq_nested(self):
"""
Concretizing of :obj:`ServiceRequest` effects happens even when they
are not directly passed as the TenantScope's toplevel Effect, but also
when they are returned from callbacks down the line.
"""
ereq = service_request(ServiceType.CLOUD_SERVERS, 'GET', 'servers')
eff = Effect(Constant("foo")).on(lambda r: ereq)
tscope = TenantScope(eff, 1)
self.assertEqual(
sync_perform(self.dispatcher, Effect(tscope)),
('concretized', self.authenticator, self.log, self.service_configs,
self.throttler, 1, ereq.intent))
def release_eff(self):
"""
Effect implementation of ``release``.
:return: ``Effect`` of ``None``
"""
def reset_node(_):
self._node = None
if self._node is not None:
return Effect(DeleteNode(path=self._node, version=-1)).on(
success=reset_node, error=catch(NoNodeError, reset_node))
else:
return Effect(Constant(None))
def test_acquire_success(self):
"""
acquire_eff creates child and gets lock as it is the smallest one
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), conste(NodeExistsError())),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000000")),
(GetChildren("/testlock"), const(["prefix0000000000"]))]
self.assertTrue(
perform_sequence(seq, self.lock.acquire_eff(False, None)))
def test_acquire_create_path_success(self):
"""
acquire_eff creates provided path if it doesn't exist
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000000")),
(GetChildren("/testlock"), const(["prefix0000000000"]))]
self.assertTrue(
perform_sequence(seq, self.lock.acquire_eff(False, None)))
def test_acquire_other_error(self):
"""
If acquire_eff internally raises any error then it tries to delete
child node before returning.
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"), conste(SessionExpiredError())),
(DeleteNode(path="/testlock/prefix0000000001",
version=-1), conste(SessionExpiredError()))]
self.assertRaises(SessionExpiredError, perform_sequence, seq,
self.lock.acquire_eff(True, 0.3))
def test_acquire_nonblocking_fails(self):
"""
acquire_eff creates child and returns False immediately after finding
its not the smallest child when blocking=False. It deletes child node
before returning.
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(DeleteNode(path="/testlock/prefix0000000001",
version=-1), noop)]
self.assertFalse(
perform_sequence(seq, self.lock.acquire_eff(False, None)))
def test_acquire_blocking_success(self):
"""
acquire_eff creates child, realizes its not the smallest. Tries again
every 0.01 seconds until it succeeds
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0)), (Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0.2)), (Delay(0.1), noop),
(GetChildren("/testlock"), const(["prefix0000000001"]))]
self.assertTrue(perform_sequence(seq, self.lock.acquire_eff(True, 1)))
def test_acquire_blocking_no_timeout(self):
"""
When acquire_eff is called without timeout, it creates child, realizes
its not the smallest, tries again every 0.1 seconds without checking
time and succeeds if its the smallest node
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0)), (Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Delay(0.1), noop),
(GetChildren("/testlock"), const(["prefix0000000001"]))]
self.assertTrue(
perform_sequence(seq, self.lock.acquire_eff(True, None)))
def Continue():
return Effect(Constant(None))
def as_effect(self):
"""Return an effect that always results in failure."""
return Effect(Constant((StepResult.FAILURE, list(self.reasons))))
def as_effect(self):
"""
Return an effect that always results in retry
"""
result = StepResult.LIMITED_RETRY if self.limited else StepResult.RETRY
return Effect(Constant((result, list(self.reasons))))
def should_retry(exc_info):
exc_type, exc_value, exc_traceback = exc_info
failure = Failure(exc_value, exc_type, exc_traceback)
return Effect(Constant(
actual_retry_intent.should_retry.can_retry(failure)))
def _game(groups, log, _print=False):
self.assertEqual(log, "log")
return [Effect(Constant(None)), Effect(Constant(['foo']))]
