def convergence_remove_server_from_group(log, transaction_id, server_id,
replace, purge, group, state):
"""
Remove a specific server from the group, optionally decrementing the
desired capacity.
The server may just be scheduled for deletion, or it may be evicted from
the group by removing otter-specific metdata from the server.
:param log: A bound logger
:param bytes trans_id: The transaction id for this operation.
:param bytes server_id: The id of the server to be removed.
:param bool replace: Should the server be replaced?
:param bool purge: Should the server be deleted from Nova?
:param group: The scaling group to remove a server from.
:type group: :class:`~otter.models.interface.IScalingGroup`
:param state: The current state of the group.
:type state: :class:`~otter.models.interface.GroupState`
:return: The updated state.
:rtype: Effect of :class:`~otter.models.interface.GroupState`
:raise: :class:`CannotDeleteServerBelowMinError` if the server cannot
be deleted without replacement, and :class:`ServerNotFoundError` if
there is no such server to be deleted.
"""
effects = [_is_server_in_group(group, server_id)]
if not replace:
effects.append(_can_scale_down(group, server_id))
# the (possibly) two checks can happen in parallel, but we want
# ServerNotFoundError to take precedence over
# CannotDeleteServerBelowMinError
both_checks = yield parallel_all_errors(effects)
for is_error, result in both_checks:
if is_error:
reraise(*result)
# Remove the server
if purge:
eff = set_nova_metadata_item(server_id, *DRAINING_METADATA)
else:
eff = Effect(
EvictServerFromScalingGroup(log=log,
transaction_id=transaction_id,
scaling_group=group,
server_id=server_id))
yield Effect(
TenantScope(
retry_effect(eff, retry_times(3), exponential_backoff_interval(2)),
group.tenant_id))
if not replace:
yield do_return(assoc_obj(state, desired=state.desired - 1))
else:
yield do_return(state)
def acquire_eff(self, blocking, timeout):
assert (self._behavior.acquired is LockBehavior.NOT_STARTED
or (not self._behavior.acquired))
assert (blocking, timeout) == self._behavior.acquire_call[:2]
ret = self._behavior.acquire_call[-1]
if isinstance(ret, Exception):
self._behavior.acquired = False
return Effect(Error(ret))
else:
self._behavior.acquired = ret
return Effect(Constant(ret))
def test_fold_effect_str():
"""str()ing a FoldError returns useful traceback/exception info."""
effs = [Effect("a"), Effect(Error(ZeroDivisionError("foo"))), Effect("c")]
dispatcher = [("a", lambda i: "Ei")]
eff = fold_effect(operator.add, "Nil", effs)
with raises(FoldError) as excinfo:
perform_sequence(dispatcher, eff)
assert str(excinfo.value).startswith(
"<FoldError after accumulating 'NilEi'> Original traceback follows:\n")
assert str(excinfo.value).endswith("ZeroDivisionError: foo")
def concretize_service_request(authenticator, log, service_configs, throttler,
tenant_id, service_request):
"""
Translate a high-level :obj:`ServiceRequest` into a low-level :obj:`Effect`
of :obj:`pure_http.Request`. This doesn't directly conform to the Intent
performer interface, but it's intended to be used by a performer.
:param ICachingAuthenticator authenticator: the caching authenticator
:param BoundLog log: info about requests will be logged to this.
:param dict service_configs: As returned by
:func:`otter.constants.get_service_configs`.
:param callable throttler: A function of ServiceType, HTTP method ->
Deferred bracketer or None, used to throttle requests. See
:obj:`_Throttle`.
:param tenant_id: tenant ID.
"""
auth_eff = Effect(Authenticate(authenticator, tenant_id, log))
invalidate_eff = Effect(InvalidateToken(authenticator, tenant_id))
if service_request.log is not None:
log = service_request.log
service_config = service_configs[service_request.service_type]
region = service_config['region']
service_name = service_config['name']
def got_auth((token, catalog)):
request_ = add_headers(otter_headers(token), request)
request_ = add_effect_on_response(invalidate_eff,
service_request.reauth_codes,
request_)
request_ = add_json_request_data(request_)
if 'url' in service_config:
request_ = add_bind_root(service_config['url'], request_)
else:
request_ = add_bind_service(catalog, service_name, region, log,
request_)
request_ = add_error_handling(service_request.success_pred, request_)
if service_request.json_response:
request_ = add_json_response(request_)
return request_(service_request.method,
service_request.url,
headers=service_request.headers,
data=service_request.data,
params=service_request.params,
log=log)
eff = auth_eff.on(got_auth)
bracket = throttler(service_request.service_type,
service_request.method.lower(), tenant_id)
if bracket is not None:
return Effect(_Throttle(bracket=bracket, effect=eff))
else:
return eff
def custom_get_or_create(project: int,
timestamp: int,
authority=None) -> SQLQueryGen:
project_f = yield Effect(Project.get(id=project))
if len(project_f()) == 0: # type: ignore
if authority is None:
raise exs.InvalidInputError
yield Effect(
Project.create(id=project,
authority=authority,
timestamp=timestamp))
def should_retry(e):
if time() >= end_time:
return Effect(Constant(False))
else:
retry_delay = should_retry.wait_secs.total_seconds()
effect = Effect(Delay(retry_delay)).on(
success=lambda x: Effect(Constant(True)))
if backoff:
should_retry.wait_secs *= 2
return effect
def test_sequence():
"""Collects each Effectful result into a list."""
effs = [Effect('a'), Effect('b'), Effect('c')]
dispatcher = [
('a', lambda i: 'Ei'),
('b', lambda i: 'Bee'),
('c', lambda i: 'Cee'),
]
eff = sequence(effs)
result = perform_sequence(dispatcher, eff)
assert result == ['Ei', 'Bee', 'Cee']
def test_sequence():
"""Collects each Effectful result into a list."""
effs = [Effect("a"), Effect("b"), Effect("c")]
dispatcher = [
("a", lambda i: "Ei"),
("b", lambda i: "Bee"),
("c", lambda i: "Cee"),
]
eff = sequence(effs)
result = perform_sequence(dispatcher, eff)
assert result == ["Ei", "Bee", "Cee"]
def test_stuff():
dispatcher = ComposedDispatcher(dispatchers=[
TypeDispatcher({Seq: perform_seq}),
base_dispatcher,
])
intent = Seq(
effects=[Effect(ConstantIntent(result=result)) for result in range(5)])
result = sync_perform(dispatcher, Effect(intent))
assert result == list(range(5))
def test_boundfields(self):
"""
When an effect is wrapped `BoundFields` then any logging effect
inside is performed with fields setup in `BoundFields`
"""
f = object()
eff = Effect(Constant("foo")).on(lambda _: err(f, "yo", a='b')).on(
lambda _: msg("foo", m='d')).on(lambda _: Effect(Constant("goo")))
eff = with_log(eff, bf='new')
self.assertEqual(sync_perform(self.disp, eff), "goo")
self.log.msg.assert_called_once_with("foo", f1='v', bf='new', m='d')
self.log.err.assert_called_once_with(f, "yo", f1='v', bf='new', a='b')
def test_perform_throttle(self):
"""
The bracket given to :obj:`_Throttle` is used to call the nested
performer.
"""
def bracket(f, *args, **kwargs):
return f(*args, **kwargs).addCallback(lambda r: ('bracketed', r))
throttle = _Throttle(bracket=bracket, effect=Effect(Constant('foo')))
dispatcher = ComposedDispatcher(
[TypeDispatcher({_Throttle: _perform_throttle}), base_dispatcher])
result = sync_perform(dispatcher, Effect(throttle))
self.assertEqual(result, ('bracketed', 'foo'))
def test_uses_step_request(self):
"""Steps are converted to requests."""
steps = [
TestStep(Effect(Constant((StepResult.SUCCESS, 'foo')))),
TestStep(Effect(Error(RuntimeError('uh oh'))))
]
effect = steps_to_effect(steps)
self.assertIs(type(effect.intent), ParallelEffects)
expected_exc_info = matches(MatchesException(RuntimeError('uh oh')))
self.assertEqual(
sync_perform(test_dispatcher(), effect),
[(StepResult.SUCCESS, 'foo'),
(StepResult.RETRY, [ErrorReason.Exception(expected_exc_info)])])
def auth(cls, member_id: int, password: str, timestamp: int) -> Any:
fields = ["password", "is_leader", "last_active", "is_active"]
member_f = yield Effect(Member.get(_fields=fields, id=member_id))
member = MemberData(*member_f()[0])
is_frozen = yield from cls.is_frozen(member)
if not verify_password(member.password, password):
raise exs.IncorrectCredentials
if is_frozen:
raise exs.UserIsFrozenError
if is_frozen != member.is_active:
yield from cls.set_is_active(member_id, not is_frozen)
yield Effect(cls.set_last_active(member_id, timestamp))
return member
def test_can_have_a_different_should_retry_function(self):
"""
The ``should_retry`` function does not have to be a
:obj:`ShouldDelayAndRetry`.
"""
expected = Retry(effect=Effect(1), should_retry=ANY)
actual = Retry(effect=Effect(1), should_retry=lambda _: False)
seq = [
retry_sequence(expected, [lambda _: raise_(Exception())])
]
self.assertRaises(Exception,
perform_sequence, seq, Effect(actual))
def publish_installer_images_effects(options):
# Create configuration directory
configuration_path = yield Effect(intent=PackerConfigure(
build_region=options["build_region"],
publish_regions=options["regions"],
template=options["template"],
source_ami_map=options["source-ami-map"],
))
# Build the Docker images
ami_map = yield Effect(
intent=PackerBuild(configuration_path=configuration_path, ))
yield Effect(intent=StandardOut(
content=json.dumps(thaw(ami_map), encoding="utf-8") + b"\n"))
def test_perform_srvreq_nested(self):
"""
Concretizing of :obj:`ServiceRequest` effects happens even when they
are not directly passed as the TenantScope's toplevel Effect, but also
when they are returned from callbacks down the line.
"""
ereq = service_request(ServiceType.CLOUD_SERVERS, 'GET', 'servers')
eff = Effect(Constant("foo")).on(lambda r: ereq)
tscope = TenantScope(eff, 1)
self.assertEqual(
sync_perform(self.dispatcher, Effect(tscope)),
('concretized', self.authenticator, self.log, self.service_configs,
self.throttler, 1, ereq.intent))
def test_sequence():
"""Collects each Effectful result into a list."""
effs = [Effect('a'), Effect('b'), Effect('c')]
dispatcher = SequenceDispatcher([
('a', lambda i: 'Ei'),
('b', lambda i: 'Bee'),
('c', lambda i: 'Cee'),
])
eff = sequence(effs)
with dispatcher.consume():
result = sync_perform(_base_and(dispatcher), eff)
assert result == ['Ei', 'Bee', 'Cee']
def update_repo(package_directory, target_bucket, target_key, source_repo,
packages, flocker_version, distribution):
"""
Update ``target_bucket`` yum repository with ``packages`` from
``source_repo`` repository.
:param FilePath package_directory: Temporary directory to download
repository to.
:param bytes target_bucket: S3 bucket to upload repository to.
:param bytes target_key: Path within S3 bucket to upload repository to.
:param bytes source_repo: Repository on the build server to get packages
from.
:param list packages: List of bytes, each specifying the name of a package
to upload to the repository.
:param bytes flocker_version: The version of flocker to upload packages
for.
:param Distribution distribution: The distribution to upload packages for.
"""
package_directory.createDirectory()
package_type = distribution.package_type()
yield Effect(
DownloadS3KeyRecursively(source_bucket=target_bucket,
source_prefix=target_key,
target_path=package_directory,
filter_extensions=('.' +
package_type.value, )))
downloaded_packages = yield Effect(
DownloadPackagesFromRepository(
source_repo=source_repo,
target_path=package_directory,
packages=packages,
flocker_version=flocker_version,
distribution=distribution,
))
new_metadata = yield Effect(
CreateRepo(
repository_path=package_directory,
distribution=distribution,
))
yield Effect(
UploadToS3Recursively(
source_path=package_directory,
target_bucket=target_bucket,
target_key=target_key,
files=downloaded_packages | new_metadata,
))
async def test_parallel(self, dispatcher):
"""
'parallel' results in a list of results of the given effects, in the
same order that they were passed to parallel.
"""
d = await asyncio_perform(
dispatcher,
parallel([
Effect(Constant('a')),
Effect(
Delay(0.01)).on(success=lambda _: Effect(Constant('...'))),
Effect(Constant('b'))
]))
assert d == ['a', '...', 'b']
def full_intents():
return legacy_intents() + [
CreateOrSet(path='foo', content='bar'),
GetScalingGroupInfo(tenant_id='foo', group_id='bar'),
EvictServerFromScalingGroup(log='log',
transaction_id='transaction_id',
scaling_group='scaling_group',
server_id='server_id'),
Log('msg', {}),
LogErr('f', 'msg', {}),
BoundFields(Effect(None), {}),
MsgWithTime('msg', Effect(None)),
CQLQueryExecute(query='q', params={}, consistency_level=7)
]
def release_eff(self):
"""
Effect implementation of ``release``.
:return: ``Effect`` of ``None``
"""
def reset_node(_):
self._node = None
if self._node is not None:
return Effect(DeleteNode(path=self._node, version=-1)).on(
success=reset_node, error=catch(NoNodeError, reset_node))
else:
return Effect(Constant(None))
def perform_get_children_with_stats(kz_client, dispatcher, intent):
"""
Perform :obj:`GetChildrenWithStats`. Must be partialed with ``kz_client``.
:param kz_client: txKazoo client
:param dispatcher: dispatcher, supplied by perform
:param GetChildrenWithStats intent: the intent
"""
path = intent.path
children = yield Effect(GetChildren(path))
stats = yield parallel(Effect(GetStat(path + '/' + p)) for p in children)
yield do_return([
c_and_s for c_and_s in zip(children, stats) if c_and_s[1] is not None
])
def test_seq():
dispatcher = TypeDispatcher({
Seq: perform_seq,
Remote: perform_remote,
})
intent = Seq(effects=[
Effect(Run(command=['echo', '-n', str(result)])) for result in range(5)
])
results = sync_perform(
dispatcher, Effect(Remote(host="localhost", effect=Effect(intent))))
assert [result.output for result in results] == map(str, range(5))
def test_stuff():
dispatcher = TypeDispatcher({
Run: perform_run_locally,
Remote: perform_remote,
})
intent = Remote(host='localhost',
effect=Effect(
Run(command=['sh', '-c', 'echo -n $SSH_CONNECTION'])))
result = sync_perform(dispatcher, Effect(intent))
assert result.return_code == 0
assert result.intent == intent.effect.intent
assert result.output != ""
def custom_get_or_create(
cls,
member: int,
password: str,
timestamp: int,
) -> SQLQueryGen:
user_f = yield Effect(Member.get(id=member))
if len(user_f()) == 0: # type: ignore
yield Effect(
Member.create(id=member,
password=password,
last_active=timestamp))
else:
yield from Member.auth(member, password, timestamp)
def _acquire_loop(self, blocking, timeout):
acquired = yield self.is_acquired_eff()
if acquired or not blocking:
yield do_return(acquired)
start = yield Effect(Func(time.time))
while True:
yield Effect(Delay(self._interval))
if (yield self.is_acquired_eff()):
yield do_return(True)
if timeout is not None:
now = yield Effect(Func(time.time))
if now - start > timeout:
raise LockTimeout(
"Failed to acquire lock on {} in {} seconds".format(
self.path, now - start))
def test_fold_effect():
"""
:func:`fold_effect` folds the given function over the results of the
effects.
"""
effs = [Effect('a'), Effect('b'), Effect('c')]
dispatcher = [
('a', lambda i: 'Ei'),
('b', lambda i: 'Bee'),
('c', lambda i: 'Cee'),
]
eff = fold_effect(operator.add, 'Nil', effs)
result = perform_sequence(dispatcher, eff)
assert result == 'NilEiBeeCee'
def test_retry_sequence_fails_if_mismatch_sequence(self):
"""
Fail if the wrong number of performers are given.
"""
r = Retry(
effect=Effect(1),
should_retry=ShouldDelayAndRetry(
can_retry=retry_times(5),
next_interval=repeating_interval(10)))
seq = [
retry_sequence(r, [lambda _: raise_(Exception()),
lambda _: raise_(Exception())])
]
self.assertRaises(AssertionError,
perform_sequence, seq, Effect(r))
def test_nested_boundfields(self):
"""
BoundFields effects can be nested and the log effects internally
will expand with all bound fields
"""
eff = Effect(Constant("foo")).on(lambda _: msg("foo", m='d')).on(
lambda _: Effect(Constant("goo")))
e = Effect(Constant("abc")).on(lambda _: with_log(eff, i='a')).on(
lambda _: Effect(Constant("def")))
self.assertEqual(sync_perform(self.disp, with_log(e, o='f')), "def")
self.log.msg.assert_called_once_with('foo',
i='a',
f1='v',
m='d',
o='f')
def test_fold_effect():
"""
:func:`fold_effect` folds the given function over the results of the
effects.
"""
effs = [Effect("a"), Effect("b"), Effect("c")]
dispatcher = [
("a", lambda i: "Ei"),
("b", lambda i: "Bee"),
("c", lambda i: "Cee"),
]
eff = fold_effect(operator.add, "Nil", effs)
result = perform_sequence(dispatcher, eff)
assert result == "NilEiBeeCee"
