def modify_and_trigger(dispatcher, group, logargs, modifier, *args, **kwargs):
"""
Modify group state and trigger convergence after that if the group is not
suspended. Otherwise fail with :obj:`TenantSuspendedError`.
:param IScalingGroup group: Scaling group whose state is getting modified
:param log: Bound logger
:param modifier: Callable as described in IScalingGroup.modify_state
:return: Deferred with None if modification and convergence succeeded.
Fails with :obj:`TenantSuspendedError` if group is suspended.
"""
def modifier_wrapper(_group, state, *_args, **_kwargs):
# Ideally this will not be allowed by repose middleware but
# adding check for mimic based integration tests
if state.suspended:
raise TenantSuspendedError(_group.tenant_id)
return modifier(_group, state, *_args, **_kwargs)
cannot_exec_pol_err = None
try:
yield group.modify_state(modifier_wrapper, *args, **kwargs)
except CannotExecutePolicyError as ce:
cannot_exec_pol_err = ce
if tenant_is_enabled(group.tenant_id, config_value):
eff = Effect(
BoundFields(
trigger_convergence(group.tenant_id, group.uuid), logargs))
yield perform(dispatcher, eff)
if cannot_exec_pol_err is not None:
raise cannot_exec_pol_err
def converge_scaling_group(self, request):
"""
Trigger convergence on given scaling group
"""
class ConvergeErrorGroup(Exception):
pass
def can_converge(group, state):
if state.paused:
raise GroupPausedError(group.tenant_id, group.uuid, "converge")
conv_on_error = extract_bool_arg(request, 'on_error', True)
if not conv_on_error and state.status == ScalingGroupStatus.ERROR:
raise ConvergeErrorGroup()
return state
def converge_error_group_header(f):
f.trap(ConvergeErrorGroup)
request.setHeader("x-not-converging", "true")
if tenant_is_enabled(self.tenant_id, config_value):
group = self.store.get_scaling_group(
self.log, self.tenant_id, self.group_id)
return controller.modify_and_trigger(
self.dispatcher,
group,
bound_log_kwargs(self.log),
can_converge).addErrback(converge_error_group_header)
else:
request.setResponseCode(404)
def modify_and_trigger(dispatcher, group, logargs, modifier, *args, **kwargs):
"""
Modify group state and trigger convergence after that if the group is not
suspended. Otherwise fail with :obj:`TenantSuspendedError`.
:param IScalingGroup group: Scaling group whose state is getting modified
:param log: Bound logger
:param modifier: Callable as described in IScalingGroup.modify_state
:return: Deferred with None if modification and convergence succeeded.
Fails with :obj:`TenantSuspendedError` if group is suspended.
"""
def modifier_wrapper(_group, state, *_args, **_kwargs):
# Ideally this will not be allowed by repose middleware but
# adding check for mimic based integration tests
if state.suspended:
raise TenantSuspendedError(_group.tenant_id)
return modifier(_group, state, *_args, **_kwargs)
cannot_exec_pol_err = None
try:
yield group.modify_state(modifier_wrapper, *args, **kwargs)
except CannotExecutePolicyError as ce:
cannot_exec_pol_err = ce
if tenant_is_enabled(group.tenant_id, config_value):
eff = Effect(
BoundFields(trigger_convergence(group.tenant_id, group.uuid),
logargs))
yield perform(dispatcher, eff)
if cannot_exec_pol_err is not None:
raise cannot_exec_pol_err
def converge(log,
transaction_id,
config,
scaling_group,
state,
launch_config,
policy,
config_value=config_value):
"""
Apply a policy's change to a scaling group, and attempt to make the
resulting state a reality. This does no cooldown checking.
This is done by dispatching to the appropriate orchestration backend for
the scaling group; currently only direct nova interaction is supported.
:param log: A bound log for logging
:param str transaction_id: the transaction id
:param dict config: the scaling group config
:param otter.models.interface.IScalingGroup scaling_group: the scaling
group object
:param otter.models.interface.GroupState state: the group state
:param dict launch_config: the scaling group launch config
:param dict policy: the policy configuration dictionary
:return: a ``Deferred`` that fires with the updated
:class:`otter.models.interface.GroupState` if successful. If no changes
are to be made to the group, None will synchronously be returned.
"""
if tenant_is_enabled(scaling_group.tenant_id, config_value):
# For convergence tenants, find delta based on group's desired
# capacity
delta = apply_delta(log, state.desired, state, config, policy)
if delta == 0:
# No change in servers. Return None synchronously
return None
else:
return defer.succeed(state)
# For non-convergence tenants, the value used for desired-capacity is
# the sum of active+pending, which is 0, so the delta ends up being
# the min entities due to constraint calculation.
delta = calculate_delta(log, state, config, policy)
execute_log = log.bind(server_delta=delta)
if delta == 0:
execute_log.msg("no change in servers")
return None
elif delta > 0:
execute_log.msg("executing launch configs")
deferred = execute_launch_config(execute_log, transaction_id, state,
launch_config, scaling_group, delta)
else:
# delta < 0 (scale down)
execute_log.msg("scaling down")
deferred = exec_scale_down(execute_log, transaction_id, state,
scaling_group, -delta)
deferred.addCallback(_do_convergence_audit_log, log, delta, state)
return deferred
def get_groups_to_converge(config_func):
"""
Get all tenant's all groups that needs convergence triggering
"""
eff = Effect(GetAllValidGroups())
eff = eff.on(
filter(lambda g: tenant_is_enabled(g["tenantId"], config_func)))
return eff.on(list)
def get_groups_to_converge(config_func):
"""
Get all tenant's all groups that needs convergence triggering
"""
eff = Effect(GetAllValidGroups())
eff = eff.on(
filter(lambda g: tenant_is_enabled(g["tenantId"], config_func)))
return eff.on(list)
def fetch_active_caches(group_states):
if not tenant_is_enabled(self.tenant_id, config_value):
return group_states, [None] * len(group_states)
d = gatherResults(
[get_active_cache(
self.store.reactor, self.store.connection, self.tenant_id,
state.group_id)
for state in group_states])
return d.addCallback(lambda cache: (group_states, cache))
def fetch_active_caches(group_states):
if not tenant_is_enabled(self.tenant_id, config_value):
return group_states, [None] * len(group_states)
d = gatherResults(
[get_active_cache(
self.store.reactor, self.store.connection, self.tenant_id,
state.group_id)
for state in group_states])
return d.addCallback(lambda cache: (group_states, cache))
def converge(log, transaction_id, config, scaling_group, state, launch_config,
policy, config_value=config_value):
"""
Apply a policy's change to a scaling group, and attempt to make the
resulting state a reality. This does no cooldown checking.
This is done by dispatching to the appropriate orchestration backend for
the scaling group; currently only direct nova interaction is supported.
:param log: A bound log for logging
:param str transaction_id: the transaction id
:param dict config: the scaling group config
:param otter.models.interface.IScalingGroup scaling_group: the scaling
group object
:param otter.models.interface.GroupState state: the group state
:param dict launch_config: the scaling group launch config
:param dict policy: the policy configuration dictionary
:return: a ``Deferred`` that fires with the updated
:class:`otter.models.interface.GroupState` if successful. If no changes
are to be made to the group, None will synchronously be returned.
"""
if tenant_is_enabled(scaling_group.tenant_id, config_value):
# For convergence tenants, find delta based on group's desired
# capacity
delta = apply_delta(log, state.desired, state, config, policy)
if delta == 0:
# No change in servers. Return None synchronously
return None
else:
return defer.succeed(state)
# For non-convergence tenants, the value used for desired-capacity is
# the sum of active+pending, which is 0, so the delta ends up being
# the min entities due to constraint calculation.
delta = calculate_delta(log, state, config, policy)
execute_log = log.bind(server_delta=delta)
if delta == 0:
execute_log.msg("no change in servers")
return None
elif delta > 0:
execute_log.msg("executing launch configs")
deferred = execute_launch_config(
execute_log, transaction_id, state, launch_config,
scaling_group, delta)
else:
# delta < 0 (scale down)
execute_log.msg("scaling down")
deferred = exec_scale_down(execute_log, transaction_id, state,
scaling_group, -delta)
deferred.addCallback(_do_convergence_audit_log, log, delta, state)
return deferred
def pause_scaling_group(log, transaction_id, scaling_group, dispatcher):
"""
Pauses the scaling group, causing all scaling policy executions to be
rejected until unpaused. This is an idempotent change, if it's already
paused, this does not raise an error.
:raises: :class:`NoSuchScalingGroup` if the scaling group does not exist.
:return: None
"""
if not tenant_is_enabled(scaling_group.tenant_id, config_value):
raise NotImplementedError("Pause is not implemented for legay groups")
return perform(dispatcher, conv_pause_group_eff(scaling_group, transaction_id))
def with_active_cache(self, get_func, *args, **kwargs):
"""
Return result of `get_func` and active cache from servers table
if this is convergence enabled tenant
"""
if tenant_is_enabled(self.tenant_id, config_value):
cache_d = get_active_cache(
self.store.reactor, self.store.connection, self.tenant_id,
self.group_id)
else:
cache_d = succeed(None)
return gatherResults([get_func(*args, **kwargs), cache_d],
consumeErrors=True)
def test_tenant_is_not_enabled(self):
"""
:obj:`convergence.tenant_is_enabled` should return ``False`` when a
given tenant ID has convergence behavior turned off.
"""
enabled_tenant_id = "some-tenant"
def get_config_value(config_key):
self.assertEqual(config_key, "convergence-tenants")
return [enabled_tenant_id + "-nope"]
self.assertEqual(tenant_is_enabled(enabled_tenant_id,
get_config_value),
False)
def test_tenant_is_not_enabled(self):
"""
:obj:`convergence.tenant_is_enabled` should return ``False`` when a
given tenant ID has convergence behavior turned off.
"""
disabled_tenant_id = "some-tenant"
def get_config_value(config_key):
self.assertEqual(config_key, "non-convergence-tenants")
return [disabled_tenant_id]
self.assertEqual(tenant_is_enabled(disabled_tenant_id,
get_config_value),
False)
def test_tenant_is_enabled(self):
"""
:obj:`convergence.tenant_is_enabled` should return ``True`` when a
given tenant ID has convergence behavior turned on.
"""
disabled_tenant_id = "some-tenant"
def get_config_value(config_key):
self.assertEqual(config_key, "non-convergence-tenants")
return [disabled_tenant_id]
self.assertEqual(tenant_is_enabled("some-other-tenant",
get_config_value),
True)
def with_active_cache(self, get_func, *args, **kwargs):
"""
Return result of `get_func` and active cache from servers table
if this is convergence enabled tenant
"""
if tenant_is_enabled(self.tenant_id, config_value):
cache_d = get_active_cache(
self.store.reactor, self.store.connection, self.tenant_id,
self.group_id)
else:
cache_d = succeed(None)
return gatherResults([get_func(*args, **kwargs), cache_d],
consumeErrors=True)
def resume_scaling_group(log, transaction_id, scaling_group, dispatcher):
"""
Resumes the scaling group, causing all scaling policy executions to be
evaluated as normal again. This is an idempotent change, if it's already
paused, this does not raise an error.
:raises: :class:`NoSuchScalingGroup` if the scaling group does not exist.
:return: None
"""
if not tenant_is_enabled(scaling_group.tenant_id, config_value):
raise NotImplementedError("Resume is not implemented for legacy groups")
return perform(dispatcher, conv_resume_group_eff(transaction_id, scaling_group))
def remove_server_from_group(dispatcher,
log,
trans_id,
server_id,
replace,
purge,
group,
state,
config_value=config_value):
"""
Remove a specific server from the group, optionally replacing it
with a new one, and optionally deleting the old one from Nova.
If the old server is not deleted from Nova, otter-specific metadata
is removed: otherwise, a different part of otter may later mistake
the server as one that *should* still be in the group.
:param log: A bound logger
:param bytes trans_id: The transaction id for this operation.
:param bytes server_id: The id of the server to be removed.
:param bool replace: Should the server be replaced?
:param bool purge: Should the server be deleted from Nova?
:param group: The scaling group to remove a server from.
:type group: :class:`~otter.models.interface.IScalingGroup`
:param state: The current state of the group.
:type state: :class:`~otter.models.interface.GroupState`
:return: The updated state.
:rtype: deferred :class:`~otter.models.interface.GroupState`
"""
# worker case
if not tenant_is_enabled(group.tenant_id, config_value):
return worker_remove_server_from_group(log, trans_id, server_id,
replace, purge, group, state)
# convergence case - requires that the convergence dispatcher handles
# EvictServerFromScalingGroup
eff = convergence_remove_server_from_group(log, trans_id, server_id,
replace, purge, group, state)
def kick_off_convergence(new_state):
ceff = trigger_convergence(group.tenant_id, group.uuid)
return ceff.on(lambda _: new_state)
return perform(
dispatcher,
with_log(eff.on(kick_off_convergence),
tenant_id=group.tenant_id,
scaling_group_id=group.uuid,
server_id=server_id,
transaction_id=trans_id))
def pause_scaling_group(log, transaction_id, scaling_group, dispatcher):
"""
Pauses the scaling group, causing all scaling policy executions to be
rejected until unpaused. This is an idempotent change, if it's already
paused, this does not raise an error.
:raises: :class:`NoSuchScalingGroup` if the scaling group does not exist.
:return: None
"""
if not tenant_is_enabled(scaling_group.tenant_id, config_value):
raise NotImplementedError("Pause is not implemented for legay groups")
return perform(dispatcher,
conv_pause_group_eff(scaling_group, transaction_id))
def resume_scaling_group(log, transaction_id, scaling_group, dispatcher):
"""
Resumes the scaling group, causing all scaling policy executions to be
evaluated as normal again. This is an idempotent change, if it's already
paused, this does not raise an error.
:raises: :class:`NoSuchScalingGroup` if the scaling group does not exist.
:return: None
"""
if not tenant_is_enabled(scaling_group.tenant_id, config_value):
raise NotImplementedError(
'Resume is not implemented for legacy groups')
return perform(dispatcher,
conv_resume_group_eff(transaction_id, scaling_group))
def remove_server_from_group(
dispatcher, log, trans_id, server_id, replace, purge, group, state, config_value=config_value
):
"""
Remove a specific server from the group, optionally replacing it
with a new one, and optionally deleting the old one from Nova.
If the old server is not deleted from Nova, otter-specific metadata
is removed: otherwise, a different part of otter may later mistake
the server as one that *should* still be in the group.
:param log: A bound logger
:param bytes trans_id: The transaction id for this operation.
:param bytes server_id: The id of the server to be removed.
:param bool replace: Should the server be replaced?
:param bool purge: Should the server be deleted from Nova?
:param group: The scaling group to remove a server from.
:type group: :class:`~otter.models.interface.IScalingGroup`
:param state: The current state of the group.
:type state: :class:`~otter.models.interface.GroupState`
:return: The updated state.
:rtype: deferred :class:`~otter.models.interface.GroupState`
"""
# worker case
if not tenant_is_enabled(group.tenant_id, config_value):
return worker_remove_server_from_group(log, trans_id, server_id, replace, purge, group, state)
# convergence case - requires that the convergence dispatcher handles
# EvictServerFromScalingGroup
eff = convergence_remove_server_from_group(log, trans_id, server_id, replace, purge, group, state)
def kick_off_convergence(new_state):
ceff = trigger_convergence(group.tenant_id, group.uuid)
return ceff.on(lambda _: new_state)
return perform(
dispatcher,
with_log(
eff.on(kick_off_convergence),
tenant_id=group.tenant_id,
scaling_group_id=group.uuid,
server_id=server_id,
transaction_id=trans_id,
),
)
def converge_scaling_group(self, request):
"""
Trigger convergence on given scaling group
"""
def can_converge(group, state):
if state.paused:
raise GroupPausedError(group.tenant_id, group.uuid, "converge")
return state
if tenant_is_enabled(self.tenant_id, config_value):
group = self.store.get_scaling_group(self.log, self.tenant_id,
self.group_id)
return controller.modify_and_trigger(self.dispatcher, group,
bound_log_kwargs(self.log),
can_converge)
else:
request.setResponseCode(404)
def converge_scaling_group(self, request):
"""
Trigger convergence on given scaling group
"""
def is_group_paused(group, state):
if state.paused:
raise GroupPausedError(group.tenant_id, group.uuid, "converge")
return state
if tenant_is_enabled(self.tenant_id, config_value):
group = self.store.get_scaling_group(
self.log, self.tenant_id, self.group_id)
return controller.modify_and_trigger(
self.dispatcher,
group,
bound_log_kwargs(log),
is_group_paused)
else:
request.setResponseCode(404)
def modify_and_trigger(dispatcher, group, logargs, modifier, *args, **kwargs):
"""
Modify group state and trigger convergence after that
:param IScalingGroup group: Scaling group whose state is getting modified
:param log: Bound logger
:param modifier: Callable as described in IScalingGroup.modify_state
:return: Deferred with None
"""
cannot_exec_pol_err = None
try:
yield group.modify_state(modifier, *args, **kwargs)
except CannotExecutePolicyError as ce:
cannot_exec_pol_err = ce
if tenant_is_enabled(group.tenant_id, config_value):
eff = Effect(BoundFields(trigger_convergence(group.tenant_id, group.uuid), logargs))
yield perform(dispatcher, eff)
if cannot_exec_pol_err is not None:
raise cannot_exec_pol_err
def delete_group(dispatcher, log, trans_id, group, force):
"""
Delete group based on the kind of tenant
:param log: Bound logger
:param str trans_id: Transaction ID of request doing this
:param otter.models.interface.IScalingGroup scaling_group: the scaling
group object
:param bool force: Should group be deleted even if it has servers?
:return: Deferred that fires with None
:raise: `GroupNotEmptyError` if group is not empty and force=False
"""
def check_and_delete(_group, state):
if state.desired == 0:
d = trigger_convergence_deletion(dispatcher, group, trans_id)
return d.addCallback(lambda _: state)
else:
raise GroupNotEmptyError(group.tenant_id, group.uuid)
if tenant_is_enabled(group.tenant_id, config_value):
if force:
# We don't care about servers in the group. So trigger deletion
# since it will take precedence over other status
d = trigger_convergence_deletion(dispatcher, group, trans_id)
else:
# Delete only if desired is 0 which must be done with a lock to
# ensure desired is not getting modified by another thread/node
# when executing policy
d = group.modify_state(
check_and_delete,
modify_state_reason='delete_group')
else:
if force:
d = empty_group(log, trans_id, group)
d.addCallback(lambda _: group.delete_group())
else:
d = group.delete_group()
return d
def modify_and_trigger(dispatcher, group, logargs, modifier, *args, **kwargs):
"""
Modify group state and trigger convergence after that
:param IScalingGroup group: Scaling group whose state is getting modified
:param log: Bound logger
:param modifier: Callable as described in IScalingGroup.modify_state
:return: Deferred with None
"""
cannot_exec_pol_err = None
try:
yield group.modify_state(modifier, *args, **kwargs)
except CannotExecutePolicyError as ce:
cannot_exec_pol_err = ce
if tenant_is_enabled(group.tenant_id, config_value):
eff = Effect(
BoundFields(trigger_convergence(group.tenant_id, group.uuid),
logargs))
yield perform(dispatcher, eff)
if cannot_exec_pol_err is not None:
raise cannot_exec_pol_err
def delete_group(dispatcher, log, trans_id, group, force):
"""
Delete group based on the kind of tenant
:param log: Bound logger
:param str trans_id: Transaction ID of request doing this
:param otter.models.interface.IScalingGroup scaling_group: the scaling
group object
:param bool force: Should group be deleted even if it has servers?
:return: Deferred that fires with None
:raise: `GroupNotEmptyError` if group is not empty and force=False
"""
def check_and_delete(_group, state):
if state.desired == 0:
d = trigger_convergence_deletion(dispatcher, group, trans_id)
return d.addCallback(lambda _: state)
else:
raise GroupNotEmptyError(group.tenant_id, group.uuid)
if tenant_is_enabled(group.tenant_id, config_value):
if force:
# We don't care about servers in the group. So trigger deletion
# since it will take precedence over other status
d = trigger_convergence_deletion(dispatcher, group, trans_id)
else:
# Delete only if desired is 0 which must be done with a lock to
# ensure desired is not getting modified by another thread/node
# when executing policy
d = group.modify_state(check_and_delete,
modify_state_reason='delete_group')
else:
if force:
d = empty_group(log, trans_id, group)
d.addCallback(lambda _: group.delete_group())
else:
d = group.delete_group()
return d
def test_all(self):
"""When the value is ``'none'``, True is returned for any tenant."""
def get_config_value(config_key):
self.assertEqual(config_key, "non-convergence-tenants")
return 'none'
self.assertEqual(tenant_is_enabled('foo', get_config_value), True)
def test_unconfigured(self):
"""
When no `non-convergence-tenants` key is available in the config,
every tenant has enabled convergence
"""
self.assertEqual(tenant_is_enabled('foo', lambda x: None), True)
def test_unconfigured(self):
"""
When no `convergence-tenants` key is available in the config, False is
returned.
"""
self.assertEqual(tenant_is_enabled('foo', lambda x: None), False)