def collect_metrics(reactor, config, log, client=None, authenticator=None,
_print=False):
"""
Start collecting the metrics
:param reactor: Twisted reactor
:param dict config: Configuration got from file containing all info
needed to collect metrics
:param :class:`silverberg.client.CQLClient` client:
Optional cassandra client. A new client will be created
if this is not given and disconnected before returing
:param :class:`otter.auth.IAuthenticator` authenticator:
Optional authenticator. A new authenticator will be created
if this is not given
:param bool _print: Should debug messages be printed to stdout?
:return: :class:`Deferred` fired with ``list`` of `GroupMetrics`
"""
_client = client or connect_cass_servers(reactor, config['cassandra'])
authenticator = authenticator or generate_authenticator(reactor,
config['identity'])
store = CassScalingGroupCollection(_client, reactor, 1000)
dispatcher = get_dispatcher(reactor, authenticator, log,
get_service_configs(config), store)
# calculate metrics on launch_server and non-paused groups
groups = yield perform(dispatcher, Effect(GetAllValidGroups()))
groups = [
g for g in groups
if json.loads(g["launch_config"]).get("type") == "launch_server" and
(not g.get("paused", False))]
tenanted_groups = groupby(lambda g: g["tenantId"], groups)
group_metrics = yield get_all_metrics(
dispatcher, tenanted_groups, log, _print=_print)
# Add to cloud metrics
metr_conf = config.get("metrics", None)
if metr_conf is not None:
eff = add_to_cloud_metrics(
metr_conf['ttl'], config['region'], group_metrics,
len(tenanted_groups), config, log, _print)
eff = Effect(TenantScope(eff, metr_conf['tenant_id']))
yield perform(dispatcher, eff)
log.msg('added to cloud metrics')
if _print:
print('added to cloud metrics')
if _print:
group_metrics.sort(key=lambda g: abs(g.desired - g.actual),
reverse=True)
print('groups sorted as per divergence')
print('\n'.join(map(str, group_metrics)))
# Disconnect only if we created the client
if not client:
yield _client.disconnect()
defer.returnValue(group_metrics)
def collect_metrics(reactor, config, log, client=None, authenticator=None,
_print=False):
"""
Start collecting the metrics
:param reactor: Twisted reactor
:param dict config: Configuration got from file containing all info
needed to collect metrics
:param :class:`silverberg.client.CQLClient` client:
Optional cassandra client. A new client will be created
if this is not given and disconnected before returing
:param :class:`otter.auth.IAuthenticator` authenticator:
Optional authenticator. A new authenticator will be created
if this is not given
:param bool _print: Should debug messages be printed to stdout?
:return: :class:`Deferred` fired with ``list`` of `GroupMetrics`
"""
convergence_tids = config.get('convergence-tenants', [])
_client = client or connect_cass_servers(reactor, config['cassandra'])
authenticator = authenticator or generate_authenticator(reactor,
config['identity'])
store = CassScalingGroupCollection(_client, reactor, 1000)
dispatcher = get_dispatcher(reactor, authenticator, log,
get_service_configs(config), store)
# calculate metrics
fpath = get_in(["metrics", "last_tenant_fpath"], config,
default="last_tenant.txt")
tenanted_groups = yield perform(
dispatcher,
get_todays_scaling_groups(convergence_tids, fpath))
group_metrics = yield get_all_metrics(
dispatcher, tenanted_groups, log, _print=_print)
# Add to cloud metrics
metr_conf = config.get("metrics", None)
if metr_conf is not None:
eff = add_to_cloud_metrics(
metr_conf['ttl'], config['region'], group_metrics,
len(tenanted_groups), log, _print)
eff = Effect(TenantScope(eff, metr_conf['tenant_id']))
yield perform(dispatcher, eff)
log.msg('added to cloud metrics')
if _print:
print('added to cloud metrics')
if _print:
group_metrics.sort(key=lambda g: abs(g.desired - g.actual),
reverse=True)
print('groups sorted as per divergence', *group_metrics, sep='\n')
# Disconnect only if we created the client
if not client:
yield _client.disconnect()
defer.returnValue(group_metrics)
def perform_run_remotely(reactor, base_dispatcher, intent):
connection_helper = get_connection_helper(
reactor,
username=intent.username, address=intent.address, port=intent.port)
context = Message.new(
username=intent.username, address=intent.address, port=intent.port)
def connect():
connection = connection_helper.secureConnection()
connection.addErrback(write_failure)
timeout(reactor, connection, 30)
return connection
connection = yield loop_until(reactor, connect)
dispatcher = ComposedDispatcher([
get_ssh_dispatcher(
connection=connection,
context=context,
),
base_dispatcher,
])
yield perform(dispatcher, intent.commands)
yield connection_helper.cleanupConnection(
connection, False)
def webhook_migrate(reactor, conn, args):
"""
Migrate webhook indexes to table
"""
store = CassScalingGroupCollection(None, None, 3)
eff = store.get_webhook_index_only().on(store.add_webhook_keys)
return perform(get_working_cql_dispatcher(reactor, conn), eff)
def modify_and_trigger(dispatcher, group, logargs, modifier, *args, **kwargs):
"""
Modify group state and trigger convergence after that if the group is not
suspended. Otherwise fail with :obj:`TenantSuspendedError`.
:param IScalingGroup group: Scaling group whose state is getting modified
:param log: Bound logger
:param modifier: Callable as described in IScalingGroup.modify_state
:return: Deferred with None if modification and convergence succeeded.
Fails with :obj:`TenantSuspendedError` if group is suspended.
"""
def modifier_wrapper(_group, state, *_args, **_kwargs):
# Ideally this will not be allowed by repose middleware but
# adding check for mimic based integration tests
if state.suspended:
raise TenantSuspendedError(_group.tenant_id)
return modifier(_group, state, *_args, **_kwargs)
cannot_exec_pol_err = None
try:
yield group.modify_state(modifier_wrapper, *args, **kwargs)
except CannotExecutePolicyError as ce:
cannot_exec_pol_err = ce
if tenant_is_enabled(group.tenant_id, config_value):
eff = Effect(
BoundFields(
trigger_convergence(group.tenant_id, group.uuid), logargs))
yield perform(dispatcher, eff)
if cannot_exec_pol_err is not None:
raise cannot_exec_pol_err
def modify_and_trigger(dispatcher, group, logargs, modifier, *args, **kwargs):
"""
Modify group state and trigger convergence after that if the group is not
suspended. Otherwise fail with :obj:`TenantSuspendedError`.
:param IScalingGroup group: Scaling group whose state is getting modified
:param log: Bound logger
:param modifier: Callable as described in IScalingGroup.modify_state
:return: Deferred with None if modification and convergence succeeded.
Fails with :obj:`TenantSuspendedError` if group is suspended.
"""
def modifier_wrapper(_group, state, *_args, **_kwargs):
# Ideally this will not be allowed by repose middleware but
# adding check for mimic based integration tests
if state.suspended:
raise TenantSuspendedError(_group.tenant_id)
return modifier(_group, state, *_args, **_kwargs)
cannot_exec_pol_err = None
try:
yield group.modify_state(modifier_wrapper, *args, **kwargs)
except CannotExecutePolicyError as ce:
cannot_exec_pol_err = ce
if tenant_is_enabled(group.tenant_id, config_value):
eff = Effect(
BoundFields(trigger_convergence(group.tenant_id, group.uuid),
logargs))
yield perform(dispatcher, eff)
if cannot_exec_pol_err is not None:
raise cannot_exec_pol_err
def webhook_migrate(reactor, conn, args):
"""
Migrate webhook indexes to table
"""
store = CassScalingGroupCollection(None, None, 3)
eff = store.get_webhook_index_only().on(store.add_webhook_keys)
return perform(get_working_cql_dispatcher(reactor, conn), eff)
def perform_run_remotely(reactor, base_dispatcher, intent):
connection_helper = get_connection_helper(reactor,
username=intent.username,
address=intent.address,
port=intent.port)
context = Message.new(username=intent.username,
address=intent.address,
port=intent.port)
def connect():
connection = connection_helper.secureConnection()
connection.addErrback(write_failure)
timeout(reactor, connection, 30)
return connection
connection = yield loop_until(reactor, connect)
dispatcher = ComposedDispatcher([
get_ssh_dispatcher(
connection=connection,
context=context,
),
base_dispatcher,
])
yield perform(dispatcher, intent.commands)
yield connection_helper.cleanupConnection(connection, False)
def webhook_index(reactor, conn, args):
"""
Show webhook indexes that is not there table connection
"""
store = CassScalingGroupCollection(None, None, 3)
eff = store.get_webhook_index_only()
return perform(get_working_cql_dispatcher(reactor, conn), eff)
def webhook_index(reactor, conn, args):
"""
Show webhook indexes that is not there table connection
"""
store = CassScalingGroupCollection(None, None, 3)
eff = store.get_webhook_index_only()
return perform(get_working_cql_dispatcher(reactor, conn), eff)
def acquire(self, blocking=True, timeout=None):
"""
Same as :meth:`kazoo.recipe.lock.Lock.acquire` except that this can be
called again on an object that has been released. It will start fresh
process to acquire the lock.
"""
return perform(self.dispatcher, self.acquire_eff(blocking, timeout))
def start_cluster(self, reactor):
"""
Provision cloud cluster for acceptance tests.
:return Cluster: The cluster to connect to for acceptance tests.
"""
metadata = {
'purpose': 'acceptance-testing',
'distribution': self.distribution,
}
metadata.update(self.metadata)
for index in range(self.num_nodes):
name = "acceptance-test-%s-%d" % (self.creator, index)
try:
print "Creating node %d: %s" % (index, name)
node = self.provisioner.create_node(
name=name,
distribution=self.distribution,
metadata=metadata,
)
except:
print "Error creating node %d: %s" % (index, name)
print "It may have leaked into the cloud."
raise
yield remove_known_host(reactor, node.address)
self.nodes.append(node)
del node
commands = parallel([
node.provision(package_source=self.package_source,
variants=self.variants)
for node in self.nodes
])
if self.dataset_backend == DatasetBackend.zfs:
zfs_commands = parallel([
configure_zfs(node, variants=self.variants)
for node in self.nodes
])
commands = commands.on(success=lambda _: zfs_commands)
yield perform(make_dispatcher(reactor), commands)
cluster = yield configured_cluster_for_nodes(
reactor,
generate_certificates(
make_cluster_id(
TestTypes.ACCEPTANCE,
_provider_for_cluster_id(self.dataset_backend),
),
self.nodes),
self.nodes,
self.dataset_backend,
self.dataset_backend_configuration,
_save_backend_configuration(self.dataset_backend,
self.dataset_backend_configuration)
)
returnValue(cluster)
def test_log_none_effectful_fields(self):
"""
When log is not passed, but there are log fields from BoundFields,
the log passed to treq has those fields.
"""
log = mock_log()
# we have to include system='otter' in the expected log here because
# the code falls back to otter.log.log, which has the system key bound.
expected_log = matches(IsBoundWith(bound='stuff', system='otter'))
req = ('GET', 'http://google.com/', None, None, None, {
'log': expected_log
})
response = StubResponse(200, {})
treq = StubTreq(reqs=[(req, response)],
contents=[(response, "content")])
req = Request(method="get", url="http://google.com/")
req.treq = treq
req_eff = Effect(req)
bound_log_eff = with_log(req_eff, bound='stuff')
dispatcher = ComposedDispatcher(
[get_simple_dispatcher(None),
get_log_dispatcher(log, {})])
self.assertEqual(
self.successResultOf(perform(dispatcher, bound_log_eff)),
(response, "content"))
def _upgrade_flocker(self, reactor, nodes, package_source):
"""
Put the version of Flocker indicated by ``package_source`` onto all of
the given nodes.
This takes a primitive approach of uninstalling the software and then
installing the new version instead of trying to take advantage of any
OS-level package upgrade support. Because it's easier. The package
removal step is allowed to fail in case the package is not installed
yet (other failures are not differentiated). The only action taken on
failure is that the failure is logged.
:param pvector nodes: The ``ManagedNode``\ s on which to upgrade the
software.
:param PackageSource package_source: The version of the software to
which to upgrade.
:return: A ``Deferred`` that fires when the software has been upgraded.
"""
dispatcher = make_dispatcher(reactor)
uninstalling = perform(dispatcher, uninstall_flocker(nodes))
uninstalling.addErrback(write_failure, logger=None)
def install(ignored):
return perform(
dispatcher,
install_flocker(nodes, package_source),
)
installing = uninstalling.addCallback(install)
return installing
def acquire(self, blocking=True, timeout=None):
"""
Same as :meth:`kazoo.recipe.lock.Lock.acquire` except that this can be
called again on an object that has been released. It will start fresh
process to acquire the lock.
"""
return perform(self.dispatcher, self.acquire_eff(blocking, timeout))
def configured_cluster_for_nodes(
reactor, certificates, nodes, dataset_backend,
dataset_backend_configuration, dataset_backend_config_file,
provider=None
):
"""
Get a ``Cluster`` with Flocker services running on the right nodes.
:param reactor: The reactor.
:param Certificates certificates: The certificates to install on the
cluster.
:param nodes: The ``ManagedNode``s on which to operate.
:param NamedConstant dataset_backend: The ``DatasetBackend`` constant
representing the dataset backend that the nodes will be configured to
use when they are "started".
:param dict dataset_backend_configuration: The backend-specific
configuration the nodes will be given for their dataset backend.
:param FilePath dataset_backend_config_file: A FilePath that has the
dataset_backend info stored.
:returns: A ``Deferred`` which fires with ``Cluster`` when it is
configured.
"""
# XXX: There is duplication between the values here and those in
# f.node.agents.test.blockdevicefactory.MINIMUM_ALLOCATABLE_SIZES. We want
# the default volume size to be greater than or equal to the minimum
# allocatable size.
#
# Ideally, the minimum allocatable size (and perhaps the default volume
# size) would be something known by an object that represents the dataset
# backend. Unfortunately:
# 1. There is no such object
# 2. There is existing confusion in the code around 'openstack' and
# 'rackspace'
#
# Here, we special-case Rackspace (presumably) because it has a minimum
# allocatable size that is different from other Openstack backends.
#
# FLOC-2584 also discusses this.
default_volume_size = GiB(1)
if dataset_backend_configuration.get('auth_plugin') == 'rackspace':
default_volume_size = RACKSPACE_MINIMUM_VOLUME_SIZE
cluster = Cluster(
all_nodes=pvector(nodes),
control_node=nodes[0],
agent_nodes=nodes,
dataset_backend=dataset_backend,
default_volume_size=int(default_volume_size.to_Byte().value),
certificates=certificates,
dataset_backend_config_file=dataset_backend_config_file
)
configuring = perform(
make_dispatcher(reactor),
configure_cluster(cluster, dataset_backend_configuration, provider)
)
configuring.addCallback(lambda ignored: cluster)
return configuring
def configured_cluster_for_nodes(
reactor, certificates, nodes, dataset_backend,
dataset_backend_configuration, dataset_backend_config_file,
provider=None
):
"""
Get a ``Cluster`` with Flocker services running on the right nodes.
:param reactor: The reactor.
:param Certificates certificates: The certificates to install on the
cluster.
:param nodes: The ``ManagedNode``s on which to operate.
:param NamedConstant dataset_backend: The ``DatasetBackend`` constant
representing the dataset backend that the nodes will be configured to
use when they are "started".
:param dict dataset_backend_configuration: The backend-specific
configuration the nodes will be given for their dataset backend.
:param FilePath dataset_backend_config_file: A FilePath that has the
dataset_backend info stored.
:returns: A ``Deferred`` which fires with ``Cluster`` when it is
configured.
"""
# XXX: There is duplication between the values here and those in
# f.node.agents.test.blockdevicefactory.MINIMUM_ALLOCATABLE_SIZES. We want
# the default volume size to be greater than or equal to the minimum
# allocatable size.
#
# Ideally, the minimum allocatable size (and perhaps the default volume
# size) would be something known by an object that represents the dataset
# backend. Unfortunately:
# 1. There is no such object
# 2. There is existing confusion in the code around 'openstack' and
# 'rackspace'
#
# Here, we special-case Rackspace (presumably) because it has a minimum
# allocatable size that is different from other Openstack backends.
#
# FLOC-2584 also discusses this.
default_volume_size = GiB(1)
if dataset_backend_configuration.get('auth_plugin') == 'rackspace':
default_volume_size = RACKSPACE_MINIMUM_VOLUME_SIZE
cluster = Cluster(
all_nodes=pvector(nodes),
control_node=nodes[0],
agent_nodes=nodes,
dataset_backend=dataset_backend,
default_volume_size=int(default_volume_size.to_Byte().value),
certificates=certificates,
dataset_backend_config_file=dataset_backend_config_file
)
configuring = perform(
make_dispatcher(reactor),
configure_cluster(cluster, dataset_backend_configuration, provider)
)
configuring.addCallback(lambda ignored: cluster)
return configuring
def is_acquired(self):
"""
Is the lock already acquired? This method does not exist in kazoo
lock recipe and is a nice addition to it.
:return: :obj:`Deferred` of ``bool``
"""
return perform(self.dispatcher, self.is_acquired_eff())
def is_acquired(self):
"""
Is the lock already acquired? This method does not exist in kazoo
lock recipe and is a nice addition to it.
:return: :obj:`Deferred` of ``bool``
"""
return perform(self.dispatcher, self.is_acquired_eff())
def get_active_cache(reactor, connection, tenant_id, group_id):
"""
Get active servers from servers cache table
"""
eff = CassScalingGroupServersCache(tenant_id, group_id).get_servers(True)
disp = get_working_cql_dispatcher(reactor, connection)
d = perform(disp, eff)
return d.addCallback(lambda (servers, _): {s['id']: s for s in servers})
def get_active_cache(reactor, connection, tenant_id, group_id):
"""
Get active servers from servers cache table
"""
eff = CassScalingGroupServersCache(tenant_id, group_id).get_servers(True)
disp = get_working_cql_dispatcher(reactor, connection)
d = perform(disp, eff)
return d.addCallback(lambda (servers, _): {s['id']: s for s in servers})
def groups_steps(groups, reactor, store, cass_client, authenticator, conf):
"""
Return [(group, steps)] list
"""
eff = parallel(map(group_steps, groups))
disp = get_full_dispatcher(
reactor, authenticator, mock_log(), get_service_configs(conf),
"kzclient", store, "supervisor", cass_client)
return perform(disp, eff).addCallback(lambda steps: zip(groups, steps))
def groups_steps(groups, reactor, store, cass_client, authenticator, conf):
"""
Return [(group, steps)] list
"""
eff = parallel(map(group_steps, groups))
disp = get_full_dispatcher(reactor, authenticator, mock_log(),
get_service_configs(conf), "kzclient", store,
"supervisor", cass_client)
return perform(disp, eff).addCallback(lambda steps: zip(groups, steps))
def check_and_call():
class DoFunc(object):
pass
@deferred_performer
def func_performer(d, i):
return maybeDeferred(func, *args, **kwargs)
comp_dispatcher = ComposedDispatcher([TypeDispatcher({DoFunc: func_performer}), dispatcher])
return perform(comp_dispatcher, call_if_acquired(lock, Effect(DoFunc())))
def start_convergence(self, log, tenant_id, group_id, perform=perform):
"""Record that a group needs converged by creating a ZooKeeper node."""
log = log.bind(tenant_id=tenant_id, scaling_group_id=group_id)
eff = mark_divergent(tenant_id, group_id)
d = perform(self.dispatcher, eff)
def success(r):
log.msg('mark-dirty-success')
return r # The result is ignored normally, but return it for tests
d.addCallbacks(success, log.err, errbackArgs=('mark-dirty-failure',))
return d
def test_performs_tenant_scope(self, deferred_lock_run):
"""
:func:`perform_tenant_scope` performs :obj:`TenantScope`, and uses the
default throttler
"""
# We want to ensure
# 1. the TenantScope can be performed
# 2. the ServiceRequest is run within a lock, since it matches the
# default throttling policy
set_config_data({
"cloud_client": {
"throttling": {
"create_server_delay": 1,
"delete_server_delay": 0.4
}
}
})
self.addCleanup(set_config_data, {})
clock = Clock()
authenticator = object()
log = object()
dispatcher = get_cloud_client_dispatcher(clock, authenticator, log,
make_service_configs())
svcreq = service_request(ServiceType.CLOUD_SERVERS, 'POST', 'servers')
tscope = TenantScope(tenant_id='111', effect=svcreq)
def run(f, *args, **kwargs):
result = f(*args, **kwargs)
result.addCallback(lambda x: (x[0], assoc(x[1], 'locked', True)))
return result
deferred_lock_run.side_effect = run
response = stub_pure_response({}, 200)
seq = SequenceDispatcher([
(Authenticate(authenticator=authenticator,
tenant_id='111',
log=log), lambda i: ('token', fake_service_catalog)),
(Request(method='POST',
url='http://dfw.openstack/servers',
headers=headers('token'),
log=log), lambda i: response),
])
disp = ComposedDispatcher([seq, dispatcher])
with seq.consume():
result = perform(disp, Effect(tscope))
self.assertNoResult(result)
clock.advance(1)
self.assertEqual(self.successResultOf(result), (response[0], {
'locked': True
}))
def start_convergence(self, log, tenant_id, group_id, perform=perform):
"""Record that a group needs converged by creating a ZooKeeper node."""
log = log.bind(tenant_id=tenant_id, scaling_group_id=group_id)
eff = mark_divergent(tenant_id, group_id)
d = perform(self.dispatcher, eff)
def success(r):
log.msg('mark-dirty-success')
return r # The result is ignored normally, but return it for tests
d.addCallbacks(success, log.err, errbackArgs=('mark-dirty-failure', ))
return d
def check_and_call():
class DoFunc(object):
pass
@deferred_performer
def func_performer(d, i):
return maybeDeferred(func, *args, **kwargs)
comp_dispatcher = ComposedDispatcher(
[TypeDispatcher({DoFunc: func_performer}), dispatcher])
return perform(comp_dispatcher,
call_if_acquired(lock, Effect(DoFunc())))
def resume_scaling_group(log, transaction_id, scaling_group, dispatcher):
"""
Resumes the scaling group, causing all scaling policy executions to be
evaluated as normal again. This is an idempotent change, if it's already
paused, this does not raise an error.
:raises: :class:`NoSuchScalingGroup` if the scaling group does not exist.
:return: None
"""
if not tenant_is_enabled(scaling_group.tenant_id, config_value):
raise NotImplementedError("Resume is not implemented for legacy groups")
return perform(dispatcher, conv_resume_group_eff(transaction_id, scaling_group))
def buckets_acquired(self, my_buckets):
"""
Get dirty flags from zookeeper and run convergence with them.
This is used as the partitioner callback.
"""
ceff = Effect(GetChildren(CONVERGENCE_DIRTY_DIR)).on(
partial(self._converge_all, my_buckets))
# Return deferred as 1-element tuple for testing only.
# Returning deferred would block otter from shutting down until
# it is fired which we don't need to do since convergence is itempotent
# and will be triggered in next start of otter
return (perform(self._dispatcher, self._with_conv_runid(ceff)), )
def pause_scaling_group(log, transaction_id, scaling_group, dispatcher):
"""
Pauses the scaling group, causing all scaling policy executions to be
rejected until unpaused. This is an idempotent change, if it's already
paused, this does not raise an error.
:raises: :class:`NoSuchScalingGroup` if the scaling group does not exist.
:return: None
"""
if not tenant_is_enabled(scaling_group.tenant_id, config_value):
raise NotImplementedError("Pause is not implemented for legay groups")
return perform(dispatcher, conv_pause_group_eff(scaling_group, transaction_id))
def buckets_acquired(self, my_buckets):
"""
Get dirty flags from zookeeper and run convergence with them.
This is used as the partitioner callback.
"""
ceff = Effect(GetChildren(CONVERGENCE_DIRTY_DIR)).on(
partial(self._converge_all, my_buckets))
# Return deferred as 1-element tuple for testing only.
# Returning deferred would block otter from shutting down until
# it is fired which we don't need to do since convergence is itempotent
# and will be triggered in next start of otter
return (perform(self._dispatcher, self._with_conv_runid(ceff)), )
def main(reactor, args, base_path, top_level):
try:
options = TestBrewOptions()
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("Error: {error}.\n".format(error=str(e)))
sys.exit(1)
eliot_to_stdout(MESSAGE_FORMATS, {})
recipe_url = options['recipe_url']
options['vmpath'] = FilePath(options['vmpath'])
# Open the recipe URL just to validate and verify that it exists.
# We do not need to read its content.
urllib2.urlopen(recipe_url)
yield run(reactor, [
"vmrun",
"revertToSnapshot",
options['vmpath'].path,
options['vmsnapshot'],
])
yield run(reactor, [
"vmrun",
"start",
options['vmpath'].path,
"nogui",
])
yield perform(
make_dispatcher(reactor),
run_remotely(
username=options['vmuser'],
address=options['vmhost'],
commands=sequence([
task_configure_brew_path(),
task_test_homebrew(recipe_url),
]),
),
)
yield run(reactor, [
"vmrun",
"stop",
options['vmpath'].path,
"hard",
])
print "Done."
except ProcessTerminated as e:
sys.stderr.write(
("Error: Command terminated with exit status {code}.\n").format(
code=e.exitCode))
raise
def remove_server_from_group(dispatcher,
log,
trans_id,
server_id,
replace,
purge,
group,
state,
config_value=config_value):
"""
Remove a specific server from the group, optionally replacing it
with a new one, and optionally deleting the old one from Nova.
If the old server is not deleted from Nova, otter-specific metadata
is removed: otherwise, a different part of otter may later mistake
the server as one that *should* still be in the group.
:param log: A bound logger
:param bytes trans_id: The transaction id for this operation.
:param bytes server_id: The id of the server to be removed.
:param bool replace: Should the server be replaced?
:param bool purge: Should the server be deleted from Nova?
:param group: The scaling group to remove a server from.
:type group: :class:`~otter.models.interface.IScalingGroup`
:param state: The current state of the group.
:type state: :class:`~otter.models.interface.GroupState`
:return: The updated state.
:rtype: deferred :class:`~otter.models.interface.GroupState`
"""
# worker case
if not tenant_is_enabled(group.tenant_id, config_value):
return worker_remove_server_from_group(log, trans_id, server_id,
replace, purge, group, state)
# convergence case - requires that the convergence dispatcher handles
# EvictServerFromScalingGroup
eff = convergence_remove_server_from_group(log, trans_id, server_id,
replace, purge, group, state)
def kick_off_convergence(new_state):
ceff = trigger_convergence(group.tenant_id, group.uuid)
return ceff.on(lambda _: new_state)
return perform(
dispatcher,
with_log(eff.on(kick_off_convergence),
tenant_id=group.tenant_id,
scaling_group_id=group.uuid,
server_id=server_id,
transaction_id=trans_id))
def pause_scaling_group(log, transaction_id, scaling_group, dispatcher):
"""
Pauses the scaling group, causing all scaling policy executions to be
rejected until unpaused. This is an idempotent change, if it's already
paused, this does not raise an error.
:raises: :class:`NoSuchScalingGroup` if the scaling group does not exist.
:return: None
"""
if not tenant_is_enabled(scaling_group.tenant_id, config_value):
raise NotImplementedError("Pause is not implemented for legay groups")
return perform(dispatcher,
conv_pause_group_eff(scaling_group, transaction_id))
def _generic_rcv3_request(operation, request_bag, lb_id, server_id):
"""
Perform a generic RCv3 bulk operation on a single (lb, server) pair.
:param callable operation: RCv3 function to perform on (lb, server) pair.
:param request_bag: An object with a bunch of useful data on it.
:param str lb_id: The id of the RCv3 load balancer to act on.
:param str server_id: The Nova server id to act on.
:return: A deferred that will fire when the request has been performed,
firing with the parsed result of the request, or :data:`None` if the
request has no body.
"""
eff = operation(pset([(lb_id, server_id)]))
scoped = Effect(TenantScope(eff, request_bag.tenant_id))
return perform(request_bag.dispatcher, scoped)
def resume_scaling_group(log, transaction_id, scaling_group, dispatcher):
"""
Resumes the scaling group, causing all scaling policy executions to be
evaluated as normal again. This is an idempotent change, if it's already
paused, this does not raise an error.
:raises: :class:`NoSuchScalingGroup` if the scaling group does not exist.
:return: None
"""
if not tenant_is_enabled(scaling_group.tenant_id, config_value):
raise NotImplementedError(
'Resume is not implemented for legacy groups')
return perform(dispatcher,
conv_resume_group_eff(transaction_id, scaling_group))
def _generic_rcv3_request(operation, request_bag, lb_id, server_id):
"""
Perform a generic RCv3 bulk operation on a single (lb, server) pair.
:param callable operation: RCv3 function to perform on (lb, server) pair.
:param request_bag: An object with a bunch of useful data on it.
:param str lb_id: The id of the RCv3 load balancer to act on.
:param str server_id: The Nova server id to act on.
:return: A deferred that will fire when the request has been performed,
firing with the parsed result of the request, or :data:`None` if the
request has no body.
"""
eff = operation(pset([(lb_id, server_id)]))
scoped = Effect(TenantScope(eff, request_bag.tenant_id))
return perform(request_bag.dispatcher, scoped)
def test_perform(self):
"""
The Request effect dispatches a request to treq, and returns a
two-tuple of the Twisted Response object and the content as bytes.
"""
req = ('GET', 'http://google.com/', None, None, None,
{'log': default_log})
response = StubResponse(200, {})
treq = StubTreq(reqs=[(req, response)],
contents=[(response, "content")])
req = Request(method="get", url="http://google.com/")
req.treq = treq
dispatcher = get_simple_dispatcher(None)
self.assertEqual(
self.successResultOf(perform(dispatcher, Effect(req))),
(response, "content"))
def test_log(self):
"""
The log specified in the Request is passed on to the treq
implementation.
"""
log = object()
req = ('GET', 'http://google.com/', None, None, None, {'log': log})
response = StubResponse(200, {})
treq = StubTreq(reqs=[(req, response)],
contents=[(response, "content")])
req = Request(method="get", url="http://google.com/", log=log)
req.treq = treq
dispatcher = get_simple_dispatcher(None)
self.assertEqual(
self.successResultOf(perform(dispatcher, Effect(req))),
(response, "content"))
def test_run_logs_stdout(self, logger):
"""
The ``Run`` intent logs the standard output of the specified command.
"""
command = run_remotely(
username="******",
address=str(self.server.ip),
port=self.server.port,
commands=run("echo test_ssh_conch:test_run_logs_stdout 1>&2"),
)
d = perform(
make_dispatcher(reactor),
command,
)
return d
def test_run_logs_stderr(self, logger):
"""
The ``Run`` intent logs the standard output of the specified command.
"""
command = run_remotely(
username="******",
address=str(self.server.ip),
port=self.server.port,
commands=run("echo test_ssh_conch:test_run_logs_stderr 1>&2"),
)
d = perform(
make_dispatcher(self.reactor),
command,
)
return d
def test_log(self):
"""
The log specified in the Request is passed on to the treq
implementation.
"""
log = object()
req = ('GET', 'http://google.com/', None, None, None, {'log': log})
response = StubResponse(200, {})
treq = StubTreq(reqs=[(req, response)],
contents=[(response, "content")])
req = Request(method="get", url="http://google.com/", log=log)
req.treq = treq
dispatcher = get_simple_dispatcher(None)
self.assertEqual(
self.successResultOf(perform(dispatcher, Effect(req))),
(response, "content"))
def _setup_control_node(self, reactor, node, index):
print "Selecting node {} for control service".format(node.name)
certificates = Certificates.generate(
directory=self.cert_path,
control_hostname=node.address,
num_nodes=0,
cluster_name=self.identity.name,
cluster_id=self.identity.id,
)
dataset_backend_config_file = save_backend_configuration(
self.dataset_backend, self.dataset_backend_configuration
)
cluster = Cluster(
all_nodes=[node],
control_node=node,
agent_nodes=[],
dataset_backend=self.dataset_backend,
default_volume_size=get_default_volume_size(
self.dataset_backend_configuration
),
certificates=certificates,
dataset_backend_config_file=dataset_backend_config_file
)
commands = configure_control_node(
cluster,
'libcloud',
logging_config=self.config.get('logging'),
)
d = perform(make_dispatcher(reactor), commands)
def configure_failed(failure):
print "Failed to configure control node"
write_failure(failure)
return failure
# It should be sufficient to configure just the control service here,
# but there is an assumption that the control node is both a control
# node and an agent node.
d.addCallbacks(
lambda _: self._add_node_to_cluster(
reactor, cluster, node, index
),
errback=configure_failed,
)
# Return the cluster.
d.addCallback(lambda _: cluster)
return d
def _setup_control_node(self, reactor, node, index):
print "Selecting node {} for control service".format(node.name)
certificates = Certificates.generate(
directory=self.cert_path,
control_hostname=node.address,
num_nodes=0,
cluster_name=self.identity.name,
cluster_id=self.identity.id,
)
dataset_backend_config_file = save_backend_configuration(
self.dataset_backend, self.dataset_backend_configuration
)
cluster = Cluster(
all_nodes=[node],
control_node=node,
agent_nodes=[],
dataset_backend=self.dataset_backend,
default_volume_size=get_default_volume_size(
self.dataset_backend_configuration
),
certificates=certificates,
dataset_backend_config_file=dataset_backend_config_file
)
commands = configure_control_node(
cluster,
'libcloud',
logging_config=self.config.get('logging'),
)
d = perform(make_dispatcher(reactor), commands)
def configure_failed(failure):
print "Failed to configure control node"
write_failure(failure)
return failure
# It should be sufficient to configure just the control service here,
# but there is an assumption that the control node is both a control
# node and an agent node.
d.addCallbacks(
lambda _: self._add_node_to_cluster(
reactor, cluster, node, index
),
errback=configure_failed,
)
# Return the cluster.
d.addCallback(lambda _: cluster)
return d
def divergent_changed(self, children):
"""
ZooKeeper children-watch callback that lets this service know when the
divergent groups have changed. If any of the divergent flags are for
tenants associated with this service's buckets, a convergence will be
triggered.
"""
if self.partitioner.get_current_state() != PartitionState.ACQUIRED:
return
my_buckets = self.partitioner.get_current_buckets()
changed_buckets = set(
bucket_of_tenant(parse_dirty_flag(child)[0], len(self._buckets))
for child in children)
if set(my_buckets).intersection(changed_buckets):
# the return value is ignored, but we return this for testing
eff = self._converge_all(my_buckets, children)
return perform(self._dispatcher, self._with_conv_runid(eff))
def test_perform(self):
"""
The Request effect dispatches a request to treq, and returns a
two-tuple of the Twisted Response object and the content as bytes.
"""
req = ('GET', 'http://google.com/', None, None, None, {
'log': default_log
})
response = StubResponse(200, {})
treq = StubTreq(reqs=[(req, response)],
contents=[(response, "content")])
req = Request(method="get", url="http://google.com/")
req.treq = treq
dispatcher = get_simple_dispatcher(None)
self.assertEqual(
self.successResultOf(perform(dispatcher, Effect(req))),
(response, "content"))
def set_desired_to_actual_group(dispatcher, cass_client, group):
"""
Set group's desired to current number of servers in the group
"""
try:
res_eff = get_all_launch_server_data(
group["tenantId"], group["groupId"], datetime.utcnow())
eff = Effect(TenantScope(res_eff, group["tenantId"]))
resources = yield perform(dispatcher, eff)
actual = active_servers_count(resources["servers"])
print("group", group, "setting desired to ", actual)
yield cass_client.execute(
('UPDATE scaling_group SET desired=:desired WHERE '
'"tenantId"=:tenantId AND "groupId"=:groupId'),
assoc(group, "desired", actual), DEFAULT_CONSISTENCY)
except Exception as e:
print("Couldn't set group {} to {} due to {}".format(group, actual, e))
def test_performs_tenant_scope(self, deferred_lock_run):
"""
:func:`perform_tenant_scope` performs :obj:`TenantScope`, and uses the
default throttler
"""
# We want to ensure
# 1. the TenantScope can be performed
# 2. the ServiceRequest is run within a lock, since it matches the
# default throttling policy
set_config_data(
{"cloud_client": {"throttling": {"create_server_delay": 1,
"delete_server_delay": 0.4}}})
self.addCleanup(set_config_data, {})
clock = Clock()
authenticator = object()
log = object()
dispatcher = get_cloud_client_dispatcher(clock, authenticator, log,
make_service_configs())
svcreq = service_request(ServiceType.CLOUD_SERVERS, 'POST', 'servers')
tscope = TenantScope(tenant_id='111', effect=svcreq)
def run(f, *args, **kwargs):
result = f(*args, **kwargs)
result.addCallback(
lambda x: (x[0], assoc(x[1], 'locked', True)))
return result
deferred_lock_run.side_effect = run
response = stub_pure_response({}, 200)
seq = SequenceDispatcher([
(Authenticate(authenticator=authenticator,
tenant_id='111', log=log),
lambda i: ('token', fake_service_catalog)),
(Request(method='POST', url='http://dfw.openstack/servers',
headers=headers('token'), log=log),
lambda i: response),
])
disp = ComposedDispatcher([seq, dispatcher])
with seq.consume():
result = perform(disp, Effect(tscope))
self.assertNoResult(result)
clock.advance(1)
self.assertEqual(self.successResultOf(result),
(response[0], {'locked': True}))
def remove_server_from_group(
dispatcher, log, trans_id, server_id, replace, purge, group, state, config_value=config_value
):
"""
Remove a specific server from the group, optionally replacing it
with a new one, and optionally deleting the old one from Nova.
If the old server is not deleted from Nova, otter-specific metadata
is removed: otherwise, a different part of otter may later mistake
the server as one that *should* still be in the group.
:param log: A bound logger
:param bytes trans_id: The transaction id for this operation.
:param bytes server_id: The id of the server to be removed.
:param bool replace: Should the server be replaced?
:param bool purge: Should the server be deleted from Nova?
:param group: The scaling group to remove a server from.
:type group: :class:`~otter.models.interface.IScalingGroup`
:param state: The current state of the group.
:type state: :class:`~otter.models.interface.GroupState`
:return: The updated state.
:rtype: deferred :class:`~otter.models.interface.GroupState`
"""
# worker case
if not tenant_is_enabled(group.tenant_id, config_value):
return worker_remove_server_from_group(log, trans_id, server_id, replace, purge, group, state)
# convergence case - requires that the convergence dispatcher handles
# EvictServerFromScalingGroup
eff = convergence_remove_server_from_group(log, trans_id, server_id, replace, purge, group, state)
def kick_off_convergence(new_state):
ceff = trigger_convergence(group.tenant_id, group.uuid)
return ceff.on(lambda _: new_state)
return perform(
dispatcher,
with_log(
eff.on(kick_off_convergence),
tenant_id=group.tenant_id,
scaling_group_id=group.uuid,
server_id=server_id,
transaction_id=trans_id,
),
)
def trigger_convergence_deletion(dispatcher, group, trans_id):
"""
Trigger deletion of group that belongs to convergence tenant
:param log: Bound logger
:param otter.models.interface.IScalingGroup scaling_group: the scaling
group object
"""
# Update group status and trigger convergence
# DELETING status will take precedence over other status
d = group.update_status(ScalingGroupStatus.DELETING)
eff = with_log(trigger_convergence(group.tenant_id, group.uuid),
tenant_id=group.tenant_id,
scaling_group_id=group.uuid,
transaction_id=trans_id)
d.addCallback(lambda _: perform(dispatcher, eff))
return d
def divergent_changed(self, children):
"""
ZooKeeper children-watch callback that lets this service know when the
divergent groups have changed. If any of the divergent flags are for
tenants associated with this service's buckets, a convergence will be
triggered.
"""
if self.partitioner.get_current_state() != PartitionState.ACQUIRED:
return
my_buckets = self.partitioner.get_current_buckets()
changed_buckets = set(
bucket_of_tenant(parse_dirty_flag(child)[0], len(self._buckets))
for child in children)
if set(my_buckets).intersection(changed_buckets):
# the return value is ignored, but we return this for testing
eff = self._converge_all(my_buckets, children)
return perform(self._dispatcher, self._with_conv_runid(eff))
def trigger_convergence_deletion(dispatcher, group, trans_id):
"""
Trigger deletion of group that belongs to convergence tenant
:param log: Bound logger
:param otter.models.interface.IScalingGroup scaling_group: the scaling
group object
"""
# Update group status and trigger convergence
# DELETING status will take precedence over other status
d = group.update_status(ScalingGroupStatus.DELETING)
eff = with_log(trigger_convergence(group.tenant_id, group.uuid),
tenant_id=group.tenant_id,
scaling_group_id=group.uuid,
transaction_id=trans_id)
d.addCallback(lambda _: perform(dispatcher, eff))
return d
def main(reactor, args, base_path, top_level):
try:
options = TestBrewOptions()
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("Error: {error}.\n".format(error=str(e)))
sys.exit(1)
add_destination(eliot_output)
recipe_url = options['recipe_url']
options['vmpath'] = FilePath(options['vmpath'])
# Open the recipe URL just to validate and verify that it exists.
# We do not need to read its content.
urllib2.urlopen(recipe_url)
yield run(reactor, [
"vmrun", "revertToSnapshot",
options['vmpath'].path, options['vmsnapshot'],
])
yield run(reactor, [
"vmrun", "start", options['vmpath'].path, "nogui",
])
yield perform(
make_dispatcher(reactor),
run_remotely(
username=options['vmuser'],
address=options['vmhost'],
commands=sequence([
task_configure_brew_path(),
task_test_homebrew(recipe_url),
]),
),
)
yield run(reactor, [
"vmrun", "stop", options['vmpath'].path, "hard",
])
print "Done."
except ProcessTerminated as e:
sys.stderr.write(
(
"Error: Command terminated with exit status {code}.\n"
).format(code=e.exitCode)
)
raise
def __call__(self, event_dict):
"""
Process event and push it to Cloud feeds
"""
if not event_dict.get("cloud_feed", False):
return
# Do further logging without cloud_feed to avoid coming back here
# in infinite recursion
log_keys = keyfilter(lambda k: k not in ("message", "cloud_feed"), event_dict)
log = self.log.bind(system="otter.cloud_feed", cf_msg=event_dict["message"][0], event_data=log_keys)
try:
eff = self.add_event(event_dict, self.tenant_id, self.region, log)
except UnsuitableMessage as me:
log.err(None, "cf-unsuitable-message", unsuitable_message=me.unsuitable_message)
else:
return perform(self.get_disp(self.reactor, self.authenticator, log, self.service_configs), eff).addErrback(
log.err, "cf-add-failure"
)
def test_run(self):
"""
The ``Run`` intent runs the specified command via ssh.
"""
command = run_remotely(
username="******",
address=str(self.server.ip),
port=self.server.port,
commands=run("touch hello"),
)
d = perform(
make_dispatcher(self.reactor),
command,
)
def check(_):
self.assertEqual(self.server.home.child('hello').getContent(), "")
return d
def _setup_convergences(self):
"""
Get groups to converge and setup scheduled calls to trigger convergence
on each of them within time_range.
"""
groups = yield perform(self.dispatcher,
get_groups_to_converge(self.config_func))
active = self._cancel_scheduled_calls()
if active:
# This should never happen
self.log.err(RuntimeError("selfheal-calls-err"),
"selfheal-calls-err",
active=active)
if not groups:
returnValue(None)
wait_time = self.time_range / len(groups)
for i, group in enumerate(groups):
self._calls.append(
self.clock.callLater(
i * wait_time, perform, self.dispatcher,
check_and_trigger(group["tenantId"], group["groupId"])))
def modify_and_trigger(dispatcher, group, logargs, modifier, *args, **kwargs):
"""
Modify group state and trigger convergence after that
:param IScalingGroup group: Scaling group whose state is getting modified
:param log: Bound logger
:param modifier: Callable as described in IScalingGroup.modify_state
:return: Deferred with None
"""
cannot_exec_pol_err = None
try:
yield group.modify_state(modifier, *args, **kwargs)
except CannotExecutePolicyError as ce:
cannot_exec_pol_err = ce
if tenant_is_enabled(group.tenant_id, config_value):
eff = Effect(
BoundFields(trigger_convergence(group.tenant_id, group.uuid),
logargs))
yield perform(dispatcher, eff)
if cannot_exec_pol_err is not None:
raise cannot_exec_pol_err
