def start_nodes(self):
# Destroy the box to begin, so that we are guaranteed
# a clean build.
check_safe_call(
['vagrant', 'destroy', '-f'],
cwd=self.vagrant_path.path)
box_version = vagrant_version(self.package_source.version)
# Boot the VMs
check_safe_call(
['vagrant', 'up'],
cwd=self.vagrant_path.path,
env=extend_environ(FLOCKER_BOX_VERSION=box_version))
for node in self.NODE_ADDRESSES:
remove_known_host(node)
perform(
dispatcher,
run_remotely(
username='******',
address=node,
commands=task_pull_docker_images()
),
)
return self.NODE_ADDRESSES
def perform_tenant_scope(
authenticator,
log,
service_configs,
throttler,
dispatcher,
tenant_scope,
box,
_concretize=concretize_service_request,
):
"""
Perform a :obj:`TenantScope` by performing its :attr:`TenantScope.effect`,
with a dispatcher extended with a performer for :obj:`ServiceRequest`
intents. The performer will use the tenant provided by the
:obj:`TenantScope`.
The first arguments before (dispatcher, tenant_scope, box) are intended
to be partially applied, and the result is a performer that can be put into
a dispatcher.
"""
@sync_performer
def scoped_performer(dispatcher, service_request):
return _concretize(authenticator, log, service_configs, throttler, tenant_scope.tenant_id, service_request)
new_disp = ComposedDispatcher([TypeDispatcher({ServiceRequest: scoped_performer}), dispatcher])
perform(new_disp, tenant_scope.effect.on(box.succeed, box.fail))
def perform_seq(dispatcher, intent, box):
results = []
effects = [effect.on(success=results.append) for effect in intent.effects]
composite = effects[0]
# TODO: failures
for effect in effects[1:]:
composite = composite.on(success=lambda _, effect=effect: effect)
perform(dispatcher, composite.on(success=lambda _: box.succeed(results)))
def __init__(self, effects):
self.lines = [
'#!/bin/bash',
'set -ex'
]
TypeDispatcher.__init__(self, {
Run: self.perform_run,
Sudo: perform_sudo,
Put: perform_put,
Comment: self.perform_comment,
Sequence: perform_sequence
})
perform(self, effects)
# Add blank line to terminate script with a newline
self.lines.append('')
self._script = '\n'.join(self.lines)
def __init__(self, effects):
self.lines = [
'#!/bin/bash',
'set -ex'
]
TypeDispatcher.__init__(self, {
Run: self.perform_run,
Sudo: perform_sudo,
Put: perform_put,
Comment: self.perform_comment,
Sequence: perform_sequence
})
perform(self, effects)
# Add blank line to terminate script with a newline
self.lines.append('')
self._script = '\n'.join(self.lines)
def start_nodes(self, reactor):
# Destroy the box to begin, so that we are guaranteed
# a clean build.
yield run(
reactor,
['vagrant', 'destroy', '-f'],
path=self.vagrant_path.path)
box_version = vagrant_version(self.package_source.version)
# Boot the VMs
yield run(
reactor,
['vagrant', 'up'],
path=self.vagrant_path.path,
env=extend_environ(FLOCKER_BOX_VERSION=box_version))
for node in self.NODE_ADDRESSES:
yield remove_known_host(reactor, node)
yield perform(
dispatcher,
run_remotely(
username='******',
address=node,
commands=task_pull_docker_images()
),
)
returnValue([
VagrantNode(address=address, distribution=self.distribution)
for address in self.NODE_ADDRESSES
])
def perform_remote(dispatcher, intent, box):
from paramiko import SSHClient
client = SSHClient()
client.load_system_host_keys()
client.connect(intent.host)
remote_runner = perform_run_remotely(client=client)
dispatcher = ComposedDispatcher(dispatchers=[
TypeDispatcher({Run: remote_runner}),
dispatcher
])
# TODO: fail
def close(_):
client.close()
return _
perform(dispatcher, intent.effect.on(success=close)
.on(success=box.succeed, error=box.fail))
def main(args, base_path, top_level):
"""
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = RunOptions(top_level=top_level)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
# We register a signal handler for SIGTERM here.
# When a signal is received, python will call this function
# from the main thread.
# We raise SystemExit to shutdown gracefully.
# In particular, we will kill any processes we spawned
# and cleanup and VMs we created.
signal.signal(signal.SIGTERM, signal_handler)
try:
nodes = runner.start_nodes()
perform(dispatcher,
configure_cluster(control_node=nodes[0], agent_nodes=nodes))
result = run_tests(
nodes=nodes,
control_node=nodes[0], agent_nodes=nodes,
trial_args=options['trial-args'])
except:
result = 1
raise
finally:
# Unless the tests failed, and the user asked to keep the nodes, we
# delete them.
if not (result != 0 and options['keep']):
runner.stop_nodes()
elif options['keep']:
print "--keep specified, not destroying nodes."
raise SystemExit(result)
def main(args):
try:
options = TestBrewOptions()
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("Error: {error}.\n".format(error=str(e)))
sys.exit(1)
recipe_url = options['recipe_url']
options['vmpath'] = FilePath(options['vmpath'])
# Open the recipe URL just to validate and verify that it exists.
# We do not need to read its content.
urllib2.urlopen(recipe_url)
check_output([
"vmrun",
"revertToSnapshot",
options['vmpath'].path,
options['vmsnapshot'],
])
check_output([
"vmrun",
"start",
options['vmpath'].path,
"nogui",
])
perform(
dispatcher,
run_remotely(username=options['vmuser'],
address=options['vmhost'],
commands=task_test_homebrew(recipe_url)),
)
check_output([
"vmrun",
"stop",
options['vmpath'].path,
"hard",
])
print "Done."
except CalledProcessError as e:
sys.stderr.write(
("Error: Command {cmd} terminated with exit status {code}.\n"
).format(cmd=" ".join(e.cmd), code=e.returncode))
raise
def perform_remote(dispatcher, intent, box):
from paramiko import SSHClient
client = SSHClient()
client.load_system_host_keys()
client.connect(intent.host)
remote_runner = perform_run_remotely(client=client)
dispatcher = ComposedDispatcher(
dispatchers=[TypeDispatcher({Run: remote_runner}), dispatcher])
# TODO: fail
def close(_):
client.close()
return _
perform(
dispatcher,
intent.effect.on(success=close).on(success=box.succeed,
error=box.fail))
def main(args, base_path, top_level):
"""
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = RunOptions(top_level=top_level)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
# We register a signal handler for SIGTERM here.
# When a signal is received, python will call this function
# from the main thread.
# We raise SystemExit to shutdown gracefully.
# In particular, we will kill any processes we spawned
# and cleanup and VMs we created.
signal.signal(signal.SIGTERM, signal_handler)
try:
nodes = runner.start_nodes()
perform(dispatcher,
configure_cluster(control_node=nodes[0], agent_nodes=nodes))
result = run_tests(nodes=nodes,
control_node=nodes[0],
agent_nodes=nodes,
trial_args=options['trial-args'])
except:
result = 1
raise
finally:
# Unless the tests failed, and the user asked to keep the nodes, we
# delete them.
if not (result != 0 and options['keep']):
runner.stop_nodes()
elif options['keep']:
print "--keep specified, not destroying nodes."
raise SystemExit(result)
def start_nodes(self):
# Destroy the box to begin, so that we are guaranteed
# a clean build.
check_safe_call(['vagrant', 'destroy', '-f'],
cwd=self.vagrant_path.path)
box_version = vagrant_version(self.package_source.version)
# Boot the VMs
check_safe_call(['vagrant', 'up'],
cwd=self.vagrant_path.path,
env=extend_environ(FLOCKER_BOX_VERSION=box_version))
for node in self.NODE_ADDRESSES:
remove_known_host(node)
perform(
dispatcher,
run_remotely(username='******',
address=node,
commands=task_pull_docker_images()),
)
return self.NODE_ADDRESSES
def main(args):
try:
options = TestBrewOptions()
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("Error: {error}.\n".format(error=str(e)))
sys.exit(1)
recipe_url = options['recipe_url']
options['vmpath'] = FilePath(options['vmpath'])
# Open the recipe URL just to validate and verify that it exists.
# We do not need to read its content.
urllib2.urlopen(recipe_url)
check_output([
"vmrun", "revertToSnapshot",
options['vmpath'].path, options['vmsnapshot'],
])
check_output([
"vmrun", "start", options['vmpath'].path, "nogui",
])
perform(
dispatcher,
run_remotely(
username=options['vmuser'],
address=options['vmhost'],
commands=task_test_homebrew(recipe_url)
),
)
check_output([
"vmrun", "stop", options['vmpath'].path, "hard",
])
print "Done."
except CalledProcessError as e:
sys.stderr.write(
(
"Error: Command {cmd} terminated with exit status {code}.\n"
).format(cmd=" ".join(e.cmd), code=e.returncode)
)
raise
def start_nodes(self):
"""
Provision cloud nodes for acceptance tests.
:return list: List of addresses of nodes to connect to, for acceptance
tests.
"""
metadata = {
'purpose': 'acceptance-testing',
'distribution': self.distribution,
}
metadata.update(self.metadata)
for index in range(2):
name = "acceptance-test-%s-%d" % (self.creator, index)
try:
print "Creating node %d: %s" % (index, name)
node = self.provisioner.create_node(
name=name,
distribution=self.distribution,
metadata=metadata,
)
except:
print "Error creating node %d: %s" % (index, name)
print "It may have leaked into the cloud."
raise
remove_known_host(node.address)
self.nodes.append(node)
del node
commands = sequence([
node.provision(package_source=self.package_source,
variants=self.variants)
for node in self.nodes
])
perform(dispatcher, commands)
return [node.address for node in self.nodes]
def start_nodes(self):
"""
Provision cloud nodes for acceptance tests.
:return list: List of addresses of nodes to connect to, for acceptance
tests.
"""
metadata = {
'purpose': 'acceptance-testing',
'distribution': self.distribution,
}
metadata.update(self.metadata)
for index in range(2):
name = "acceptance-test-%s-%d" % (self.creator, index)
try:
print "Creating node %d: %s" % (index, name)
node = self.provisioner.create_node(
name=name,
distribution=self.distribution,
metadata=metadata,
)
except:
print "Error creating node %d: %s" % (index, name)
print "It may have leaked into the cloud."
raise
remove_known_host(node.address)
self.nodes.append(node)
del node
commands = sequence([
node.provision(package_source=self.package_source,
variants=self.variants) for node in self.nodes
])
perform(dispatcher, commands)
return [node.address for node in self.nodes]
def perform_tenant_scope(
authenticator, log, service_configs, throttler,
dispatcher, tenant_scope, box,
_concretize=concretize_service_request):
"""
Perform a :obj:`TenantScope` by performing its :attr:`TenantScope.effect`,
with a dispatcher extended with a performer for :obj:`ServiceRequest`
intents. The performer will use the tenant provided by the
:obj:`TenantScope`.
The first arguments before (dispatcher, tenant_scope, box) are intended
to be partially applied, and the result is a performer that can be put into
a dispatcher.
"""
@sync_performer
def scoped_performer(dispatcher, service_request):
return _concretize(
authenticator, log, service_configs, throttler,
tenant_scope.tenant_id, service_request)
new_disp = ComposedDispatcher([
TypeDispatcher({ServiceRequest: scoped_performer}),
dispatcher])
perform(new_disp, tenant_scope.effect.on(box.succeed, box.fail))
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = RunOptions(top_level=top_level)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
try:
nodes = yield runner.start_nodes(reactor)
yield perform(
dispatcher,
configure_cluster(control_node=nodes[0], agent_nodes=nodes))
result = yield run_tests(
reactor=reactor,
nodes=nodes,
control_node=nodes[0], agent_nodes=nodes,
trial_args=options['trial-args'])
except:
result = 1
raise
finally:
# Unless the tests failed, and the user asked to keep the nodes, we
# delete them.
if not (result != 0 and options['keep']):
runner.stop_nodes(reactor)
elif options['keep']:
print "--keep specified, not destroying nodes."
raise SystemExit(result)
def bound_log(log, all_fields, disp, intent, box):
""" Perform BoundFields intent """
new_disp = ComposedDispatcher(
[get_log_dispatcher(log, all_fields), disp])
perform(new_disp, intent.effect.on(box.succeed, box.fail))