def main():
""" Get file list from args and execute """
global args
args = parser.parse_args()
add_destination(log_stdout)
for filename in args.filename:
try:
print('Loading {}'.format(color.cyan(filename)))
if from_file(filename):
result = color.green('Success.')
else:
result = color.red('No match.')
print('==> {}'.format(result))
except ParserException as exc:
print(color.yellow('ERROR parsing file %s: %s' % (filename, exc)))
except (ConnectionException, ProxyException) as exc:
print(
color.yellow('ERROR connecting to host in file %s: %s' %
(filename, exc)))
except ExploException as exc:
print(color.yellow('ERROR in file %s: %s' % (filename, exc)))
def startService(self):
self.stdlib_cleanup = stdlib_logging_to_eliot_configuration(getLogger())
self.twisted_observer = TwistedLoggerToEliotObserver()
globalLogPublisher.addObserver(self.twisted_observer)
for dest in self.destinations:
add_destination(dest)
def bg(wait, command, raweliot, targets):
"""Run the default oam operation on targets"""
if raweliot:
eliot.to_file(sys.stdout)
else:
# eliottree.render_tasks(sys.stdout.write, tasks, colorize=True) #py3
eliot.add_destination(render_stdout)
procs = []
if len(targets)==0:
targets = ['localhost']
with eliot.start_task(action_type='run_ops', targets=targets):
with eliot.start_action(action_type='start_ops', targets=targets):
for server in targets:
if wait:
cmd = FG_CMD.format(server, command)
else:
cmd = BG_CMD.format(server, SESSION_NAME, command)
logging.debug('%s start, cmd: %s', server, cmd)
with eliot.start_action(action_type='start_process', target=server, cmd=cmd):
procs.append(subprocess.Popen(cmd, shell=True))
finished = 0
with eliot.start_action(action_type='wait_finishes', targets=targets):
while finished != len(procs):
for index, server in enumerate(procs):
logging.debug('looping at %s %d', targets[index], finished)
if not server.poll() is None:
eliot.Message.log(message_type='finish', target=targets[index])
finished += 1
time.sleep(1)
with eliot.start_action(action_type='wait_terminations', targets=targets):
for index, server in enumerate(procs):
with eliot.start_action(action_type='wait_process', target=targets[index]):
server.wait()
logging.debug('%s finish, returncode=%d', targets[index], server.returncode)
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
try:
nodes = yield runner.start_nodes(reactor, node_count=1)
yield perform(
make_dispatcher(reactor),
install_cli(runner.package_source, nodes[0]))
result = yield run_client_tests(reactor=reactor, node=nodes[0])
except:
result = 1
raise
finally:
# Unless the tests failed, and the user asked to keep the nodes, we
# delete them.
if not (result != 0 and options['keep']):
runner.stop_nodes(reactor)
elif options['keep']:
print "--keep specified, not destroying nodes."
raise SystemExit(result)
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the Flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
def cluster_cleanup():
print ("stopping cluster")
return runner.stop_cluster(reactor)
cleanup_trigger_id = reactor.addSystemEventTrigger("before", "shutdown", cluster_cleanup)
from flocker.common.script import eliot_logging_service
log_writer = eliot_logging_service(
destination=FileDestination(file=open("%s.log" % (base_path.basename(),), "a")),
reactor=reactor,
capture_stdout=False,
)
log_writer.startService()
reactor.addSystemEventTrigger("before", "shutdown", log_writer.stopService)
yield runner.ensure_keys(reactor)
cluster = yield runner.start_cluster(reactor)
save_managed_config(options["cert-directory"], options["config"], cluster)
managed_config_file = options["cert-directory"].child("managed.yaml")
managed_config = create_managed_config(options["config"], cluster)
managed_config_file.setContent(yaml.safe_dump(managed_config, default_flow_style=False))
if options["distribution"] in ("centos-7",):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
capture_journal(reactor, node.address, remote_logs_file).addErrback(write_failure)
elif options["distribution"] in ("ubuntu-14.04", "ubuntu-15.10"):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
capture_upstart(reactor, node.address, remote_logs_file).addErrback(write_failure)
flocker_client = make_client(reactor, cluster)
yield wait_for_nodes(reactor, flocker_client, len(cluster.agent_nodes))
if options["no-keep"]:
print ("not keeping cluster")
else:
save_environment(options["cert-directory"], cluster)
reactor.removeSystemEventTrigger(cleanup_trigger_id)
def setup_logging(logger_name, human=False):
"""Patch in the Eliot logger and twisted log interception"""
global LOGGER, HUMAN
LOGGER = "-".join([logger_name, pkg_resources.get_distribution("autopush").version])
HUMAN = human
add_destination(stdout)
ellie = EliotObserver()
ellie.start()
return ellie
def setup_logging(logger_name, human=False):
"""Patch in the Eliot logger and twisted log interception"""
global LOGGER, HUMAN
LOGGER = "-".join(
[logger_name,
pkg_resources.get_distribution("autopush").version])
HUMAN = human
add_destination(stdout)
ellie = EliotObserver()
ellie.start()
return ellie
def _setUp(self):
self.logs = []
add_destination(self.logs.append)
self.addCleanup(lambda: remove_destination(self.logs.append))
self.addDetail(
self.LOG_DETAIL_NAME,
Content(
UTF8_TEXT,
# Safeguard the logs against _tearDown. Capture the list
# object in the lambda's defaults.
lambda logs=self.logs: [_eliottree(logs)],
),
)
def main(reactor, args, base_path, top_level):
try:
options = TestBrewOptions()
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("Error: {error}.\n".format(error=str(e)))
sys.exit(1)
add_destination(eliot_output)
recipe_url = options['recipe_url']
options['vmpath'] = FilePath(options['vmpath'])
# Open the recipe URL just to validate and verify that it exists.
# We do not need to read its content.
urllib2.urlopen(recipe_url)
yield run(reactor, [
"vmrun",
"revertToSnapshot",
options['vmpath'].path,
options['vmsnapshot'],
])
yield run(reactor, [
"vmrun",
"start",
options['vmpath'].path,
"nogui",
])
yield perform(
make_dispatcher(reactor),
run_remotely(
username=options['vmuser'],
address=options['vmhost'],
commands=sequence([
task_configure_brew_path(),
task_test_homebrew(recipe_url),
]),
),
)
yield run(reactor, [
"vmrun",
"stop",
options['vmpath'].path,
"hard",
])
print "Done."
except ProcessTerminated as e:
sys.stderr.write(
("Error: Command terminated with exit status {code}.\n").format(
code=e.exitCode))
raise
def __init__(self, stream, tbformat='default', realtime=False,
publisher=None, logger=None):
# TODO: Trial has a pretty confusing set of expectations for
# reporters. In particular, it's not clear what it needs to construct
# a reporter. It's also not clear what it expects as public
# properties. The IReporter interface and the tests for the reporter
# interface cover somewhat different things.
self._stream = stream
self.tbformat = tbformat
self.shouldStop = False
self.testsRun = 0
add_destination(self._write_message)
self._current_test = None
self._successful = True
self._logger = logger
def main(reactor, argv, environ):
add_destination(eliot_output)
try:
options = ContainerOptions()
options.parseOptions(argv[1:])
except usage.UsageError as e:
sys.stderr.write(e.args[0])
sys.stderr.write('\n\n')
sys.stderr.write(options.getSynopsis())
sys.stderr.write('\n')
sys.stderr.write(options.getUsage())
raise SystemExit(1)
container_deployment = ClusterContainerDeployment(reactor,
environ,
options)
return container_deployment.deploy_and_wait_for_creation()
def main(reactor, args, base_path, top_level):
try:
options = TestBrewOptions()
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("Error: {error}.\n".format(error=str(e)))
sys.exit(1)
add_destination(eliot_output)
recipe_url = options['recipe_url']
options['vmpath'] = FilePath(options['vmpath'])
# Open the recipe URL just to validate and verify that it exists.
# We do not need to read its content.
urllib2.urlopen(recipe_url)
yield run(reactor, [
"vmrun", "revertToSnapshot",
options['vmpath'].path, options['vmsnapshot'],
])
yield run(reactor, [
"vmrun", "start", options['vmpath'].path, "nogui",
])
yield perform(
make_dispatcher(reactor),
run_remotely(
username=options['vmuser'],
address=options['vmhost'],
commands=sequence([
task_configure_brew_path(),
task_test_homebrew(recipe_url),
]),
),
)
yield run(reactor, [
"vmrun", "stop", options['vmpath'].path, "hard",
])
print "Done."
except ProcessTerminated as e:
sys.stderr.write(
(
"Error: Command terminated with exit status {code}.\n"
).format(code=e.exitCode)
)
raise
def test_tree(self):
"""
``_eliottree`` returns a ``bytes`` string containing a rendered tree of
Eliot actions and messages.
"""
events = []
add_destination(events.append)
self.addCleanup(lambda: remove_destination(events.append))
with start_action(action_type=u"foo"):
pass
# I don't know exactly what the tree rendering looks like. That's why
# I'm using eliot-tree! So this assertion is sort of lame.
self.assertThat(
_eliottree(events).decode("utf-8"),
Contains(u"foo@1/started"),
)
def from_content(content, log=None):
""" Load, validate and process blocks """
if not content:
raise ExploException('no exploitation content')
try:
blocks = load_blocks(content)
except yaml.YAMLError as err:
raise ExploException('error parsing document: %s' % err)
if not validate_blocks(blocks):
raise ExploException('error parsing document:' \
'not all blocks specify the required fields %s' % FIELDS_REQUIRED)
if log:
add_destination(log)
return process_blocks(blocks)
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
from flocker.common.script import eliot_logging_service
log_file = open("%s.log" % base_path.basename(), "a")
log_writer = eliot_logging_service(log_file=log_file,
reactor=reactor,
capture_stdout=False)
log_writer.startService()
reactor.addSystemEventTrigger('before', 'shutdown', log_writer.stopService)
try:
nodes = yield runner.start_nodes(reactor, node_count=1)
yield perform(make_dispatcher(reactor),
install_cli(runner.package_source, nodes[0]))
result = yield run_client_tests(reactor=reactor, node=nodes[0])
except:
result = 1
raise
finally:
# Unless the tests failed, and the user asked to keep the nodes, we
# delete them.
if not (result != 0 and options['keep']):
runner.stop_nodes(reactor)
elif options['keep']:
print "--keep specified, not destroying nodes."
raise SystemExit(result)
def main(reactor, argv, environ):
# Setup eliot to print better human-readable output to standard
# output
add_destination(eliot_output)
try:
options = ContainerOptions()
options.parseOptions(argv[1:])
except usage.UsageError as e:
sys.stderr.write(e.args[0])
sys.stderr.write("\n\n")
sys.stderr.write(options.getSynopsis())
sys.stderr.write("\n")
sys.stderr.write(options.getUsage())
raise SystemExit(1)
container_deployment = ClusterContainerDeployment.from_options(reactor, options)
def deploy_and_wait(cluster_container_deployment):
return cluster_container_deployment.deploy_and_wait_for_creation()
container_deployment.addCallback(deploy_and_wait)
return container_deployment
def eliot_to_stdout(message_formats, action_formats, stdout=sys.stdout):
"""
Write pretty versions of eliot log messages to stdout.
"""
def eliot_output(message):
message_type = message.get('message_type')
action_type = message.get('action_type')
action_status = message.get('action_status')
message_format = '%s'
if message_type is not None:
if message_type == 'twisted:log' and message.get('error'):
message_format = '%(message)s'
else:
message_format = message_formats.get(message_type, '%s')
elif action_type is not None:
if action_status == 'started':
message_format = action_formats.get('action_type', '%s')
# We don't consider other status, since we
# have no meaningful messages to write.
stdout.write(message_format % message)
stdout.flush()
add_destination(eliot_output)
def eliot_to_stdout(message_formats, action_formats, stdout=sys.stdout):
"""
Write pretty versions of eliot log messages to stdout.
"""
def eliot_output(message):
message_type = message.get('message_type')
action_type = message.get('action_type')
action_status = message.get('action_status')
message_format = ''
if message_type is not None:
if message_type == 'twisted:log' and message.get('error'):
message_format = '%(message)s'
else:
message_format = message_formats.get(message_type, '')
elif action_type is not None:
if action_status == 'started':
message_format = action_formats.get('action_type', '')
# We don't consider other status, since we
# have no meaningful messages to write.
stdout.write(message_format % message)
stdout.flush()
add_destination(eliot_output)
def onOpen(self):
"""
WebSocket callback
"""
eliot.add_destination(self._received_eliot_log)
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
from flocker.common.script import eliot_logging_service
log_writer = eliot_logging_service(
destination=FileDestination(
file=open("%s.log" % (base_path.basename(),), "a")
),
reactor=reactor,
capture_stdout=False)
log_writer.startService()
reactor.addSystemEventTrigger(
'before', 'shutdown', log_writer.stopService)
cluster = None
results = []
setup_succeeded = False
reached_finally = False
def cluster_cleanup():
if not reached_finally:
print "interrupted..."
print "stopping cluster"
return runner.stop_cluster(reactor)
cleanup_trigger_id = reactor.addSystemEventTrigger('before', 'shutdown',
cluster_cleanup)
try:
yield runner.ensure_keys(reactor)
cluster = yield runner.start_cluster(reactor)
if options['distribution'] in ('centos-7',):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
results.append(capture_journal(reactor,
node.address,
remote_logs_file)
)
elif options['distribution'] in ('ubuntu-14.04',):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
results.append(capture_upstart(reactor,
node.address,
remote_logs_file)
)
gather_deferreds(results)
if not options["no-pull"]:
yield perform(
make_dispatcher(reactor),
parallel([
run_remotely(
username='******',
address=node.address,
commands=task_pull_docker_images()
) for node in cluster.agent_nodes
]),
)
setup_succeeded = True
result = yield run_tests(
reactor=reactor,
cluster=cluster,
trial_args=options['trial-args'])
finally:
reached_finally = True
# We delete the nodes if the user hasn't asked to keep them
# or if we failed to provision the cluster.
if not setup_succeeded:
print "cluster provisioning failed"
elif not options['keep']:
print "not keeping cluster"
else:
print "--keep specified, not destroying nodes."
print ("To run acceptance tests against these nodes, "
"set the following environment variables: ")
environment_variables = get_trial_environment(cluster)
for environment_variable in environment_variables:
print "export {name}={value};".format(
name=environment_variable,
value=shell_quote(
environment_variables[environment_variable]),
)
reactor.removeSystemEventTrigger(cleanup_trigger_id)
raise SystemExit(result)
"""
Write some logs to journald.
"""
from __future__ import print_function
from eliot import Message, start_action, add_destination
from eliot.journald import JournaldDestination
add_destination(JournaldDestination())
def divide(a, b):
with start_action(action_type="divide", a=a, b=b):
return a / b
print(divide(10, 2))
Message.log(message_type="inbetween")
print(divide(10, 0))
"""
Write some logs to journald.
"""
from __future__ import print_function
from eliot import Message, start_action, add_destination
from eliot.journald import JournaldDestination
add_destination(JournaldDestination())
def divide(a, b):
with start_action(action_type="divide", a=a, b=b):
return a / b
print(divide(10, 2))
Message.log(message_type="inbetween")
print(divide(10, 0))
parser.add_argument(
'database',
nargs='?',
help='The directory in which to create or open the database. If not'
' supplied, a temporary database will be used')
parser.add_argument('--debug',
action='store_true',
help='Enable pudb post mortem debugging')
parser.add_argument('--port',
type=int,
default=8034,
help='Port to listen for connections on')
parser.add_argument('--host', default='0.0.0.0', help='Host to bind to')
args = parser.parse_args()
eliot.add_destination(eliot.FileDestination(sys.stderr))
dbdir = args.database or tempfile.mkdtemp()
try:
with Database(dbdir) as db:
the_app = app(db)
if args.debug:
the_app = debug_wrapper(the_app)
httpd = Server((args.host, args.port), the_app)
print('Serving "{dbdir}" on {args.host}:{args.port}'.format(
dbdir=dbdir, args=args))
httpd.safe_start()
finally:
if not args.database:
shutil.rmtree(dbdir)
elif os.getenv(_DB_DIR_ENV_VAR):
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the Flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
def cluster_cleanup():
print("stopping cluster")
return runner.stop_cluster(reactor)
cleanup_trigger_id = reactor.addSystemEventTrigger('before', 'shutdown',
cluster_cleanup)
from flocker.common.script import eliot_logging_service
log_writer = eliot_logging_service(
destination=FileDestination(
file=open("%s.log" % (base_path.basename(),), "a")
),
reactor=reactor,
capture_stdout=False)
log_writer.startService()
reactor.addSystemEventTrigger(
'before', 'shutdown', log_writer.stopService)
yield runner.ensure_keys(reactor)
cluster = yield runner.start_cluster(reactor)
managed_config_file = options['cert-directory'].child("managed.yaml")
managed_config = create_managed_config(options['config'], cluster)
managed_config_file.setContent(
yaml.safe_dump(managed_config, default_flow_style=False)
)
if options['distribution'] in ('centos-7',):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
capture_journal(reactor, node.address,
remote_logs_file).addErrback(write_failure)
elif options['distribution'] in ('ubuntu-14.04', 'ubuntu-15.10'):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
capture_upstart(reactor, node.address,
remote_logs_file).addErrback(write_failure)
flocker_client = _make_client(reactor, cluster)
yield _wait_for_nodes(reactor, flocker_client, len(cluster.agent_nodes))
if options['no-keep']:
print("not keeping cluster")
else:
environment_variables = get_trial_environment(cluster)
environment_strings = list()
for environment_variable in environment_variables:
environment_strings.append(
"export {name}={value};\n".format(
name=environment_variable,
value=shell_quote(
environment_variables[environment_variable]
),
)
)
environment = ''.join(environment_strings)
print("The following variables describe the cluster:")
print(environment)
env_file = options['cert-directory'].child("environment.env")
env_file.setContent(environment)
print("The variables are also saved in {}".format(
env_file.path
))
print("Be sure to preserve the required files.")
reactor.removeSystemEventTrigger(cleanup_trigger_id)
def eliot_to_twisted_logging():
"""
Ship eliot logs to twisted.
"""
add_destination(_destination)
def __enter__(self):
eliot.add_destination(self)
return self
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the Flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
from flocker.common.script import eliot_logging_service
log_writer = eliot_logging_service(
destination=FileDestination(
file=open("%s.log" % (base_path.basename(),), "a")
),
reactor=reactor,
capture_stdout=False)
log_writer.startService()
reactor.addSystemEventTrigger(
'before', 'shutdown', log_writer.stopService)
cluster = None
results = []
try:
yield runner.ensure_keys(reactor)
cluster = yield runner.start_cluster(reactor)
if options['distribution'] in ('centos-7',):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
results.append(capture_journal(reactor,
node.address,
remote_logs_file)
)
elif options['distribution'] in ('ubuntu-14.04', 'ubuntu-15.10'):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
results.append(capture_upstart(reactor,
node.address,
remote_logs_file)
)
gather_deferreds(results)
if options['apps-per-node'] > 0:
config = _build_config(cluster, options['template'],
options['apps-per-node'])
yield _configure(reactor, cluster, config)
result = 0
except BaseException:
result = 1
raise
finally:
if options['no-keep'] or result == 1:
runner.stop_cluster(reactor)
else:
if cluster is None:
print("Didn't finish creating the cluster.")
runner.stop_cluster(reactor)
else:
print("The following variables describe the cluster:")
environment_variables = get_trial_environment(cluster)
for environment_variable in environment_variables:
print("export {name}={value};".format(
name=environment_variable,
value=shell_quote(
environment_variables[environment_variable]),
))
print("Be sure to preserve the required files.")
raise SystemExit(result)
from sphinxmixcrypto import PacketReplayCacheDict, GroupCurve25519, SphinxParams, SECURITY_PARAMETER
from sphinxmixcrypto import IReader, IKeyState
from txmix.interfaces import IMixTransport
from txmix.mix import ThresholdMixNode
from txmix.client import MixClient, RandomRouteFactory
from txmix.utils import DummyPKI
# tell eliot to log a line of json for each message to stdout
def stdout(message):
sys.stdout.write(json.dumps(message) + "\n")
add_destination(stdout)
def generate_node_id(rand_reader):
idnum = rand_reader.read(4)
node_id = b"\xff" + idnum + (b"\x00" * (SECURITY_PARAMETER - len(idnum) - 1))
return node_id
def generate_node_keypair(rand_reader):
group = GroupCurve25519()
private_key = group.gensecret(rand_reader)
public_key = group.expon(group.generator, private_key)
return public_key, private_key
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
from flocker.common.script import eliot_logging_service
log_file = open("%s.log" % base_path.basename(), "a")
log_writer = eliot_logging_service(log_file=log_file,
reactor=reactor,
capture_stdout=False)
log_writer.startService()
reactor.addSystemEventTrigger('before', 'shutdown', log_writer.stopService)
cluster = None
try:
cluster = yield runner.start_cluster(reactor)
if options['distribution'] in ('centos-7', ):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
capture_journal(reactor, node.address, remote_logs_file)
if not options["no-pull"]:
yield perform(
make_dispatcher(reactor),
parallel([
run_remotely(username='******',
address=node.address,
commands=task_pull_docker_images())
for node in cluster.agent_nodes
]),
)
result = yield run_tests(reactor=reactor,
cluster=cluster,
trial_args=options['trial-args'])
except:
result = 1
raise
finally:
# Unless the tests failed, and the user asked to keep the nodes, we
# delete them.
if not options['keep']:
runner.stop_cluster(reactor)
else:
print "--keep specified, not destroying nodes."
if cluster is None:
print("Didn't finish creating the cluster.")
else:
print(
"To run acceptance tests against these nodes, "
"set the following environment variables: ")
environment_variables = get_trial_environment(cluster)
for environment_variable in environment_variables:
print "export {name}={value};".format(
name=environment_variable,
value=shell_quote(
environment_variables[environment_variable]),
)
raise SystemExit(result)
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the Flocker repository.
"""
add_destination(eliot_output)
options = RunOptions(top_level=top_level)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
sys.stderr.write("\n")
sys.stderr.write(str(options))
raise SystemExit(1)
# Existing nodes must be described in a managed section
# of the configuration.
existing_nodes = make_managed_nodes(
options['config']['managed']['addresses'],
options['distribution'],
)
# The following code assumes that one of the managed nodes
# is both a control node and an agent node.
[control_node] = [
node for node in existing_nodes
if node.address == options['control-node']
]
dataset_backend_config_file = save_backend_configuration(
options.dataset_backend(),
options.dataset_backend_configuration(),
)
cluster = Cluster(
all_nodes=list(existing_nodes),
control_node=control_node,
agent_nodes=list(existing_nodes),
dataset_backend=options.dataset_backend(),
default_volume_size=get_default_volume_size(
options.dataset_backend_configuration()
),
certificates=Certificates(options['cert-directory']),
dataset_backend_config_file=dataset_backend_config_file,
)
flocker_client = make_client(reactor, cluster)
existing_count = len(existing_nodes)
yield wait_for_nodes(reactor, flocker_client, existing_count)
if options['starting-index'] is None:
options['starting-index'] = existing_count
print(
"Adding {} node(s) to the cluster of {} nodes "
"starting at index {}".format(
options['number-of-nodes'],
existing_count,
options['starting-index'],
)
)
runner = options.runner
cleanup_id = reactor.addSystemEventTrigger('before', 'shutdown',
runner.stop_cluster, reactor)
from flocker.common.script import eliot_logging_service
log_writer = eliot_logging_service(
destination=FileDestination(
file=open("%s.log" % (base_path.basename(),), "a")
),
reactor=reactor,
capture_stdout=False)
log_writer.startService()
reactor.addSystemEventTrigger(
'before', 'shutdown', log_writer.stopService)
control_node = options['control-node']
if options['distribution'] in ('centos-7',):
remote_logs_file = open("remote_logs.log", "a")
capture_journal(reactor, control_node, remote_logs_file)
elif options['distribution'] in ('ubuntu-14.04', 'ubuntu-15.10'):
remote_logs_file = open("remote_logs.log", "a")
capture_upstart(reactor, control_node, remote_logs_file)
yield runner.ensure_keys(reactor)
deferreds = runner.extend_cluster(
reactor,
cluster,
options['number-of-nodes'],
options['tag'],
options['starting-index'],
)
results = yield DeferredList(deferreds)
failed_count = 0
for (success, value) in results:
if not success:
failed_count = failed_count + 1
if failed_count:
print "Failed to create {} nodes, see logs.".format(failed_count)
yield wait_for_nodes(
reactor,
flocker_client,
len(cluster.agent_nodes),
)
save_managed_config(options['cert-directory'], options['config'], cluster)
save_environment(options['cert-directory'], cluster)
reactor.removeSystemEventTrigger(cleanup_id)
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the Flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
from flocker.common.script import eliot_logging_service
log_writer = eliot_logging_service(destination=FileDestination(
file=open("%s.log" % (base_path.basename(), ), "a")),
reactor=reactor,
capture_stdout=False)
log_writer.startService()
reactor.addSystemEventTrigger('before', 'shutdown', log_writer.stopService)
cluster = None
results = []
try:
yield runner.ensure_keys(reactor)
cluster = yield runner.start_cluster(reactor)
if options['distribution'] in ('centos-7', ):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
results.append(
capture_journal(reactor, node.address, remote_logs_file))
elif options['distribution'] in ('ubuntu-14.04', 'ubuntu-15.10'):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
results.append(
capture_upstart(reactor, node.address, remote_logs_file))
gather_deferreds(results)
if options['apps-per-node'] > 0:
config = _build_config(cluster, options['template'],
options['apps-per-node'])
yield _configure(reactor, cluster, config)
result = 0
except BaseException:
result = 1
raise
finally:
if options['no-keep'] or result == 1:
runner.stop_cluster(reactor)
else:
if cluster is None:
print("Didn't finish creating the cluster.")
runner.stop_cluster(reactor)
else:
print("The following variables describe the cluster:")
environment_variables = get_trial_environment(cluster)
for environment_variable in environment_variables:
print("export {name}={value};".format(
name=environment_variable,
value=shell_quote(
environment_variables[environment_variable]),
))
print("Be sure to preserve the required files.")
raise SystemExit(result)
def onOpen(self):
"""
WebSocket callback
"""
eliot.add_destination(self._received_eliot_log)
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
from flocker.common.script import eliot_logging_service
log_writer = eliot_logging_service(
destination=FileDestination(
file=open("%s.log" % (base_path.basename(),), "a")
),
reactor=reactor,
capture_stdout=False)
log_writer.startService()
reactor.addSystemEventTrigger(
'before', 'shutdown', log_writer.stopService)
cluster = None
try:
yield runner.ensure_keys(reactor)
cluster = yield runner.start_cluster(reactor)
if options['distribution'] in ('centos-7',):
remote_logs_file = open("remote_logs.log", "a")
for node in cluster.all_nodes:
capture_journal(reactor, node.address, remote_logs_file)
if not options["no-pull"]:
yield perform(
make_dispatcher(reactor),
parallel([
run_remotely(
username='******',
address=node.address,
commands=task_pull_docker_images()
) for node in cluster.agent_nodes
]),
)
result = yield run_tests(
reactor=reactor,
cluster=cluster,
trial_args=options['trial-args'])
except:
result = 1
raise
finally:
# Unless the tests failed, and the user asked to keep the nodes, we
# delete them.
if not options['keep']:
runner.stop_cluster(reactor)
else:
print "--keep specified, not destroying nodes."
if cluster is None:
print ("Didn't finish creating the cluster.")
else:
print ("To run acceptance tests against these nodes, "
"set the following environment variables: ")
environment_variables = get_trial_environment(cluster)
for environment_variable in environment_variables:
print "export {name}={value};".format(
name=environment_variable,
value=shell_quote(
environment_variables[environment_variable]),
)
raise SystemExit(result)
def main(reactor, args, base_path, top_level):
"""
:param reactor: Reactor to use.
:param list args: The arguments passed to the script.
:param FilePath base_path: The executable being run.
:param FilePath top_level: The top-level of the flocker repository.
"""
options = RunOptions(top_level=top_level)
add_destination(eliot_output)
try:
options.parseOptions(args)
except UsageError as e:
sys.stderr.write("%s: %s\n" % (base_path.basename(), e))
raise SystemExit(1)
runner = options.runner
try:
nodes = yield runner.start_nodes(reactor)
ca_directory = FilePath(mkdtemp())
print("Generating certificates in: {}".format(ca_directory.path))
certificates = Certificates.generate(ca_directory, nodes[0].address,
len(nodes))
yield perform(
make_dispatcher(reactor),
parallel([
run_remotely(
username='******',
address=node.address,
commands=task_pull_docker_images()
) for node in nodes
]),
)
control_node = nodes[0]
dataset_backend = options.dataset_backend
yield perform(
make_dispatcher(reactor),
configure_cluster(control_node=control_node, agent_nodes=nodes,
certificates=certificates,
dataset_backend=dataset_backend))
result = yield run_tests(
reactor=reactor,
nodes=nodes,
control_node=control_node,
agent_nodes=nodes,
dataset_backend=dataset_backend,
trial_args=options['trial-args'],
certificates_path=ca_directory)
except:
result = 1
raise
finally:
# Unless the tests failed, and the user asked to keep the nodes, we
# delete them.
if not (result != 0 and options['keep']):
runner.stop_nodes(reactor)
elif options['keep']:
print "--keep specified, not destroying nodes."
print ("To run acceptance tests against these nodes, "
"set the following environment variables: ")
environment_variables = {
'FLOCKER_ACCEPTANCE_NODES':
':'.join(node.address for node in nodes),
'FLOCKER_ACCEPTANCE_CONTROL_NODE': control_node.address,
'FLOCKER_ACCEPTANCE_AGENT_NODES':
':'.join(node.address for node in nodes),
'FLOCKER_ACCEPTANCE_VOLUME_BACKEND': dataset_backend.name,
'FLOCKER_ACCEPTANCE_API_CERTIFICATES_PATH': ca_directory.path,
}
for environment_variable in environment_variables:
print "export {name}={value};".format(
name=environment_variable,
value=environment_variables[environment_variable],
)
raise SystemExit(result)
