def test_prints_when_isnotquiet_and_randomized_assume_tty_or_istty(self, patched_isatty, patched_flush, seed):
random.seed(seed)
random_boolean = random.choice([True, False])
console.init(quiet=False, assume_tty=random_boolean)
console.RALLY_RUNNING_IN_DOCKER = False
patched_isatty.return_value = not random_boolean
message = "Unit test message"
width = random.randint(20, 140)
mock_printer = mock.Mock()
progress_reporter = console.CmdLineProgressReporter(width=width, printer=mock_printer)
progress_reporter.print(message=message, progress=".")
mock_printer.assert_has_calls([
mock.call(" " * width, end=""),
mock.call("\x1b[{}D{}{}.".format(width, message, " "*(width-len(message)-1)), end="")
])
patched_flush.assert_called_once_with()
def main():
check_python_version()
console.init()
parser = argparse.ArgumentParser(
prog=PROGRAM_NAME,
description=BANNER + "\n\n Rally daemon to support remote benchmarks",
epilog="Find out more about Rally at %s" %
console.format.link(DOC_LINK),
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--version',
action='version',
version="%(prog)s " + version.version())
subparsers = parser.add_subparsers(title="subcommands",
dest="subcommand",
help="")
subparsers.required = True
start_command = subparsers.add_parser("start",
help="Starts the Rally daemon")
restart_command = subparsers.add_parser("restart",
help="Restarts the Rally daemon")
for p in [start_command, restart_command]:
p.add_argument("--node-ip", help="The IP of this node.")
p.add_argument("--coordinator-ip",
help="The IP of the coordinator node.")
subparsers.add_parser("stop", help="Stops the Rally daemon")
args = parser.parse_args()
if args.subcommand == "start":
start(args)
elif args.subcommand == "stop":
stop()
elif args.subcommand == "restart":
stop(raise_errors=False)
start(args)
else:
raise exceptions.RallyError("Unknown subcommand [%s]" %
args.subcommand)
def main():
check_python_version()
console.init()
parser = argparse.ArgumentParser(prog=PROGRAM_NAME,
description=BANNER + "\n\n Rally daemon to support remote benchmarks",
epilog="Find out more about Rally at %s" % console.format.link(DOC_LINK),
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--version', action='version', version="%(prog)s " + version.version())
subparsers = parser.add_subparsers(
title="subcommands",
dest="subcommand",
help="")
subparsers.required = True
start_command = subparsers.add_parser("start", help="Starts the Rally daemon")
restart_command = subparsers.add_parser("restart", help="Restarts the Rally daemon")
for p in [start_command, restart_command]:
p.add_argument(
"--node-ip",
help="The IP of this node.")
p.add_argument(
"--coordinator-ip",
help="The IP of the coordinator node."
)
subparsers.add_parser("stop", help="Stops the Rally daemon")
args = parser.parse_args()
if args.subcommand == "start":
start(args)
elif args.subcommand == "stop":
stop()
elif args.subcommand == "restart":
stop(raise_errors=False)
start(args)
else:
raise exceptions.RallyError("Unknown subcommand [%s]" % args.subcommand)
def main():
check_python_version()
log.install_default_log_config()
log.configure_logging()
logger = logging.getLogger(__name__)
start = time.time()
# Early init of console output so we start to show everything consistently.
console.init(quiet=False)
arg_parser = create_arg_parser()
args = arg_parser.parse_args()
console.init(quiet=args.quiet)
console.println(BANNER)
cfg = config.Config(config_name=args.configuration_name)
sub_command = derive_sub_command(args, cfg)
ensure_configuration_present(cfg, args, sub_command)
if args.effective_start_date:
cfg.add(config.Scope.application, "system", "time.start",
args.effective_start_date)
cfg.add(config.Scope.application, "system", "time.start.user_provided",
True)
else:
cfg.add(config.Scope.application, "system", "time.start",
datetime.datetime.utcnow())
cfg.add(config.Scope.application, "system", "time.start.user_provided",
False)
cfg.add(config.Scope.applicationOverride, "system", "trial.id",
str(uuid.uuid4()))
cfg.add(config.Scope.applicationOverride, "system", "quiet.mode",
args.quiet)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
args.offline)
# Local config per node
cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root())
cfg.add(config.Scope.application, "node", "rally.cwd", os.getcwd())
cfg.add(config.Scope.applicationOverride, "mechanic", "source.revision",
args.revision)
if args.distribution_version:
cfg.add(config.Scope.applicationOverride, "mechanic",
"distribution.version", args.distribution_version)
cfg.add(config.Scope.applicationOverride, "mechanic",
"distribution.repository", args.distribution_repository)
cfg.add(config.Scope.applicationOverride, "mechanic", "car.names",
opts.csv_to_list(args.car))
if args.team_path:
cfg.add(config.Scope.applicationOverride, "mechanic", "team.path",
os.path.abspath(io.normalize_path(args.team_path)))
cfg.add(config.Scope.applicationOverride, "mechanic",
"repository.name", None)
else:
cfg.add(config.Scope.applicationOverride, "mechanic",
"repository.name", args.team_repository)
cfg.add(config.Scope.applicationOverride, "mechanic", "car.plugins",
opts.csv_to_list(args.elasticsearch_plugins))
cfg.add(config.Scope.applicationOverride, "mechanic", "car.params",
opts.to_dict(args.car_params))
cfg.add(config.Scope.applicationOverride, "mechanic", "plugin.params",
opts.to_dict(args.plugin_params))
if args.keep_cluster_running:
cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running",
True)
# force-preserve the cluster nodes.
cfg.add(config.Scope.applicationOverride, "mechanic",
"preserve.install", True)
else:
cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running",
False)
cfg.add(config.Scope.applicationOverride, "mechanic",
"preserve.install", convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "mechanic", "runtime.jdk",
args.runtime_jdk)
cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.devices",
opts.csv_to_list(args.telemetry))
cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.params",
opts.to_dict(args.telemetry_params))
cfg.add(config.Scope.applicationOverride, "race", "pipeline",
args.pipeline)
cfg.add(config.Scope.applicationOverride, "race", "laps", args.laps)
cfg.add(config.Scope.applicationOverride, "race", "user.tag",
args.user_tag)
# We can assume here that if a track-path is given, the user did not specify a repository either (although argparse sets it to
# its default value)
if args.track_path:
cfg.add(config.Scope.applicationOverride, "track", "track.path",
os.path.abspath(io.normalize_path(args.track_path)))
cfg.add(config.Scope.applicationOverride, "track", "repository.name",
None)
if args.track:
# stay as close as possible to argparse errors although we have a custom validation.
arg_parser.error(
"argument --track not allowed with argument --track-path")
# cfg.add(config.Scope.applicationOverride, "track", "track.name", None)
else:
# cfg.add(config.Scope.applicationOverride, "track", "track.path", None)
cfg.add(config.Scope.applicationOverride, "track", "repository.name",
args.track_repository)
# set the default programmatically because we need to determine whether the user has provided a value
chosen_track = args.track if args.track else "geonames"
cfg.add(config.Scope.applicationOverride, "track", "track.name",
chosen_track)
cfg.add(config.Scope.applicationOverride, "track", "params",
opts.to_dict(args.track_params))
cfg.add(config.Scope.applicationOverride, "track", "challenge.name",
args.challenge)
cfg.add(config.Scope.applicationOverride, "track", "include.tasks",
opts.csv_to_list(args.include_tasks))
cfg.add(config.Scope.applicationOverride, "track", "test.mode.enabled",
args.test_mode)
cfg.add(config.Scope.applicationOverride, "reporting", "format",
args.report_format)
cfg.add(config.Scope.applicationOverride, "reporting", "values",
args.show_in_report)
cfg.add(config.Scope.applicationOverride, "reporting", "output.path",
args.report_file)
if sub_command == "compare":
cfg.add(config.Scope.applicationOverride, "reporting",
"baseline.timestamp", args.baseline)
cfg.add(config.Scope.applicationOverride, "reporting",
"contender.timestamp", args.contender)
if sub_command == "generate":
cfg.add(config.Scope.applicationOverride, "generator", "chart.type",
args.chart_type)
cfg.add(config.Scope.applicationOverride, "generator", "output.path",
args.output_path)
if args.chart_spec_path and (args.track or args.challenge or args.car
or args.node_count):
console.println(
"You need to specify either --chart-spec-path or --track, --challenge, --car and "
"--node-count but not both.")
exit(1)
if args.chart_spec_path:
cfg.add(config.Scope.applicationOverride, "generator",
"chart.spec.path", args.chart_spec_path)
else:
# other options are stored elsewhere already
cfg.add(config.Scope.applicationOverride, "generator",
"node.count", args.node_count)
cfg.add(config.Scope.applicationOverride, "driver", "profiling",
args.enable_driver_profiling)
cfg.add(config.Scope.applicationOverride, "driver", "on.error",
args.on_error)
cfg.add(config.Scope.applicationOverride, "driver", "load_driver_hosts",
opts.csv_to_list(args.load_driver_hosts))
if sub_command != "list":
# Also needed by mechanic (-> telemetry) - duplicate by module?
target_hosts = opts.TargetHosts(args.target_hosts)
cfg.add(config.Scope.applicationOverride, "client", "hosts",
target_hosts)
client_options = opts.ClientOptions(args.client_options,
target_hosts=target_hosts)
cfg.add(config.Scope.applicationOverride, "client", "options",
client_options)
if "timeout" not in client_options.default:
console.info(
"You did not provide an explicit timeout in the client options. Assuming default of 10 seconds."
)
if list(target_hosts.all_hosts) != list(
client_options.all_client_options):
console.println(
"--target-hosts and --client-options must define the same keys for multi cluster setups."
)
exit(1)
# split by component?
if sub_command == "list":
cfg.add(config.Scope.applicationOverride, "system",
"list.config.option", args.configuration)
cfg.add(config.Scope.applicationOverride, "system",
"list.races.max_results", args.limit)
logger.info("OS [%s]", str(os.uname()))
logger.info("Python [%s]", str(sys.implementation))
logger.info("Rally version [%s]", version.version())
logger.info("Command line arguments: %s", args)
# Configure networking
net.init()
if not args.offline:
if not net.has_internet_connection():
console.warn(
"No Internet connection detected. Automatic download of track data sets etc. is disabled.",
logger=logger)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
True)
else:
logger.info("Detected a working Internet connection.")
success = dispatch_sub_command(cfg, sub_command)
end = time.time()
if success:
console.println("")
console.info("SUCCESS (took %d seconds)" % (end - start),
overline="-",
underline="-")
else:
console.println("")
console.info("FAILURE (took %d seconds)" % (end - start),
overline="-",
underline="-")
sys.exit(64)
def test_global_rally_running_in_docker_is_true(self):
console.init()
self.assertEqual(True, console.RALLY_RUNNING_IN_DOCKER)
def test_global_rally_running_in_docker_is_false_if_unset(self):
console.init()
self.assertEqual(False, console.RALLY_RUNNING_IN_DOCKER)
def main():
check_python_version()
start = time.time()
# Early init of console output so we start to show everything consistently.
console.init(quiet=False)
# allow to see a thread-dump on SIGQUIT
faulthandler.register(signal.SIGQUIT, file=sys.stderr)
pre_configure_logging()
args = parse_args()
console.init(quiet=args.quiet)
console.println(BANNER)
cfg = config.Config(config_name=args.configuration_name)
sub_command = derive_sub_command(args, cfg)
ensure_configuration_present(cfg, args, sub_command)
if args.effective_start_date:
cfg.add(config.Scope.application, "system", "time.start",
args.effective_start_date)
cfg.add(config.Scope.application, "system", "time.start.user_provided",
True)
else:
cfg.add(config.Scope.application, "system", "time.start",
datetime.datetime.utcnow())
cfg.add(config.Scope.application, "system", "time.start.user_provided",
False)
cfg.add(config.Scope.applicationOverride, "system", "quiet.mode",
args.quiet)
# per node?
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
args.offline)
cfg.add(config.Scope.applicationOverride, "system", "logging.output",
args.logging)
# only temporary to ignore unknown actor messages
cfg.add(config.Scope.applicationOverride, "system",
"ignore.unknown.return", args.ignore_unknown_return_values)
# Local config per node
cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root())
cfg.add(config.Scope.application, "node", "rally.cwd", os.getcwd())
cfg.add(config.Scope.applicationOverride, "mechanic", "source.revision",
args.revision)
#TODO dm: Consider renaming this one. It's used by different modules
if args.distribution_version:
cfg.add(config.Scope.applicationOverride, "mechanic",
"distribution.version", args.distribution_version)
cfg.add(config.Scope.applicationOverride, "mechanic",
"distribution.repository", args.distribution_repository)
cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name",
args.team_repository)
cfg.add(config.Scope.applicationOverride, "mechanic", "car.name", args.car)
cfg.add(config.Scope.applicationOverride, "mechanic", "car.plugins",
csv_to_list(args.elasticsearch_plugins))
cfg.add(config.Scope.applicationOverride, "mechanic", "node.datapaths",
csv_to_list(args.data_paths))
cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install",
convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.devices",
csv_to_list(args.telemetry))
if args.override_src_dir is not None:
cfg.add(config.Scope.applicationOverride, "source", "local.src.dir",
args.override_src_dir)
cfg.add(config.Scope.applicationOverride, "race", "pipeline",
args.pipeline)
cfg.add(config.Scope.applicationOverride, "race", "laps", args.laps)
cfg.add(config.Scope.applicationOverride, "race", "user.tag",
args.user_tag)
cfg.add(config.Scope.applicationOverride, "track", "repository.name",
args.track_repository)
cfg.add(config.Scope.applicationOverride, "track", "track.name",
args.track)
cfg.add(config.Scope.applicationOverride, "track", "challenge.name",
args.challenge)
cfg.add(config.Scope.applicationOverride, "track", "test.mode.enabled",
args.test_mode)
cfg.add(config.Scope.applicationOverride, "track", "auto_manage_indices",
to_bool(args.auto_manage_indices))
cfg.add(config.Scope.applicationOverride, "reporting", "format",
args.report_format)
cfg.add(config.Scope.applicationOverride, "reporting", "output.path",
args.report_file)
if sub_command == "compare":
cfg.add(config.Scope.applicationOverride, "reporting",
"baseline.timestamp", args.baseline)
cfg.add(config.Scope.applicationOverride, "reporting",
"contender.timestamp", args.contender)
################################
# new section name: driver
################################
cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health",
args.cluster_health)
cfg.add(config.Scope.applicationOverride, "driver", "profiling",
args.enable_driver_profiling)
if sub_command != "list":
# Also needed by mechanic (-> telemetry) - duplicate by module?
cfg.add(config.Scope.applicationOverride, "client", "hosts",
_normalize_hosts(csv_to_list(args.target_hosts)))
client_options = kv_to_map(csv_to_list(args.client_options))
cfg.add(config.Scope.applicationOverride, "client", "options",
client_options)
if "timeout" not in client_options:
console.info(
"You did not provide an explicit timeout in the client options. Assuming default of 10 seconds."
)
# split by component?
if sub_command == "list":
cfg.add(config.Scope.applicationOverride, "system",
"list.config.option", args.configuration)
cfg.add(config.Scope.applicationOverride, "system",
"list.races.max_results", args.limit)
configure_logging(cfg)
logger.info("OS [%s]" % str(os.uname()))
logger.info("Python [%s]" % str(sys.implementation))
logger.info("Rally version [%s]" % version.version())
logger.info("Command line arguments: %s" % args)
# Configure networking
net.init()
if not args.offline:
if not net.has_internet_connection():
console.warn(
"No Internet connection detected. Automatic download of track data sets etc. is disabled.",
logger=logger)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
True)
else:
logger.info("Detected a working Internet connection.")
# Kill any lingering Rally processes before attempting to continue - the actor system needs to be a singleton on this machine
# noinspection PyBroadException
try:
process.kill_running_rally_instances()
except BaseException:
logger.exception(
"Could not terminate potentially running Rally instances correctly. Attempting to go on anyway."
)
success = dispatch_sub_command(cfg, sub_command)
end = time.time()
if success:
console.println("")
console.info("SUCCESS (took %d seconds)" % (end - start),
overline="-",
underline="-")
else:
console.println("")
console.info("FAILURE (took %d seconds)" % (end - start),
overline="-",
underline="-")
sys.exit(64)
def main():
start = time.time()
# Early init of console output so we start to show everything consistently.
console.init(quiet=False)
pre_configure_logging()
args = parse_args()
console.init(quiet=args.quiet)
console.println(BANNER)
cfg = config.Config(config_name=args.configuration_name)
sub_command = derive_sub_command(args, cfg)
ensure_configuration_present(cfg, args, sub_command)
# Add global meta info derived by rally itself
cfg.add(config.Scope.application, "meta", "time.start",
args.effective_start_date)
cfg.add(config.Scope.application, "system", "rally.root",
rally_root_path())
cfg.add(config.Scope.application, "system", "invocation.root.dir",
paths.Paths(cfg).invocation_root())
# Add command line config
cfg.add(config.Scope.applicationOverride, "source", "revision",
args.revision)
cfg.add(config.Scope.applicationOverride, "source", "distribution.version",
args.distribution_version)
cfg.add(config.Scope.applicationOverride, "source",
"distribution.repository", args.distribution_repository)
cfg.add(config.Scope.applicationOverride, "system", "pipeline",
args.pipeline)
cfg.add(config.Scope.applicationOverride, "system", "track.repository",
args.track_repository)
cfg.add(config.Scope.applicationOverride, "system", "quiet.mode",
args.quiet)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
args.offline)
cfg.add(config.Scope.applicationOverride, "system", "user.tag",
args.user_tag)
cfg.add(config.Scope.applicationOverride, "system", "logging.output",
args.logging)
cfg.add(config.Scope.applicationOverride, "telemetry", "devices",
csv_to_list(args.telemetry))
cfg.add(config.Scope.applicationOverride, "benchmarks", "track",
args.track)
cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge",
args.challenge)
cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car)
cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health",
args.cluster_health)
cfg.add(config.Scope.applicationOverride, "benchmarks", "laps", args.laps)
cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths",
csv_to_list(args.data_paths))
cfg.add(config.Scope.applicationOverride, "provisioning",
"install.preserve", convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "launcher",
"external.target.hosts",
convert_hosts(csv_to_list(args.target_hosts)))
cfg.add(config.Scope.applicationOverride, "launcher", "client.options",
kv_to_map(csv_to_list(args.client_options)))
cfg.add(config.Scope.applicationOverride, "report", "reportformat",
args.report_format)
cfg.add(config.Scope.applicationOverride, "report", "reportfile",
args.report_file)
if args.override_src_dir is not None:
cfg.add(config.Scope.applicationOverride, "source", "local.src.dir",
args.override_src_dir)
if sub_command == "list":
cfg.add(config.Scope.applicationOverride, "system",
"list.config.option", args.configuration)
cfg.add(config.Scope.applicationOverride, "system",
"list.races.max_results", args.limit)
if sub_command == "compare":
cfg.add(config.Scope.applicationOverride, "report",
"comparison.baseline.timestamp", args.baseline)
cfg.add(config.Scope.applicationOverride, "report",
"comparison.contender.timestamp", args.contender)
configure_logging(cfg)
logger.info("Rally version [%s]" % version())
logger.info("Command line arguments: %s" % args)
# Configure networking
net.init()
if not args.offline:
if not net.has_internet_connection():
console.warn(
"No Internet connection detected. Automatic download of track data sets etc. is disabled.",
logger=logger)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
True)
else:
logger.info("Detected a working Internet connection.")
# Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine
# noinspection PyBroadException
try:
process.kill_running_rally_instances()
except BaseException:
logger.exception(
"Could not terminate potentially running Rally instances correctly. Attempting to go on anyway."
)
try:
actors = bootstrap_actor_system(cfg)
except RuntimeError as e:
logger.exception("Could not bootstrap actor system.")
if str(e) == "Unable to determine valid external socket address.":
console.warn(
"Could not determine a socket address. Are you running without any network?",
logger=logger)
actors = bootstrap_actor_system(cfg,
system_base="multiprocQueueBase")
else:
raise
success = False
try:
success = dispatch_sub_command(cfg, sub_command)
finally:
shutdown_complete = False
times_interrupted = 0
while not shutdown_complete and times_interrupted < 2:
try:
logger.info("Attempting to shutdown internal actor system.")
actors.shutdown()
shutdown_complete = True
logger.info("Shutdown completed.")
except KeyboardInterrupt:
times_interrupted += 1
logger.warn(
"User interrupted shutdown of internal actor system.")
console.info(
"Please wait a moment for Rally's internal components to shutdown."
)
if not shutdown_complete and times_interrupted > 0:
logger.warn(
"Terminating after user has interrupted actor system shutdown explicitly for [%d] times."
% times_interrupted)
console.println("")
console.warn(
"Terminating now at the risk of leaving child processes behind."
)
console.println("")
console.warn("The next race may fail due to an unclean shutdown.")
console.println("")
console.println(SKULL)
console.println("")
end = time.time()
if success:
console.println("")
console.info("SUCCESS (took %d seconds)" % (end - start),
overline="-",
underline="-")
else:
console.println("")
console.info("FAILURE (took %d seconds)" % (end - start),
overline="-",
underline="-")
sys.exit(64)
def test_global_rally_running_in_docker_is_true(self):
console.init()
assert console.RALLY_RUNNING_IN_DOCKER is True
def main():
check_python_version()
log.install_default_log_config()
log.configure_logging()
logger = logging.getLogger(__name__)
start = time.time()
# Early init of console output so we start to show everything consistently.
console.init(quiet=False)
arg_parser = create_arg_parser()
args = arg_parser.parse_args()
if args.subcommand is None:
arg_parser.print_help()
sys.exit(0)
console.init(quiet=args.quiet)
console.println(BANNER)
cfg = config.Config(config_name=args.configuration_name)
if not cfg.config_present():
cfg.install_default_config()
cfg.load_config(auto_upgrade=True)
cfg.add(config.Scope.application, "system", "time.start",
datetime.datetime.utcnow())
# Local config per node
cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root())
cfg.add(config.Scope.application, "node", "rally.cwd", os.getcwd())
logger.info("OS [%s]", str(platform.uname()))
logger.info("Python [%s]", str(sys.implementation))
logger.info("Rally version [%s]", version.version())
logger.debug("Command line arguments: %s", args)
# Configure networking
net.init()
if not args.offline:
probing_url = cfg.opts("system",
"probing.url",
default_value="https://github.com",
mandatory=False)
if not net.has_internet_connection(probing_url):
console.warn(
"No Internet connection detected. Automatic download of track data sets etc. is disabled.",
logger=logger)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
True)
else:
logger.info("Detected a working Internet connection.")
result = dispatch_sub_command(arg_parser, args, cfg)
end = time.time()
if result == ExitStatus.SUCCESSFUL:
console.println("")
console.info("SUCCESS (took %d seconds)" % (end - start),
overline="-",
underline="-")
elif result == ExitStatus.INTERRUPTED:
console.println("")
console.info("ABORTED (took %d seconds)" % (end - start),
overline="-",
underline="-")
sys.exit(130)
elif result == ExitStatus.ERROR:
console.println("")
console.info("FAILURE (took %d seconds)" % (end - start),
overline="-",
underline="-")
sys.exit(64)
def main():
check_python_version()
start = time.time()
# Early init of console output so we start to show everything consistently.
console.init(quiet=False)
# allow to see a thread-dump on SIGQUIT
faulthandler.register(signal.SIGQUIT, file=sys.stderr)
pre_configure_logging()
arg_parser = create_arg_parser()
args = arg_parser.parse_args()
console.init(quiet=args.quiet)
console.println(BANNER)
cfg = config.Config(config_name=args.configuration_name)
sub_command = derive_sub_command(args, cfg)
ensure_configuration_present(cfg, args, sub_command)
if args.effective_start_date:
cfg.add(config.Scope.application, "system", "time.start",
args.effective_start_date)
cfg.add(config.Scope.application, "system", "time.start.user_provided",
True)
else:
cfg.add(config.Scope.application, "system", "time.start",
datetime.datetime.utcnow())
cfg.add(config.Scope.application, "system", "time.start.user_provided",
False)
cfg.add(config.Scope.applicationOverride, "system", "quiet.mode",
args.quiet)
# per node?
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
args.offline)
cfg.add(config.Scope.applicationOverride, "system", "logging.output",
args.logging)
# Local config per node
cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root())
cfg.add(config.Scope.application, "node", "rally.cwd", os.getcwd())
cfg.add(config.Scope.applicationOverride, "mechanic", "source.revision",
args.revision)
if args.distribution_version:
cfg.add(config.Scope.applicationOverride, "mechanic",
"distribution.version", args.distribution_version)
cfg.add(config.Scope.applicationOverride, "mechanic",
"distribution.repository", args.distribution_repository)
cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name",
args.team_repository)
cfg.add(config.Scope.applicationOverride, "mechanic", "car.names",
csv_to_list(args.car))
cfg.add(config.Scope.applicationOverride, "mechanic", "car.plugins",
csv_to_list(args.elasticsearch_plugins))
cfg.add(config.Scope.applicationOverride, "mechanic", "node.datapaths",
csv_to_list(args.data_paths))
if args.keep_cluster_running:
cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running",
True)
# force-preserve the cluster nodes.
cfg.add(config.Scope.applicationOverride, "mechanic",
"preserve.install", True)
else:
cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running",
False)
cfg.add(config.Scope.applicationOverride, "mechanic",
"preserve.install", convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.devices",
csv_to_list(args.telemetry))
cfg.add(config.Scope.applicationOverride, "race", "pipeline",
args.pipeline)
cfg.add(config.Scope.applicationOverride, "race", "laps", args.laps)
cfg.add(config.Scope.applicationOverride, "race", "user.tag",
args.user_tag)
# We can assume here that if a track-path is given, the user did not specify a repository either (although argparse sets it to
# its default value)
if args.track_path:
cfg.add(config.Scope.applicationOverride, "track", "track.path",
os.path.abspath(io.normalize_path(args.track_path)))
cfg.add(config.Scope.applicationOverride, "track", "repository.name",
None)
if args.track:
# stay as close as possible to argparse errors although we have a custom validation.
arg_parser.error(
"argument --track not allowed with argument --track-path")
# cfg.add(config.Scope.applicationOverride, "track", "track.name", None)
else:
# cfg.add(config.Scope.applicationOverride, "track", "track.path", None)
cfg.add(config.Scope.applicationOverride, "track", "repository.name",
args.track_repository)
# set the default programmatically because we need to determine whether the user has provided a value
chosen_track = args.track if args.track else "geonames"
cfg.add(config.Scope.applicationOverride, "track", "track.name",
chosen_track)
cfg.add(config.Scope.applicationOverride, "track", "params",
kv_to_map(csv_to_list(args.track_params)))
cfg.add(config.Scope.applicationOverride, "track", "challenge.name",
args.challenge)
cfg.add(config.Scope.applicationOverride, "track", "include.tasks",
csv_to_list(args.include_tasks))
cfg.add(config.Scope.applicationOverride, "track", "test.mode.enabled",
args.test_mode)
cfg.add(config.Scope.applicationOverride, "track", "auto_manage_indices",
to_bool(args.auto_manage_indices))
cfg.add(config.Scope.applicationOverride, "reporting", "format",
args.report_format)
cfg.add(config.Scope.applicationOverride, "reporting", "values",
args.show_in_report)
cfg.add(config.Scope.applicationOverride, "reporting", "output.path",
args.report_file)
if sub_command == "compare":
cfg.add(config.Scope.applicationOverride, "reporting",
"baseline.timestamp", args.baseline)
cfg.add(config.Scope.applicationOverride, "reporting",
"contender.timestamp", args.contender)
cfg.add(config.Scope.applicationOverride, "driver", "cluster.health",
args.cluster_health)
if args.cluster_health != "green":
console.warn(
"--cluster-health is deprecated and will be removed in a future version of Rally."
)
cfg.add(config.Scope.applicationOverride, "driver", "profiling",
args.enable_driver_profiling)
cfg.add(config.Scope.applicationOverride, "driver", "on.error",
args.on_error)
cfg.add(config.Scope.applicationOverride, "driver", "load_driver_hosts",
csv_to_list(args.load_driver_hosts))
if sub_command != "list":
# Also needed by mechanic (-> telemetry) - duplicate by module?
cfg.add(config.Scope.applicationOverride, "client", "hosts",
_normalize_hosts(csv_to_list(args.target_hosts)))
client_options = kv_to_map(csv_to_list(args.client_options))
cfg.add(config.Scope.applicationOverride, "client", "options",
client_options)
if "timeout" not in client_options:
console.info(
"You did not provide an explicit timeout in the client options. Assuming default of 10 seconds."
)
# split by component?
if sub_command == "list":
cfg.add(config.Scope.applicationOverride, "system",
"list.config.option", args.configuration)
cfg.add(config.Scope.applicationOverride, "system",
"list.races.max_results", args.limit)
configure_logging(cfg)
logger.info("OS [%s]" % str(os.uname()))
logger.info("Python [%s]" % str(sys.implementation))
logger.info("Rally version [%s]" % version.version())
logger.info("Command line arguments: %s" % args)
# Configure networking
net.init()
if not args.offline:
if not net.has_internet_connection():
console.warn(
"No Internet connection detected. Automatic download of track data sets etc. is disabled.",
logger=logger)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
True)
else:
logger.info("Detected a working Internet connection.")
# Kill any lingering Rally processes before attempting to continue - the actor system needs to be a singleton on this machine
# noinspection PyBroadException
try:
process.kill_running_rally_instances()
except BaseException:
logger.exception(
"Could not terminate potentially running Rally instances correctly. Attempting to go on anyway."
)
success = dispatch_sub_command(cfg, sub_command)
end = time.time()
if success:
console.println("")
console.info("SUCCESS (took %d seconds)" % (end - start),
overline="-",
underline="-")
else:
console.println("")
console.info("FAILURE (took %d seconds)" % (end - start),
overline="-",
underline="-")
sys.exit(64)
def main():
check_python_version()
start = time.time()
# Early init of console output so we start to show everything consistently.
console.init(quiet=False)
# allow to see a thread-dump on SIGQUIT
faulthandler.register(signal.SIGQUIT, file=sys.stderr)
pre_configure_logging()
args = parse_args()
console.init(quiet=args.quiet)
console.println(BANNER)
cfg = config.Config(config_name=args.configuration_name)
sub_command = derive_sub_command(args, cfg)
ensure_configuration_present(cfg, args, sub_command)
if args.effective_start_date:
cfg.add(config.Scope.application, "system", "time.start", args.effective_start_date)
cfg.add(config.Scope.application, "system", "time.start.user_provided", True)
else:
cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.utcnow())
cfg.add(config.Scope.application, "system", "time.start.user_provided", False)
cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet)
# per node?
cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline)
cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging)
# Local config per node
cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root())
cfg.add(config.Scope.application, "node", "rally.cwd", os.getcwd())
cfg.add(config.Scope.applicationOverride, "mechanic", "source.revision", args.revision)
#TODO dm: Consider renaming this one. It's used by different modules
if args.distribution_version:
cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.version", args.distribution_version)
cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.repository", args.distribution_repository)
cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name", args.team_repository)
cfg.add(config.Scope.applicationOverride, "mechanic", "car.names", csv_to_list(args.car))
cfg.add(config.Scope.applicationOverride, "mechanic", "car.plugins", csv_to_list(args.elasticsearch_plugins))
cfg.add(config.Scope.applicationOverride, "mechanic", "node.datapaths", csv_to_list(args.data_paths))
if args.keep_cluster_running:
cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running", True)
# force-preserve the cluster nodes.
cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install", True)
else:
cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running", False)
cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install", convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.devices", csv_to_list(args.telemetry))
cfg.add(config.Scope.applicationOverride, "race", "pipeline", args.pipeline)
cfg.add(config.Scope.applicationOverride, "race", "laps", args.laps)
cfg.add(config.Scope.applicationOverride, "race", "user.tag", args.user_tag)
cfg.add(config.Scope.applicationOverride, "track", "repository.name", args.track_repository)
cfg.add(config.Scope.applicationOverride, "track", "track.name", args.track)
cfg.add(config.Scope.applicationOverride, "track", "challenge.name", args.challenge)
cfg.add(config.Scope.applicationOverride, "track", "include.tasks", csv_to_list(args.include_tasks))
cfg.add(config.Scope.applicationOverride, "track", "test.mode.enabled", args.test_mode)
cfg.add(config.Scope.applicationOverride, "track", "auto_manage_indices", to_bool(args.auto_manage_indices))
cfg.add(config.Scope.applicationOverride, "reporting", "format", args.report_format)
cfg.add(config.Scope.applicationOverride, "reporting", "output.path", args.report_file)
if sub_command == "compare":
cfg.add(config.Scope.applicationOverride, "reporting", "baseline.timestamp", args.baseline)
cfg.add(config.Scope.applicationOverride, "reporting", "contender.timestamp", args.contender)
################################
# new section name: driver
################################
cfg.add(config.Scope.applicationOverride, "driver", "cluster.health", args.cluster_health)
cfg.add(config.Scope.applicationOverride, "driver", "profiling", args.enable_driver_profiling)
cfg.add(config.Scope.applicationOverride, "driver", "load_driver_hosts", csv_to_list(args.load_driver_hosts))
if sub_command != "list":
# Also needed by mechanic (-> telemetry) - duplicate by module?
cfg.add(config.Scope.applicationOverride, "client", "hosts", _normalize_hosts(csv_to_list(args.target_hosts)))
client_options = kv_to_map(csv_to_list(args.client_options))
cfg.add(config.Scope.applicationOverride, "client", "options", client_options)
if "timeout" not in client_options:
console.info("You did not provide an explicit timeout in the client options. Assuming default of 10 seconds.")
# split by component?
if sub_command == "list":
cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration)
cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit)
configure_logging(cfg)
logger.info("OS [%s]" % str(os.uname()))
logger.info("Python [%s]" % str(sys.implementation))
logger.info("Rally version [%s]" % version.version())
logger.info("Command line arguments: %s" % args)
# Configure networking
net.init()
if not args.offline:
if not net.has_internet_connection():
console.warn("No Internet connection detected. Automatic download of track data sets etc. is disabled.",
logger=logger)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True)
else:
logger.info("Detected a working Internet connection.")
# Kill any lingering Rally processes before attempting to continue - the actor system needs to be a singleton on this machine
# noinspection PyBroadException
try:
process.kill_running_rally_instances()
except BaseException:
logger.exception("Could not terminate potentially running Rally instances correctly. Attempting to go on anyway.")
success = dispatch_sub_command(cfg, sub_command)
end = time.time()
if success:
console.println("")
console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-")
else:
console.println("")
console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-")
sys.exit(64)
def main():
start = time.time()
# Early init of console output so we start to show everything consistently.
console.init(quiet=False)
pre_configure_logging()
args = parse_args()
console.init(quiet=args.quiet)
console.println(BANNER)
cfg = config.Config(config_name=args.configuration_name)
sub_command = derive_sub_command(args, cfg)
ensure_configuration_present(cfg, args, sub_command)
# Add global meta info derived by rally itself
cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date)
cfg.add(config.Scope.application, "system", "rally.root", rally_root_path())
cfg.add(config.Scope.application, "system", "rally.cwd", os.getcwd())
cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root())
# Add command line config
cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision)
cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version)
cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository)
cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline)
cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository)
cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline)
cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag)
cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging)
cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry))
cfg.add(config.Scope.applicationOverride, "benchmarks", "track", args.track)
cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge)
cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car)
cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health", args.cluster_health)
cfg.add(config.Scope.applicationOverride, "benchmarks", "laps", args.laps)
cfg.add(config.Scope.applicationOverride, "benchmarks", "test.mode", args.test_mode)
cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths))
cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", convert_hosts(csv_to_list(args.target_hosts)))
cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options)))
cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format)
cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file)
if args.override_src_dir is not None:
cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir)
if sub_command == "list":
cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration)
cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit)
if sub_command == "compare":
cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline)
cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender)
configure_logging(cfg)
logger.info("Rally version [%s]" % version())
logger.info("Command line arguments: %s" % args)
# Configure networking
net.init()
if not args.offline:
if not net.has_internet_connection():
console.warn("No Internet connection detected. Automatic download of track data sets etc. is disabled.",
logger=logger)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True)
else:
logger.info("Detected a working Internet connection.")
# Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine
# noinspection PyBroadException
try:
process.kill_running_rally_instances()
except BaseException:
logger.exception("Could not terminate potentially running Rally instances correctly. Attempting to go on anyway.")
try:
actors = bootstrap_actor_system(cfg)
except RuntimeError as e:
logger.exception("Could not bootstrap actor system.")
if str(e) == "Unable to determine valid external socket address.":
console.warn("Could not determine a socket address. Are you running without any network?", logger=logger)
actors = bootstrap_actor_system(cfg, system_base="multiprocQueueBase")
else:
raise
success = False
try:
success = dispatch_sub_command(cfg, sub_command)
finally:
shutdown_complete = False
times_interrupted = 0
while not shutdown_complete and times_interrupted < 2:
try:
logger.info("Attempting to shutdown internal actor system.")
actors.shutdown()
shutdown_complete = True
logger.info("Shutdown completed.")
except KeyboardInterrupt:
times_interrupted += 1
logger.warn("User interrupted shutdown of internal actor system.")
console.info("Please wait a moment for Rally's internal components to shutdown.")
if not shutdown_complete and times_interrupted > 0:
logger.warn("Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted)
console.println("")
console.warn("Terminating now at the risk of leaving child processes behind.")
console.println("")
console.warn("The next race may fail due to an unclean shutdown.")
console.println("")
console.println(SKULL)
console.println("")
end = time.time()
if success:
console.println("")
console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-")
else:
console.println("")
console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-")
sys.exit(64)