def with_actor_system(runnable, cfg):
import thespian.actors
logger = logging.getLogger(__name__)
already_running = actor.actor_system_already_running()
logger.info("Actor system already running locally? [%s]",
str(already_running))
try:
actors = actor.bootstrap_actor_system(
try_join=already_running, prefer_local_only=not already_running)
# We can only support remote benchmarks if we have a dedicated daemon that is not only bound to 127.0.0.1
cfg.add(config.Scope.application, "system",
"remote.benchmarking.supported", already_running)
# This happens when the admin process could not be started, e.g. because it could not open a socket.
except thespian.actors.InvalidActorAddress:
logger.info("Falling back to offline actor system.")
actor.use_offline_actor_system()
actors = actor.bootstrap_actor_system(try_join=True)
except Exception as e:
logger.exception("Could not bootstrap actor system.")
if str(e) == "Unable to determine valid external socket address.":
console.warn(
"Could not determine a socket address. Are you running without any network? Switching to degraded mode.",
logger=logger)
logger.info("Falling back to offline actor system.")
actor.use_offline_actor_system()
actors = actor.bootstrap_actor_system(try_join=True)
else:
raise
try:
runnable(cfg)
finally:
# We only shutdown the actor system if it was not already running before
if not already_running:
shutdown_complete = False
times_interrupted = 0
# give some time for any outstanding messages to be delivered to the actor system
time.sleep(3)
while not shutdown_complete and times_interrupted < 2:
try:
logger.info(
"Attempting to shutdown internal actor system.")
actors.shutdown()
# note that this check will only evaluate to True for a TCP-based actor system.
timeout = 15
while actor.actor_system_already_running() and timeout > 0:
logger.info(
"Actor system is still running. Waiting...")
time.sleep(1)
timeout -= 1
if timeout > 0:
shutdown_complete = True
logger.info("Shutdown completed.")
else:
logger.warning(
"Shutdown timed out. Actor system is still running."
)
break
except KeyboardInterrupt:
times_interrupted += 1
logger.warning(
"User interrupted shutdown of internal actor system.")
console.info(
"Please wait a moment for Rally's internal components to shutdown."
)
if not shutdown_complete and times_interrupted > 0:
logger.warning(
"Terminating after user has interrupted actor system shutdown explicitly for [%d] times.",
times_interrupted)
console.println("")
console.warn(
"Terminating now at the risk of leaving child processes behind."
)
console.println("")
console.warn(
"The next race may fail due to an unclean shutdown.")
console.println("")
console.println(SKULL)
console.println("")
elif not shutdown_complete:
console.warn(
"Could not terminate all internal processes within timeout. Please check and force-terminate all Rally processes."
)
def main():
pre_configure_logging()
args = parse_args()
if not args.quiet:
print(BANNER)
cfg = config.Config(config_name=args.configuration_name)
sub_command = derive_sub_command(args, cfg)
ensure_configuration_present(cfg, args, sub_command)
# Add global meta info derived by rally itself
cfg.add(config.Scope.application, "meta", "time.start",
args.effective_start_date)
cfg.add(config.Scope.application, "system", "rally.root",
rally_root_path())
cfg.add(config.Scope.application, "system", "invocation.root.dir",
paths.Paths(cfg).invocation_root())
# Add command line config
cfg.add(config.Scope.applicationOverride, "source", "revision",
args.revision)
cfg.add(config.Scope.applicationOverride, "source", "distribution.version",
args.distribution_version)
cfg.add(config.Scope.applicationOverride, "source",
"distribution.repository", args.distribution_repository)
cfg.add(config.Scope.applicationOverride, "system", "pipeline",
args.pipeline)
cfg.add(config.Scope.applicationOverride, "system", "track.repository",
args.track_repository)
cfg.add(config.Scope.applicationOverride, "system", "track", args.track)
cfg.add(config.Scope.applicationOverride, "system", "quiet.mode",
args.quiet)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
args.offline)
cfg.add(config.Scope.applicationOverride, "system", "user.tag",
args.user_tag)
cfg.add(config.Scope.applicationOverride, "telemetry", "devices",
csv_to_list(args.telemetry))
cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge",
args.challenge)
cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car)
cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths",
csv_to_list(args.data_paths))
cfg.add(config.Scope.applicationOverride, "provisioning",
"install.preserve", convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "launcher",
"external.target.hosts",
convert_hosts(csv_to_list(args.target_hosts)))
cfg.add(config.Scope.applicationOverride, "launcher", "client.options",
kv_to_map(csv_to_list(args.client_options)))
cfg.add(config.Scope.applicationOverride, "report", "reportformat",
args.report_format)
cfg.add(config.Scope.applicationOverride, "report", "reportfile",
args.report_file)
if args.override_src_dir is not None:
cfg.add(config.Scope.applicationOverride, "source", "local.src.dir",
args.override_src_dir)
if sub_command == "list":
cfg.add(config.Scope.applicationOverride, "system",
"list.config.option", args.configuration)
cfg.add(config.Scope.applicationOverride, "system",
"list.races.max_results", args.limit)
if sub_command == "compare":
cfg.add(config.Scope.applicationOverride, "report",
"comparison.baseline.timestamp", args.baseline)
cfg.add(config.Scope.applicationOverride, "report",
"comparison.contender.timestamp", args.contender)
configure_logging(cfg)
logger.info("Rally version [%s]" % version())
logger.info("Command line arguments: %s" % args)
# Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine
try:
process.kill_running_rally_instances()
except BaseException:
logger.exception(
"Could not terminate potentially running Rally instances correctly. Attempting to go on anyway."
)
# bootstrap Rally's Actor system
try:
actors = thespian.actors.ActorSystem(
"multiprocTCPBase", logDefs=configure_actor_logging(cfg))
except thespian.actors.ActorSystemException:
logger.exception(
"Could not initialize internal actor system. Terminating.")
print("ERROR: Could not initialize successfully.")
print("")
print(
"The most likely cause is that there are still processes running from a previous race."
)
print(
"Please check for running Python processes and terminate them before running Rally again."
)
print("")
print_help_on_errors(cfg)
sys.exit(70)
success = False
try:
success = dispatch_sub_command(cfg, sub_command)
finally:
shutdown_complete = False
times_interrupted = 0
while not shutdown_complete and times_interrupted < 2:
try:
actors.shutdown()
shutdown_complete = True
except KeyboardInterrupt:
times_interrupted += 1
logger.warn(
"User interrupted shutdown of internal actor system.")
print(
"Please wait a moment for Rally's internal components to shutdown."
)
if not shutdown_complete and times_interrupted > 0:
logger.warn(
"Terminating after user has interrupted actor system shutdown explicitly for [%d] times."
% times_interrupted)
print(
"**********************************************************************"
)
print("")
print(
"WARN: Terminating now at the risk of leaving child processes behind."
)
print("")
print("The next race may fail due to an unclean shutdown.")
print("")
print(SKULL)
print("")
print(
"**********************************************************************"
)
if not success:
sys.exit(64)
def main():
start = time.time()
# Early init of console output so we start to show everything consistently.
console.init(quiet=False)
pre_configure_logging()
args = parse_args()
console.init(quiet=args.quiet)
console.println(BANNER)
cfg = config.Config(config_name=args.configuration_name)
sub_command = derive_sub_command(args, cfg)
ensure_configuration_present(cfg, args, sub_command)
# Add global meta info derived by rally itself
cfg.add(config.Scope.application, "meta", "time.start",
args.effective_start_date)
cfg.add(config.Scope.application, "system", "rally.root",
rally_root_path())
cfg.add(config.Scope.application, "system", "invocation.root.dir",
paths.Paths(cfg).invocation_root())
# Add command line config
cfg.add(config.Scope.applicationOverride, "source", "revision",
args.revision)
cfg.add(config.Scope.applicationOverride, "source", "distribution.version",
args.distribution_version)
cfg.add(config.Scope.applicationOverride, "source",
"distribution.repository", args.distribution_repository)
cfg.add(config.Scope.applicationOverride, "system", "pipeline",
args.pipeline)
cfg.add(config.Scope.applicationOverride, "system", "track.repository",
args.track_repository)
cfg.add(config.Scope.applicationOverride, "system", "quiet.mode",
args.quiet)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
args.offline)
cfg.add(config.Scope.applicationOverride, "system", "user.tag",
args.user_tag)
cfg.add(config.Scope.applicationOverride, "system", "logging.output",
args.logging)
cfg.add(config.Scope.applicationOverride, "telemetry", "devices",
csv_to_list(args.telemetry))
cfg.add(config.Scope.applicationOverride, "benchmarks", "track",
args.track)
cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge",
args.challenge)
cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car)
cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health",
args.cluster_health)
cfg.add(config.Scope.applicationOverride, "benchmarks", "laps", args.laps)
cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths",
csv_to_list(args.data_paths))
cfg.add(config.Scope.applicationOverride, "provisioning",
"install.preserve", convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "launcher",
"external.target.hosts",
convert_hosts(csv_to_list(args.target_hosts)))
cfg.add(config.Scope.applicationOverride, "launcher", "client.options",
kv_to_map(csv_to_list(args.client_options)))
cfg.add(config.Scope.applicationOverride, "report", "reportformat",
args.report_format)
cfg.add(config.Scope.applicationOverride, "report", "reportfile",
args.report_file)
if args.override_src_dir is not None:
cfg.add(config.Scope.applicationOverride, "source", "local.src.dir",
args.override_src_dir)
if sub_command == "list":
cfg.add(config.Scope.applicationOverride, "system",
"list.config.option", args.configuration)
cfg.add(config.Scope.applicationOverride, "system",
"list.races.max_results", args.limit)
if sub_command == "compare":
cfg.add(config.Scope.applicationOverride, "report",
"comparison.baseline.timestamp", args.baseline)
cfg.add(config.Scope.applicationOverride, "report",
"comparison.contender.timestamp", args.contender)
configure_logging(cfg)
logger.info("Rally version [%s]" % version())
logger.info("Command line arguments: %s" % args)
# Configure networking
net.init()
if not args.offline:
if not net.has_internet_connection():
console.warn(
"No Internet connection detected. Automatic download of track data sets etc. is disabled.",
logger=logger)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode",
True)
else:
logger.info("Detected a working Internet connection.")
# Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine
# noinspection PyBroadException
try:
process.kill_running_rally_instances()
except BaseException:
logger.exception(
"Could not terminate potentially running Rally instances correctly. Attempting to go on anyway."
)
try:
actors = bootstrap_actor_system(cfg)
except RuntimeError as e:
logger.exception("Could not bootstrap actor system.")
if str(e) == "Unable to determine valid external socket address.":
console.warn(
"Could not determine a socket address. Are you running without any network?",
logger=logger)
actors = bootstrap_actor_system(cfg,
system_base="multiprocQueueBase")
else:
raise
success = False
try:
success = dispatch_sub_command(cfg, sub_command)
finally:
shutdown_complete = False
times_interrupted = 0
while not shutdown_complete and times_interrupted < 2:
try:
logger.info("Attempting to shutdown internal actor system.")
actors.shutdown()
shutdown_complete = True
logger.info("Shutdown completed.")
except KeyboardInterrupt:
times_interrupted += 1
logger.warn(
"User interrupted shutdown of internal actor system.")
console.info(
"Please wait a moment for Rally's internal components to shutdown."
)
if not shutdown_complete and times_interrupted > 0:
logger.warn(
"Terminating after user has interrupted actor system shutdown explicitly for [%d] times."
% times_interrupted)
console.println("")
console.warn(
"Terminating now at the risk of leaving child processes behind."
)
console.println("")
console.warn("The next race may fail due to an unclean shutdown.")
console.println("")
console.println(SKULL)
console.println("")
end = time.time()
if success:
console.println("")
console.info("SUCCESS (took %d seconds)" % (end - start),
overline="-",
underline="-")
else:
console.println("")
console.info("FAILURE (took %d seconds)" % (end - start),
overline="-",
underline="-")
sys.exit(64)
def main():
start = time.time()
# Early init of console output so we start to show everything consistently.
console.init(quiet=False)
pre_configure_logging()
args = parse_args()
console.init(quiet=args.quiet)
console.println(BANNER)
cfg = config.Config(config_name=args.configuration_name)
sub_command = derive_sub_command(args, cfg)
ensure_configuration_present(cfg, args, sub_command)
# Add global meta info derived by rally itself
cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date)
cfg.add(config.Scope.application, "system", "rally.root", rally_root_path())
cfg.add(config.Scope.application, "system", "rally.cwd", os.getcwd())
cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root())
# Add command line config
cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision)
cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version)
cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository)
cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline)
cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository)
cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline)
cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag)
cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging)
cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry))
cfg.add(config.Scope.applicationOverride, "benchmarks", "track", args.track)
cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge)
cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car)
cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health", args.cluster_health)
cfg.add(config.Scope.applicationOverride, "benchmarks", "laps", args.laps)
cfg.add(config.Scope.applicationOverride, "benchmarks", "test.mode", args.test_mode)
cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths))
cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", convert_hosts(csv_to_list(args.target_hosts)))
cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options)))
cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format)
cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file)
if args.override_src_dir is not None:
cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir)
if sub_command == "list":
cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration)
cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit)
if sub_command == "compare":
cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline)
cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender)
configure_logging(cfg)
logger.info("Rally version [%s]" % version())
logger.info("Command line arguments: %s" % args)
# Configure networking
net.init()
if not args.offline:
if not net.has_internet_connection():
console.warn("No Internet connection detected. Automatic download of track data sets etc. is disabled.",
logger=logger)
cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True)
else:
logger.info("Detected a working Internet connection.")
# Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine
# noinspection PyBroadException
try:
process.kill_running_rally_instances()
except BaseException:
logger.exception("Could not terminate potentially running Rally instances correctly. Attempting to go on anyway.")
try:
actors = bootstrap_actor_system(cfg)
except RuntimeError as e:
logger.exception("Could not bootstrap actor system.")
if str(e) == "Unable to determine valid external socket address.":
console.warn("Could not determine a socket address. Are you running without any network?", logger=logger)
actors = bootstrap_actor_system(cfg, system_base="multiprocQueueBase")
else:
raise
success = False
try:
success = dispatch_sub_command(cfg, sub_command)
finally:
shutdown_complete = False
times_interrupted = 0
while not shutdown_complete and times_interrupted < 2:
try:
logger.info("Attempting to shutdown internal actor system.")
actors.shutdown()
shutdown_complete = True
logger.info("Shutdown completed.")
except KeyboardInterrupt:
times_interrupted += 1
logger.warn("User interrupted shutdown of internal actor system.")
console.info("Please wait a moment for Rally's internal components to shutdown.")
if not shutdown_complete and times_interrupted > 0:
logger.warn("Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted)
console.println("")
console.warn("Terminating now at the risk of leaving child processes behind.")
console.println("")
console.warn("The next race may fail due to an unclean shutdown.")
console.println("")
console.println(SKULL)
console.println("")
end = time.time()
if success:
console.println("")
console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-")
else:
console.println("")
console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-")
sys.exit(64)
