def with_actor_system(runnable, cfg): import thespian.actors logger = logging.getLogger(__name__) already_running = actor.actor_system_already_running() logger.info("Actor system already running locally? [%s]", str(already_running)) try: actors = actor.bootstrap_actor_system( try_join=already_running, prefer_local_only=not already_running) # We can only support remote benchmarks if we have a dedicated daemon that is not only bound to 127.0.0.1 cfg.add(config.Scope.application, "system", "remote.benchmarking.supported", already_running) # This happens when the admin process could not be started, e.g. because it could not open a socket. except thespian.actors.InvalidActorAddress: logger.info("Falling back to offline actor system.") actor.use_offline_actor_system() actors = actor.bootstrap_actor_system(try_join=True) except Exception as e: logger.exception("Could not bootstrap actor system.") if str(e) == "Unable to determine valid external socket address.": console.warn( "Could not determine a socket address. Are you running without any network? Switching to degraded mode.", logger=logger) logger.info("Falling back to offline actor system.") actor.use_offline_actor_system() actors = actor.bootstrap_actor_system(try_join=True) else: raise try: runnable(cfg) finally: # We only shutdown the actor system if it was not already running before if not already_running: shutdown_complete = False times_interrupted = 0 # give some time for any outstanding messages to be delivered to the actor system time.sleep(3) while not shutdown_complete and times_interrupted < 2: try: logger.info( "Attempting to shutdown internal actor system.") actors.shutdown() # note that this check will only evaluate to True for a TCP-based actor system. timeout = 15 while actor.actor_system_already_running() and timeout > 0: logger.info( "Actor system is still running. Waiting...") time.sleep(1) timeout -= 1 if timeout > 0: shutdown_complete = True logger.info("Shutdown completed.") else: logger.warning( "Shutdown timed out. Actor system is still running." ) break except KeyboardInterrupt: times_interrupted += 1 logger.warning( "User interrupted shutdown of internal actor system.") console.info( "Please wait a moment for Rally's internal components to shutdown." ) if not shutdown_complete and times_interrupted > 0: logger.warning( "Terminating after user has interrupted actor system shutdown explicitly for [%d] times.", times_interrupted) console.println("") console.warn( "Terminating now at the risk of leaving child processes behind." ) console.println("") console.warn( "The next race may fail due to an unclean shutdown.") console.println("") console.println(SKULL) console.println("") elif not shutdown_complete: console.warn( "Could not terminate all internal processes within timeout. Please check and force-terminate all Rally processes." )
def main(): pre_configure_logging() args = parse_args() if not args.quiet: print(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) # Add global meta info derived by rally itself cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "rally.root", rally_root_path()) cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root()) # Add command line config cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision) cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository) cfg.add(config.Scope.applicationOverride, "system", "track", args.track) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge) cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car) cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", convert_hosts(csv_to_list(args.target_hosts))) cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options))) cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format) cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file) if args.override_src_dir is not None: cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir) if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender) configure_logging(cfg) logger.info("Rally version [%s]" % version()) logger.info("Command line arguments: %s" % args) # Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine try: process.kill_running_rally_instances() except BaseException: logger.exception( "Could not terminate potentially running Rally instances correctly. Attempting to go on anyway." ) # bootstrap Rally's Actor system try: actors = thespian.actors.ActorSystem( "multiprocTCPBase", logDefs=configure_actor_logging(cfg)) except thespian.actors.ActorSystemException: logger.exception( "Could not initialize internal actor system. Terminating.") print("ERROR: Could not initialize successfully.") print("") print( "The most likely cause is that there are still processes running from a previous race." ) print( "Please check for running Python processes and terminate them before running Rally again." ) print("") print_help_on_errors(cfg) sys.exit(70) success = False try: success = dispatch_sub_command(cfg, sub_command) finally: shutdown_complete = False times_interrupted = 0 while not shutdown_complete and times_interrupted < 2: try: actors.shutdown() shutdown_complete = True except KeyboardInterrupt: times_interrupted += 1 logger.warn( "User interrupted shutdown of internal actor system.") print( "Please wait a moment for Rally's internal components to shutdown." ) if not shutdown_complete and times_interrupted > 0: logger.warn( "Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted) print( "**********************************************************************" ) print("") print( "WARN: Terminating now at the risk of leaving child processes behind." ) print("") print("The next race may fail due to an unclean shutdown.") print("") print(SKULL) print("") print( "**********************************************************************" ) if not success: sys.exit(64)
def main(): start = time.time() # Early init of console output so we start to show everything consistently. console.init(quiet=False) pre_configure_logging() args = parse_args() console.init(quiet=args.quiet) console.println(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) # Add global meta info derived by rally itself cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "rally.root", rally_root_path()) cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root()) # Add command line config cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision) cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging) cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "benchmarks", "track", args.track) cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge) cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car) cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health", args.cluster_health) cfg.add(config.Scope.applicationOverride, "benchmarks", "laps", args.laps) cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", convert_hosts(csv_to_list(args.target_hosts))) cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options))) cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format) cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file) if args.override_src_dir is not None: cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir) if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender) configure_logging(cfg) logger.info("Rally version [%s]" % version()) logger.info("Command line arguments: %s" % args) # Configure networking net.init() if not args.offline: if not net.has_internet_connection(): console.warn( "No Internet connection detected. Automatic download of track data sets etc. is disabled.", logger=logger) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True) else: logger.info("Detected a working Internet connection.") # Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine # noinspection PyBroadException try: process.kill_running_rally_instances() except BaseException: logger.exception( "Could not terminate potentially running Rally instances correctly. Attempting to go on anyway." ) try: actors = bootstrap_actor_system(cfg) except RuntimeError as e: logger.exception("Could not bootstrap actor system.") if str(e) == "Unable to determine valid external socket address.": console.warn( "Could not determine a socket address. Are you running without any network?", logger=logger) actors = bootstrap_actor_system(cfg, system_base="multiprocQueueBase") else: raise success = False try: success = dispatch_sub_command(cfg, sub_command) finally: shutdown_complete = False times_interrupted = 0 while not shutdown_complete and times_interrupted < 2: try: logger.info("Attempting to shutdown internal actor system.") actors.shutdown() shutdown_complete = True logger.info("Shutdown completed.") except KeyboardInterrupt: times_interrupted += 1 logger.warn( "User interrupted shutdown of internal actor system.") console.info( "Please wait a moment for Rally's internal components to shutdown." ) if not shutdown_complete and times_interrupted > 0: logger.warn( "Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted) console.println("") console.warn( "Terminating now at the risk of leaving child processes behind." ) console.println("") console.warn("The next race may fail due to an unclean shutdown.") console.println("") console.println(SKULL) console.println("") end = time.time() if success: console.println("") console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-") else: console.println("") console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(64)
def main(): start = time.time() # Early init of console output so we start to show everything consistently. console.init(quiet=False) pre_configure_logging() args = parse_args() console.init(quiet=args.quiet) console.println(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) # Add global meta info derived by rally itself cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "rally.root", rally_root_path()) cfg.add(config.Scope.application, "system", "rally.cwd", os.getcwd()) cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root()) # Add command line config cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision) cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging) cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "benchmarks", "track", args.track) cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge) cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car) cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health", args.cluster_health) cfg.add(config.Scope.applicationOverride, "benchmarks", "laps", args.laps) cfg.add(config.Scope.applicationOverride, "benchmarks", "test.mode", args.test_mode) cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", convert_hosts(csv_to_list(args.target_hosts))) cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options))) cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format) cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file) if args.override_src_dir is not None: cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir) if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender) configure_logging(cfg) logger.info("Rally version [%s]" % version()) logger.info("Command line arguments: %s" % args) # Configure networking net.init() if not args.offline: if not net.has_internet_connection(): console.warn("No Internet connection detected. Automatic download of track data sets etc. is disabled.", logger=logger) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True) else: logger.info("Detected a working Internet connection.") # Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine # noinspection PyBroadException try: process.kill_running_rally_instances() except BaseException: logger.exception("Could not terminate potentially running Rally instances correctly. Attempting to go on anyway.") try: actors = bootstrap_actor_system(cfg) except RuntimeError as e: logger.exception("Could not bootstrap actor system.") if str(e) == "Unable to determine valid external socket address.": console.warn("Could not determine a socket address. Are you running without any network?", logger=logger) actors = bootstrap_actor_system(cfg, system_base="multiprocQueueBase") else: raise success = False try: success = dispatch_sub_command(cfg, sub_command) finally: shutdown_complete = False times_interrupted = 0 while not shutdown_complete and times_interrupted < 2: try: logger.info("Attempting to shutdown internal actor system.") actors.shutdown() shutdown_complete = True logger.info("Shutdown completed.") except KeyboardInterrupt: times_interrupted += 1 logger.warn("User interrupted shutdown of internal actor system.") console.info("Please wait a moment for Rally's internal components to shutdown.") if not shutdown_complete and times_interrupted > 0: logger.warn("Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted) console.println("") console.warn("Terminating now at the risk of leaving child processes behind.") console.println("") console.warn("The next race may fail due to an unclean shutdown.") console.println("") console.println(SKULL) console.println("") end = time.time() if success: console.println("") console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-") else: console.println("") console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(64)