def prepare_track(ctx): track_name = ctx.config.opts("system", "track") try: ctx.track = track.load_track(ctx.config, track_name) except FileNotFoundError: logger.error("Cannot load track [%s]" % track_name) raise exceptions.ImproperlyConfigured( "Cannot load track %s. You can list the available tracks with %s list tracks." % (track_name, PROGRAM_NAME)) track.prepare_track(ctx.track, ctx.config) race_paths = paths.Paths(ctx.config) track_root = race_paths.track_root(track_name) ctx.config.add(config.Scope.benchmark, "system", "track.root.dir", track_root) selected_challenge = ctx.config.opts("benchmarks", "challenge") for challenge in ctx.track.challenges: if challenge.name == selected_challenge: ctx.challenge = challenge if not ctx.challenge: raise exceptions.ImproperlyConfigured( "Unknown challenge [%s] for track [%s]. You can list the available tracks and their " "challenges with %s list tracks." % (selected_challenge, ctx.track.name, PROGRAM_NAME)) race_paths = paths.Paths(ctx.config) ctx.config.add( config.Scope.challenge, "system", "challenge.root.dir", race_paths.challenge_root(ctx.track.name, ctx.challenge.name)) ctx.config.add( config.Scope.challenge, "system", "challenge.log.dir", race_paths.challenge_logs(ctx.track.name, ctx.challenge.name))
def __call__(self, track, challenge, car): invocation_root = paths.Paths(self._config).invocation_root() log_root = paths.Paths(self._config).log_root() # for external benchmarks, there is no match to a car if car: car_suffix = "-%s" % car.name else: car_suffix = "" archive_path = "%s/logs-%s-%s%s.zip" % (invocation_root, track.name, challenge.name, car_suffix) io.compress(log_root, archive_path) print("\nLogs for this race are archived in %s" % archive_path) shutil.rmtree(log_root)
def sweep(ctx): invocation_root = paths.Paths(ctx.config).invocation_root() track_name = ctx.config.opts("system", "track") challenge_name = ctx.config.opts("benchmarks", "challenge") car_name = ctx.config.opts("benchmarks", "car") log_root = paths.Paths(ctx.config).log_root() # for external benchmarks, there is no match to a car car_suffix = "-%s" % car_name if car_name else "" archive_path = "%s/logs-%s-%s%s.zip" % (invocation_root, track_name, challenge_name, car_suffix) io.compress(log_root, archive_path) print("\nLogs for this race are archived in %s" % archive_path) shutil.rmtree(log_root)
def configure_logging(cfg): log_dir = paths.Paths(cfg).log_root() io.ensure_dir(log_dir) cfg.add(config.Scope.application, "system", "log.dir", log_dir) log_file = "%s/rally_out.log" % log_dir print("\nWriting additional logs to %s\n" % log_file) # Remove all handlers associated with the root logger object so we can start over with an entirely fresh log configuration for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) if cfg.opts("system", "quiet.mode"): log_level = logging.ERROR else: log_level = logging.INFO ch = logging.FileHandler(filename=log_file, mode="a") ch.setLevel(log_level) formatter = logging.Formatter( "%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") formatter.converter = time.gmtime ch.setFormatter(formatter) logging.root.addHandler(ch)
def run(self, track): selected_setups = self.ctx.config.opts("benchmarks", "tracksetups.selected") any_selected = False for track_setup in track.track_setups: if track_setup.name in selected_setups: any_selected = True race_paths = paths.Paths(self.ctx.config) self.ctx.config.add( config.Scope.trackSetup, "system", "track.setup.root.dir", race_paths.track_setup_root(track.name, track_setup.name)) self.ctx.config.add( config.Scope.trackSetup, "system", "track.setup.log.dir", race_paths.track_setup_logs(track.name, track_setup.name)) print("Racing on track '%s' with setup '%s'" % (track.name, track_setup.name)) for step in self.steps: step.run(track, track_setup) else: logger.debug("Skipping track setup [%s] (not selected)." % track_setup.name) if not any_selected: raise exceptions.ImproperlyConfigured( "Unknown track setup(s) %s for track [%s]. You can list the available tracks and their " "track setups with esrally list tracks." % (selected_setups, track.name))
def log_file_path(cfg): log_dir = paths.Paths(cfg).log_root() node_name = cfg.opts("system", "node.name", mandatory=False) if node_name: return "%s/rally_out_%s.log" % (log_dir, node_name) else: return "%s/rally_out.log" % log_dir
def configure_actor_logging(cfg): class ActorLogFilter(logging.Filter): def filter(self, logrecord): return "actorAddress" in logrecord.__dict__ class NotActorLogFilter(logging.Filter): def filter(self, logrecord): return "actorAddress" not in logrecord.__dict__ log_dir = paths.Paths(cfg).log_root() return { "version": 1, "formatters": { "normal": { "format": "%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s" }, "actor": { "format": "%(asctime)s,%(msecs)d %(name)s %(levelname)s %(actorAddress)s => %(message)s" } }, "filters": { "isActorLog": { "()": ActorLogFilter }, "notActorLog": { "()": NotActorLogFilter } }, "handlers": { "h1": { "class": "logging.FileHandler", "filename": "%s/rally-actors.log" % log_dir, "formatter": "normal", "filters": ["notActorLog"], "level": logging.INFO }, "h2": { "class": "logging.FileHandler", "filename": "%s/rally-actor-messages.log" % log_dir, "formatter": "actor", "filters": ["isActorLog"], "level": logging.INFO }, }, "loggers": { "": { "handlers": ["h1", "h2"], "level": logging.INFO } } }
def sweep(self): invocation_root = self.cfg.opts("system", "invocation.root.dir") track_name = self.cfg.opts("benchmarks", "track") challenge_name = self.cfg.opts("benchmarks", "challenge") car_name = self.cfg.opts("benchmarks", "car") log_root = paths.Paths(self.cfg).log_root() archive_path = "%s/logs-%s-%s-%s.zip" % (invocation_root, track_name, challenge_name, car_name) io.compress(log_root, archive_path) console.println("") console.info("Archiving logs in %s" % archive_path) shutil.rmtree(log_root)
def __init__(self, cfg, supply, p, l): self._config = cfg self.supply = supply self.provisioner = p self.launcher = l # TODO dm: Check whether we can remove this completely # ensure we don't mix ES installs track_name = self._config.opts("benchmarks", "track") challenge_name = self._config.opts("benchmarks", "challenge") race_paths = paths.Paths(self._config) self._config.add(config.Scope.challenge, "system", "challenge.root.dir", race_paths.challenge_root(track_name, challenge_name)) self._config.add(config.Scope.challenge, "system", "challenge.log.dir", race_paths.challenge_logs(track_name, challenge_name))
def main(): pre_configure_logging() args = parse_args() print(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) # Add global meta info derived by rally itself cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "rally.root", rally_root_path()) cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root()) # Add command line config cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision) cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository) cfg.add(config.Scope.applicationOverride, "system", "track", args.track) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge) cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car) cfg.add(config.Scope.applicationOverride, "benchmarks", "rounds", args.rounds) cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", csv_to_list(args.target_hosts)) cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options))) cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format) cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file) if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender) configure_logging(cfg) logger.info("Rally version [%s]" % version()) logger.info("Command line arguments: %s" % args) success = dispatch_sub_command(cfg, sub_command) if not success: sys.exit(1)
def __init__(self, cfg): self._config = cfg self._supplier = supplier.Supplier(cfg) self._builder = builder.Builder(cfg) self._provisioner = provisioner.Provisioner(cfg) self._launcher = launcher.InProcessLauncher(cfg) self._metrics_store = None # TODO dm module refactoring: just moved it to the right place. Simplify (this should actually not be needed at all. It's just there # to ensure we don't mix ES installs) track_name = self._config.opts("system", "track") challenge_name = self._config.opts("benchmarks", "challenge") race_paths = paths.Paths(self._config) self._config.add(config.Scope.challenge, "system", "challenge.root.dir", race_paths.challenge_root(track_name, challenge_name)) self._config.add(config.Scope.challenge, "system", "challenge.log.dir", race_paths.challenge_logs(track_name, challenge_name))
def run(self, track): invocation_root = paths.Paths(self._config).invocation_root() log_root = paths.Paths(self._config).log_root() io.zip(log_root, "%s/logs-%s.zip" % (invocation_root, track.name)) shutil.rmtree(log_root)
def main(): start = time.time() # Early init of console output so we start to show everything consistently. console.init(quiet=False) pre_configure_logging() args = parse_args() console.init(quiet=args.quiet) console.println(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) # Add global meta info derived by rally itself cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "rally.root", rally_root_path()) cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root()) # Add command line config cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision) cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging) cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "benchmarks", "track", args.track) cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge) cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car) cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health", args.cluster_health) cfg.add(config.Scope.applicationOverride, "benchmarks", "laps", args.laps) cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", convert_hosts(csv_to_list(args.target_hosts))) cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options))) cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format) cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file) if args.override_src_dir is not None: cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir) if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender) configure_logging(cfg) logger.info("Rally version [%s]" % version()) logger.info("Command line arguments: %s" % args) # Configure networking net.init() if not args.offline: if not net.has_internet_connection(): console.warn( "No Internet connection detected. Automatic download of track data sets etc. is disabled.", logger=logger) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True) else: logger.info("Detected a working Internet connection.") # Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine # noinspection PyBroadException try: process.kill_running_rally_instances() except BaseException: logger.exception( "Could not terminate potentially running Rally instances correctly. Attempting to go on anyway." ) try: actors = bootstrap_actor_system(cfg) except RuntimeError as e: logger.exception("Could not bootstrap actor system.") if str(e) == "Unable to determine valid external socket address.": console.warn( "Could not determine a socket address. Are you running without any network?", logger=logger) actors = bootstrap_actor_system(cfg, system_base="multiprocQueueBase") else: raise success = False try: success = dispatch_sub_command(cfg, sub_command) finally: shutdown_complete = False times_interrupted = 0 while not shutdown_complete and times_interrupted < 2: try: logger.info("Attempting to shutdown internal actor system.") actors.shutdown() shutdown_complete = True logger.info("Shutdown completed.") except KeyboardInterrupt: times_interrupted += 1 logger.warn( "User interrupted shutdown of internal actor system.") console.info( "Please wait a moment for Rally's internal components to shutdown." ) if not shutdown_complete and times_interrupted > 0: logger.warn( "Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted) console.println("") console.warn( "Terminating now at the risk of leaving child processes behind." ) console.println("") console.warn("The next race may fail due to an unclean shutdown.") console.println("") console.println(SKULL) console.println("") end = time.time() if success: console.println("") console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-") else: console.println("") console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(64)
def configure_actor_logging(cfg): class ActorLogFilter(logging.Filter): def filter(self, logrecord): return "actorAddress" in logrecord.__dict__ class NotActorLogFilter(logging.Filter): def filter(self, logrecord): return "actorAddress" not in logrecord.__dict__ log_dir = paths.Paths(cfg).log_root() logging_output = cfg.opts("system", "logging.output") if logging_output == "file": actor_log_handler = { "class": "logging.FileHandler", "filename": "%s/rally-actors.log" % log_dir } actor_messages_handler = { "class": "logging.FileHandler", "filename": "%s/rally-actor-messages.log" % log_dir } else: actor_log_handler = { "class": "logging.StreamHandler", "stream": sys.stdout } actor_messages_handler = { "class": "logging.StreamHandler", "stream": sys.stdout } actor_log_handler["formatter"] = "normal" actor_log_handler["filters"] = ["notActorLog"] actor_log_handler["level"] = logging.INFO actor_messages_handler["formatter"] = "actor" actor_messages_handler["filters"] = ["isActorLog"] actor_messages_handler["level"] = logging.INFO return { "version": 1, "formatters": { "normal": { "format": "%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s" }, "actor": { "format": "%(asctime)s,%(msecs)d %(name)s %(levelname)s %(actorAddress)s => %(message)s" } }, "filters": { "isActorLog": { "()": ActorLogFilter }, "notActorLog": { "()": NotActorLogFilter } }, "handlers": { "h1": actor_log_handler, "h2": actor_messages_handler }, "loggers": { "": { "handlers": ["h1", "h2"], "level": logging.INFO } } }
def prepare_track(ctx, track): ctx.marshal.setup(track) race_paths = paths.Paths(ctx.config) track_root = race_paths.track_root(track.name) ctx.config.add(config.Scope.benchmark, "system", "track.root.dir", track_root)
def log_file_path(cfg): log_dir = paths.Paths(cfg).log_root() return "%s/rally_out.log" % log_dir
def main(): pre_configure_logging() args = parse_args() if not args.quiet: print(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) # Add global meta info derived by rally itself cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "rally.root", rally_root_path()) cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root()) # Add command line config cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision) cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository) cfg.add(config.Scope.applicationOverride, "system", "track", args.track) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge) cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car) cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", convert_hosts(csv_to_list(args.target_hosts))) cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options))) cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format) cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file) if args.override_src_dir is not None: cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir) if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender) configure_logging(cfg) logger.info("Rally version [%s]" % version()) logger.info("Command line arguments: %s" % args) # Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine try: process.kill_running_rally_instances() except BaseException: logger.exception( "Could not terminate potentially running Rally instances correctly. Attempting to go on anyway." ) # bootstrap Rally's Actor system try: actors = thespian.actors.ActorSystem( "multiprocTCPBase", logDefs=configure_actor_logging(cfg)) except thespian.actors.ActorSystemException: logger.exception( "Could not initialize internal actor system. Terminating.") print("ERROR: Could not initialize successfully.") print("") print( "The most likely cause is that there are still processes running from a previous race." ) print( "Please check for running Python processes and terminate them before running Rally again." ) print("") print_help_on_errors(cfg) sys.exit(70) success = False try: success = dispatch_sub_command(cfg, sub_command) finally: shutdown_complete = False times_interrupted = 0 while not shutdown_complete and times_interrupted < 2: try: actors.shutdown() shutdown_complete = True except KeyboardInterrupt: times_interrupted += 1 logger.warn( "User interrupted shutdown of internal actor system.") print( "Please wait a moment for Rally's internal components to shutdown." ) if not shutdown_complete and times_interrupted > 0: logger.warn( "Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted) print( "**********************************************************************" ) print("") print( "WARN: Terminating now at the risk of leaving child processes behind." ) print("") print("The next race may fail due to an unclean shutdown.") print("") print(SKULL) print("") print( "**********************************************************************" ) if not success: sys.exit(64)
def main(): preconfigure_logging() args = parse_args() cfg = config.Config(config_name=args.configuration_name) subcommand = derive_subcommand(args, cfg) if subcommand == "configure": cfg.create_config(advanced_config=args.advanced_config) exit(0) else: if cfg.config_present(): cfg.load_config() if not cfg.config_compatible(): # logger.info("Detected incompatible configuration file. Trying to upgrade.") cfg.migrate_config() # Reload config after upgrading cfg.load_config() else: print( "Error: No config present. Please run 'esrally configure' first." ) exit(64) # Add global meta info derived by rally itself cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "rally.root", os.path.dirname(os.path.realpath(__file__))) cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root()) # Add command line config cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision) cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "system", "track", args.track) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "benchmarks", "tracksetups.selected", csv_to_list(args.track_setup)) cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", args.preserve_install) cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", csv_to_list(args.target_hosts)) if subcommand == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) configure_logging(cfg) print_banner() race_control = racecontrol.RaceControl(cfg) success = race_control.start(subcommand) if not success: sys.exit(1)