def configure_track_params(arg_parser, args, cfg, command_requires_track=True): cfg.add(config.Scope.applicationOverride, "track", "repository.revision", args.track_revision) # We can assume here that if a track-path is given, the user did not specify a repository either (although argparse sets it to # its default value) if args.track_path: cfg.add(config.Scope.applicationOverride, "track", "track.path", os.path.abspath(io.normalize_path(args.track_path))) cfg.add(config.Scope.applicationOverride, "track", "repository.name", None) if args.track_revision: # stay as close as possible to argparse errors although we have a custom validation. arg_parser.error("argument --track-revision not allowed with argument --track-path") if command_requires_track and args.track: # stay as close as possible to argparse errors although we have a custom validation. arg_parser.error("argument --track not allowed with argument --track-path") else: cfg.add(config.Scope.applicationOverride, "track", "repository.name", args.track_repository) if command_requires_track: # TODO #1176: We should not choose a track implicitly. # set the default programmatically because we need to determine whether the user has provided a value if args.track: chosen_track = args.track else: chosen_track = "geonames" console.warn(f"Starting Rally without --track is deprecated. Add --track={chosen_track} to your parameters.") cfg.add(config.Scope.applicationOverride, "track", "track.name", chosen_track) if command_requires_track: cfg.add(config.Scope.applicationOverride, "track", "params", opts.to_dict(args.track_params)) cfg.add(config.Scope.applicationOverride, "track", "challenge.name", args.challenge) cfg.add(config.Scope.applicationOverride, "track", "include.tasks", opts.csv_to_list(args.include_tasks)) cfg.add(config.Scope.applicationOverride, "track", "exclude.tasks", opts.csv_to_list(args.exclude_tasks))
def _create_index(self, index_spec, mapping_dir): index_name = self._r(index_spec, "name") if self.override_auto_manage_indices is not None: auto_managed = self.override_auto_manage_indices logger.info( "User explicitly forced auto-managed indices to [%s] on the command line." % str(auto_managed)) else: auto_managed = self._r(index_spec, "auto-managed", mandatory=False, default_value=True) logger.info( "Using index auto-management setting from track which is set to [%s]." % str(auto_managed)) types = [ self._create_type(type_spec, mapping_dir) for type_spec in self._r( index_spec, "types", mandatory=auto_managed, default_value=[]) ] valid_document_data = False for type in types: if type.has_valid_document_data(): valid_document_data = True break if not valid_document_data: console.warn( "None of the types for index [%s] defines documents. Please check that you either don't want to index data or " "parameter sources are defined for indexing." % index_name, logger=logger) return track.Index(name=index_name, auto_managed=auto_managed, types=types)
def __init__(self, remote_url, root_dir, repo_name, resource_name, offline, fetch=True): # If no URL is found, we consider this a local only repo (but still require that it is a git repo) self.url = remote_url self.remote = self.url is not None and self.url.strip() != "" self.repo_dir = os.path.join(root_dir, repo_name) self.resource_name = resource_name self.offline = offline self.logger = logging.getLogger(__name__) if self.remote and not self.offline and fetch: # a normal git repo with a remote if not git.is_working_copy(self.repo_dir): git.clone(src=self.repo_dir, remote=self.url) else: try: git.fetch(src=self.repo_dir) except exceptions.SupplyError: console.warn( "Could not update %s. Continuing with your locally available state." % self.resource_name) else: if not git.is_working_copy(self.repo_dir): raise exceptions.SystemSetupError( "[{src}] must be a git repository.\n\nPlease run:\ngit -C {src} init" .format(src=self.repo_dir))
def start(self, node_configurations=None): hosts = self.cfg.opts("client", "hosts") client_options = self.cfg.opts("client", "options") es = self.client_factory(hosts, client_options).create() # cannot enable custom telemetry devices here t = telemetry.Telemetry(devices=[ # This is needed to actually populate the nodes telemetry.ClusterMetaDataInfo(es), # will gather node specific meta-data for all nodes telemetry.ExternalEnvironmentInfo(es, self.metrics_store), ]) # We create a pseudo-cluster here to get information about all nodes. # cluster nodes will be populated by the external environment info telemetry device. We cannot know this upfront. c = cluster.Cluster(hosts, [], t) user_defined_version = self.cfg.opts("mechanic", "distribution.version", mandatory=False) distribution_version = es.info()["version"]["number"] if not user_defined_version or user_defined_version.strip() == "": logger.info("Distribution version was not specified by user. Rally-determined version is [%s]" % distribution_version) self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version) elif user_defined_version != distribution_version: console.warn( "Specified distribution version '%s' on the command line differs from version '%s' reported by the cluster." % (user_defined_version, distribution_version), logger=logger) t.attach_to_cluster(c) return c.nodes
def start(self, car=None, binary=None, data_paths=None): console.println(ExternalLauncher.BOGUS_RESULTS_WARNING) hosts = self.cfg.opts("client", "hosts") client_options = self.cfg.opts("client", "options") es = self.client_factory(hosts, client_options).create() # cannot enable custom telemetry devices here t = telemetry.Telemetry(devices=[ telemetry.ClusterMetaDataInfo(es), telemetry.ExternalEnvironmentInfo(es, self.metrics_store), telemetry.NodeStats(es, self.metrics_store), telemetry.IndexStats(es, self.metrics_store) ]) # cluster nodes will be populated by the external environment info telemetry device. We cannot know this upfront. c = cluster.Cluster(hosts, [], t) user_defined_version = self.cfg.opts("mechanic", "distribution.version", mandatory=False) distribution_version = es.info()["version"]["number"] if not user_defined_version or user_defined_version.strip() == "": logger.info("Distribution version was not specified by user. Rally-determined version is [%s]" % distribution_version) self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version) elif user_defined_version != distribution_version: console.warn( "Specified distribution version '%s' on the command line differs from version '%s' reported by the cluster." % (user_defined_version, distribution_version), logger=logger) t.attach_to_cluster(c) return c
def _create_index(self, index_spec, mapping_dir, data_dir): index_name = self._r(index_spec, "name") auto_managed = self._r(index_spec, "auto-managed", mandatory=False, default_value=True) types = [ self._create_type(type_spec, mapping_dir, data_dir) for type_spec in self._r( index_spec, "types", mandatory=auto_managed, default_value=[]) ] valid_document_data = False for type in types: if type.has_valid_document_data(): valid_document_data = True break if not valid_document_data: console.warn( "None of the types for index [%s] defines documents. Please check that you either don't want to index data or " "parameter sources are defined for indexing." % index_name, logger=logger) return track.Index(name=index_name, auto_managed=auto_managed, types=types)
def report(self): print_header(FINAL_SCORE) stats = self.results warnings = [] metrics_table = [] metrics_table.extend(self.report_totals(stats)) metrics_table.extend(self.report_ml_processing_times(stats)) metrics_table.extend(self.report_gc_metrics(stats)) metrics_table.extend(self.report_disk_usage(stats)) metrics_table.extend(self.report_segment_memory(stats)) metrics_table.extend(self.report_segment_counts(stats)) for record in stats.op_metrics: task = record["task"] metrics_table.extend(self.report_throughput(record, task)) metrics_table.extend(self.report_latency(record, task)) metrics_table.extend(self.report_service_time(record, task)) # this is mostly needed for debugging purposes but not so relevant to end users if self.show_processing_time: metrics_table.extend(self.report_processing_time(record, task)) metrics_table.extend(self.report_error_rate(record, task)) self.add_warnings(warnings, record, task) self.write_report(metrics_table) if warnings: for warning in warnings: console.warn(warning)
def update(self, distribution_version): try: if self.remote: branch = versions.best_match( git.branches(self.repo_dir, remote=self.remote), distribution_version) if branch: # Allow uncommitted changes iff we do not have to change the branch self.logger.info( "Checking out [%s] in [%s] for distribution version [%s].", branch, self.repo_dir, distribution_version) git.checkout(self.repo_dir, branch=branch) self.logger.info( "Rebasing on [%s] in [%s] for distribution version [%s].", branch, self.repo_dir, distribution_version) try: git.rebase(self.repo_dir, branch=branch) self.revision = git.head_revision(self.repo_dir) except exceptions.SupplyError: self.logger.exception( "Cannot rebase due to local changes in [%s]", self.repo_dir) console.warn( "Local changes in [%s] prevent %s update from remote. Please commit your changes." % (self.repo_dir, self.resource_name)) return else: msg = "Could not find %s remotely for distribution version [%s]. Trying to find %s locally." % \ (self.resource_name, distribution_version, self.resource_name) self.logger.warning(msg) branch = versions.best_match( git.branches(self.repo_dir, remote=False), distribution_version) if branch: if git.current_branch(self.repo_dir) != branch: self.logger.info( "Checking out [%s] in [%s] for distribution version [%s].", branch, self.repo_dir, distribution_version) git.checkout(self.repo_dir, branch=branch) self.revision = git.head_revision(self.repo_dir) else: self.logger.info( "No local branch found for distribution version [%s] in [%s]. Checking tags.", distribution_version, self.repo_dir) tag = self._find_matching_tag(distribution_version) if tag: self.logger.info( "Checking out tag [%s] in [%s] for distribution version [%s].", tag, self.repo_dir, distribution_version) git.checkout(self.repo_dir, branch=tag) self.revision = git.head_revision(self.repo_dir) else: raise exceptions.SystemSetupError( "Cannot find %s for distribution version %s" % (self.resource_name, distribution_version)) except exceptions.SupplyError as e: tb = sys.exc_info()[2] raise exceptions.DataError("Cannot update %s in [%s] (%s)." % (self.resource_name, self.repo_dir, e.message)).with_traceback(tb)
def __init__(self, cfg, fetch=True): self.cfg = cfg self.name = cfg.opts("mechanic", "repository.name") self.offline = cfg.opts("system", "offline.mode") # If no URL is found, we consider this a local only repo (but still require that it is a git repo) self.url = cfg.opts("teams", "%s.url" % self.name, mandatory=False) self.remote = self.url is not None and self.url.strip() != "" root = cfg.opts("node", "root.dir") team_repositories = cfg.opts("mechanic", "team.repository.dir") self.teams_dir = os.path.join(root, team_repositories, self.name) if self.remote and not self.offline and fetch: # a normal git repo with a remote if not git.is_working_copy(self.teams_dir): git.clone(src=self.teams_dir, remote=self.url) else: try: git.fetch(src=self.teams_dir) except exceptions.SupplyError: console.warn( "Could not update teams. Continuing with your locally available state.", logger=logger) else: if not git.is_working_copy(self.teams_dir): raise exceptions.SystemSetupError( "[{src}] must be a git repository.\n\nPlease run:\ngit -C {src} init" .format(src=self.teams_dir))
def start(self, node_configurations=None): hosts = self.cfg.opts("client", "hosts").default client_options = self.cfg.opts("client", "options").default es = self.client_factory(hosts, client_options).create() # cannot enable custom telemetry devices here t = telemetry.Telemetry(devices=[ # This is needed to actually populate the nodes telemetry.ClusterMetaDataInfo(es), # will gather node specific meta-data for all nodes telemetry.ExternalEnvironmentInfo(es, self.metrics_store), ]) # We create a pseudo-cluster here to get information about all nodes. # cluster nodes will be populated by the external environment info telemetry device. We cannot know this upfront. c = cluster.Cluster(hosts, [], t) user_defined_version = self.cfg.opts("mechanic", "distribution.version", mandatory=False) distribution_version = es.info()["version"]["number"] if not user_defined_version or user_defined_version.strip() == "": self.logger.info("Distribution version was not specified by user. Rally-determined version is [%s]", distribution_version) self.cfg.add(config.Scope.benchmark, "mechanic", "distribution.version", distribution_version) elif user_defined_version != distribution_version: console.warn( "Specified distribution version '%s' on the command line differs from version '%s' reported by the cluster." % (user_defined_version, distribution_version), logger=self.logger) t.attach_to_cluster(c) return c.nodes
def report(self): if self.is_final_report(): print_internal("") print_header( "------------------------------------------------------") print_header( " _______ __ _____ ") print_header( " / ____(_)___ ____ _/ / / ___/_________ ________ ") print_header( " / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \\") print_header( " / __/ / / / / / /_/ / / ___/ / /__/ /_/ / / / __/") print_header( "/_/ /_/_/ /_/\__,_/_/ /____/\___/\____/_/ \___/ ") print_header( "------------------------------------------------------") print_internal("") else: print_internal("") print_header("--------------------------------------------------") print_header(" __ _____ ") print_header(" / / ____ _____ / ___/_________ ________ ") print_header(" / / / __ `/ __ \ \__ \/ ___/ __ \/ ___/ _ \\") print_header(" / /___/ /_/ / /_/ / ___/ / /__/ /_/ / / / __/") print_header("/_____/\__,_/ .___/ /____/\___/\____/_/ \___/ ") print_header(" /_/ ") print_header("--------------------------------------------------") print_internal("") stats = self.results warnings = [] metrics_table = [] meta_info_table = [] metrics_table += self.report_total_times(stats) metrics_table += self.report_merge_part_times(stats) metrics_table += self.report_cpu_usage(stats) metrics_table += self.report_gc_times(stats) metrics_table += self.report_disk_usage(stats) metrics_table += self.report_segment_memory(stats) metrics_table += self.report_segment_counts(stats) for record in stats.op_metrics: operation = record["operation"] metrics_table += self.report_throughput(record, operation) metrics_table += self.report_latency(record, operation) metrics_table += self.report_service_time(record, operation) metrics_table += self.report_error_rate(record, operation) self.add_warnings(warnings, record, operation) meta_info_table += self.report_meta_info() self.write_report(metrics_table, meta_info_table) if warnings: for warning in warnings: console.warn(warning, logger=logger)
def teardown(self, cancelled=False, error=False): logger.info("Asking mechanic to stop the engine.") result = self.actor_system.ask(self.mechanic, mechanic.StopEngine()) if isinstance(result, mechanic.EngineStopped): logger.info("Mechanic has stopped engine successfully.") logger.info("Bulk adding system metrics to metrics store.") self.metrics_store.bulk_add(result.system_metrics) elif isinstance(result, mechanic.Failure): logger.info("Stopping engine has failed. Reason [%s]." % result.message) raise exceptions.RallyError(result.message, result.cause) else: if self.ignore_unknown_return: console.warn( "Mechanic has not stopped engine but instead [%s]. Ignoring." % str(result), logger=logger) else: raise exceptions.RallyError( "Mechanic has not stopped engine but instead [%s]. Terminating race without result." % str(result)) self.metrics_store.flush() if not cancelled and not error: final_results = reporter.calculate_results(self.metrics_store, self.race) self.race.add_final_results(final_results) reporter.summarize(self.race, self.cfg) self.race_store.store_race(self.race) else: logger.info( "Suppressing output of summary report. Cancelled = [%r], Error = [%r]." % (cancelled, error)) self.metrics_store.close()
def _update(self, distribution_version): try: if self.remote and not self.offline: branch = versions.best_match(git.branches(self.tracks_dir, remote=self.remote), distribution_version) if branch: # Allow uncommitted changes iff we do not have to change the branch logger.info( "Checking out [%s] in [%s] for distribution version [%s]." % (branch, self.tracks_dir, distribution_version)) git.checkout(self.tracks_dir, branch=branch) logger.info("Rebasing on [%s] in [%s] for distribution version [%s]." % (branch, self.tracks_dir, distribution_version)) try: git.rebase(self.tracks_dir, branch=branch) except exceptions.SupplyError: logger.exception("Cannot rebase due to local changes in [%s]" % self.tracks_dir) console.warn( "Local changes in [%s] prevent track update from remote. Please commit your changes." % self.tracks_dir) return else: msg = "Could not find track data remotely for distribution version [%s]. " \ "Trying to find track data locally." % distribution_version logger.warn(msg) branch = versions.best_match(git.branches(self.tracks_dir, remote=False), distribution_version) if branch: logger.info("Checking out [%s] in [%s] for distribution version [%s]." % (branch, self.tracks_dir, distribution_version)) git.checkout(self.tracks_dir, branch=branch) else: raise exceptions.SystemSetupError("Cannot find track data for distribution version %s" % distribution_version) except exceptions.SupplyError: tb = sys.exc_info()[2] raise exceptions.DataError("Cannot update track data in [%s]." % self.tracks_dir).with_traceback(tb)
def from_sources_skip_build(cfg): console.warn( "The pipeline from-sources-skip-build is deprecated. Rally caches artifacts now automatically. " "Use the pipeline \"from-sources\" instead") port = cfg.opts("provisioning", "node.http.port") set_default_hosts(cfg, port=port) return race(cfg, sources=True, build=False)
def report(self): if self.is_final_report(): print_header(FINAL_SCORE) else: print_header(LAP_SCORE) stats = self.results warnings = [] metrics_table = [] metrics_table.extend(self.report_totals(stats)) metrics_table.extend(self.report_merge_part_times(stats)) metrics_table.extend(self.report_ml_processing_times(stats)) metrics_table.extend(self.report_gc_times(stats)) metrics_table.extend(self.report_disk_usage(stats)) metrics_table.extend(self.report_segment_memory(stats)) metrics_table.extend(self.report_segment_counts(stats)) for record in stats.op_metrics: task = record["task"] metrics_table.extend(self.report_throughput(record, task)) metrics_table.extend(self.report_latency(record, task)) metrics_table.extend(self.report_service_time(record, task)) metrics_table.extend(self.report_error_rate(record, task)) self.add_warnings(warnings, record, task) self.write_report(metrics_table) if warnings: for warning in warnings: console.warn(warning)
def _update(self, distribution_version): try: if self.remote and not self.offline: branch = versions.best_match(git.branches(self.tracks_dir, remote=self.remote), distribution_version) if branch: # Allow uncommitted changes iff we do not have to change the branch logger.info( "Checking out [%s] in [%s] for distribution version [%s]." % (branch, self.tracks_dir, distribution_version)) git.checkout(self.tracks_dir, branch=branch) logger.info("Rebasing on [%s] in [%s] for distribution version [%s]." % (branch, self.tracks_dir, distribution_version)) try: git.rebase(self.tracks_dir, branch=branch) except exceptions.SupplyError: logger.exception("Cannot rebase due to local changes in [%s]" % self.tracks_dir) console.warn( "Local changes in [%s] prevent track update from remote. Please commit your changes." % self.tracks_dir) return else: msg = "Could not find track data remotely for distribution version [%s]. " \ "Trying to find track data locally." % distribution_version logger.warning(msg) branch = versions.best_match(git.branches(self.tracks_dir, remote=False), distribution_version) if branch: logger.info("Checking out [%s] in [%s] for distribution version [%s]." % (branch, self.tracks_dir, distribution_version)) git.checkout(self.tracks_dir, branch=branch) else: raise exceptions.SystemSetupError("Cannot find track data for distribution version %s" % distribution_version) except exceptions.SupplyError: tb = sys.exc_info()[2] raise exceptions.DataError("Cannot update track data in [%s]." % self.tracks_dir).with_traceback(tb)
def param_source_for_name(name, track, params): param_source = __PARAM_SOURCES_BY_NAME[name] # we'd rather use callable() but this will erroneously also classify a class as callable... if isinstance(param_source, types.FunctionType): # TODO: Remove me after some grace period try: s = inspect.signature(param_source, follow_wrapped=False) except TypeError: # follow_wrapped has been introduced in Python 3.5 s = inspect.signature(param_source) if len(s.parameters) == 2 and s.parameters.get("indices"): console.warn("Parameter source '%s' is using deprecated method signature (indices, params). Please change it " "to (track, params, **kwargs). For details please see the migration guide in the docs." % name, logger=logger) return LegacyDelegatingParamSource(track, params, param_source) else: return DelegatingParamSource(track, params, param_source) else: try: s = inspect.signature(param_source.__init__, follow_wrapped=False) except TypeError: # follow_wrapped has been introduced in Python 3.5 s = inspect.signature(param_source) # self, indices, params if len(s.parameters) == 3 and s.parameters.get("indices"): console.warn("Parameter source '%s' is using deprecated method signature (indices, params). Please change it " "to (track, params, **kwargs). For details please see the migration guide in the docs." % name, logger=logger) return param_source(track.indices, params) else: return param_source(track, params)
def parse_operation(self, op_spec, error_ctx="operations"): # just a name, let's assume it is a simple operation like force-merge and create a full operation if isinstance(op_spec, str): op_name = op_spec meta_data = None op_type_name = op_spec param_source = None else: meta_data = self._r(op_spec, "meta", error_ctx=error_ctx, mandatory=False) # Rally's core operations will still use enums then but we'll allow users to define arbitrary operations op_type_name = self._r(op_spec, "operation-type", error_ctx=error_ctx) # fallback to use the operation type as the operation name op_name = self._r(op_spec, "name", error_ctx=error_ctx, mandatory=False, default_value=op_type_name) param_source = self._r(op_spec, "param-source", error_ctx=error_ctx, mandatory=False) try: # TODO #370: Remove this warning. # Add a deprecation warning but not for built-in tracks (they need to keep the name for backwards compatibility in the meantime) if op_type_name == "index" and \ self.name not in ["geonames", "geopoint", "noaa", "logging", "nyc_taxis", "pmc", "percolator", "nested"] and \ not self.index_op_type_warning_issued: console.warn( "The track %s uses the deprecated operation-type [index] for bulk index operations. Please rename this " "operation type to [bulk]." % self.name) # Don't spam the console... self.index_op_type_warning_issued = True op_type = track.OperationType.from_hyphenated_string( op_type_name).name logger.debug( "Using built-in operation type [%s] for operation [%s]." % (op_type, op_name)) except KeyError: logger.info( "Using user-provided operation type [%s] for operation [%s]." % (op_type_name, op_name)) op_type = op_type_name try: return track.Operation(name=op_name, meta_data=meta_data, operation_type=op_type, params=op_spec, param_source=param_source) except exceptions.InvalidSyntax as e: raise TrackSyntaxError("Invalid operation [%s]: %s" % (op_name, str(e)))
def report(self): if self.is_final_report(): print_internal("") print_header( "------------------------------------------------------") print_header( " _______ __ _____ ") print_header( " / ____(_)___ ____ _/ / / ___/_________ ________ ") print_header( " / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \\") print_header( " / __/ / / / / / /_/ / / ___/ / /__/ /_/ / / / __/") print_header( "/_/ /_/_/ /_/\__,_/_/ /____/\___/\____/_/ \___/ ") print_header( "------------------------------------------------------") print_internal("") else: print_internal("") print_header("--------------------------------------------------") print_header(" __ _____ ") print_header(" / / ____ _____ / ___/_________ ________ ") print_header(" / / / __ `/ __ \ \__ \/ ___/ __ \/ ___/ _ \\") print_header(" / /___/ /_/ / /_/ / ___/ / /__/ /_/ / / / __/") print_header("/_____/\__,_/ .___/ /____/\___/\____/_/ \___/ ") print_header(" /_/ ") print_header("--------------------------------------------------") print_internal("") stats = self.results warnings = [] metrics_table = [] metrics_table.extend(self.report_total_times(stats)) metrics_table.extend(self.report_merge_part_times(stats)) metrics_table.extend(self.report_ml_max_processing_time(stats)) metrics_table.extend(self.report_cpu_usage(stats)) metrics_table.extend(self.report_gc_times(stats)) metrics_table.extend(self.report_disk_usage(stats)) metrics_table.extend(self.report_segment_memory(stats)) metrics_table.extend(self.report_segment_counts(stats)) for record in stats.op_metrics: task = record["task"] metrics_table.extend(self.report_throughput(record, task)) metrics_table.extend(self.report_latency(record, task)) metrics_table.extend(self.report_service_time(record, task)) metrics_table.extend(self.report_error_rate(record, task)) self.add_warnings(warnings, record, task) self.write_report(metrics_table) if warnings: for warning in warnings: console.warn(warning)
def derive_sub_command(args): sub_command = args.subcommand if sub_command is None: logger = logging.getLogger(__name__) console.warn("Invoking Rally without a subcommand is deprecated and will be required with Rally 2.1.0. Specify " "the 'race' subcommand explicitly.", logger=logger) return "race" else: return sub_command
def on_benchmark_start(self): # we only determine this value at the start of the benchmark (in the first lap). This is actually only useful for # the pipeline "benchmark-only" where we don't have control over the cluster and the user might not have restarted # the cluster so we can at least tell them. if self.first_time: index_times = self.index_times(self.primaries_index_stats()) for k, v in index_times.items(): if v > 0: console.warn("%s is %d ms indicating that the cluster is not in a defined clean state. Recorded index time " "metrics may be misleading." % (k, v), logger=logger) self.first_time = False
def report(self): if self.is_final_report(): print_internal("") print_header("------------------------------------------------------") print_header(" _______ __ _____ ") print_header(" / ____(_)___ ____ _/ / / ___/_________ ________ ") print_header(" / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \\") print_header(" / __/ / / / / / /_/ / / ___/ / /__/ /_/ / / / __/") print_header("/_/ /_/_/ /_/\__,_/_/ /____/\___/\____/_/ \___/ ") print_header("------------------------------------------------------") print_internal("") else: print_internal("") print_header("--------------------------------------------------") print_header(" __ _____ ") print_header(" / / ____ _____ / ___/_________ ________ ") print_header(" / / / __ `/ __ \ \__ \/ ___/ __ \/ ___/ _ \\") print_header(" / /___/ /_/ / /_/ / ___/ / /__/ /_/ / / / __/") print_header("/_____/\__,_/ .___/ /____/\___/\____/_/ \___/ ") print_header(" /_/ ") print_header("--------------------------------------------------") print_internal("") stats = self.results warnings = [] metrics_table = [] meta_info_table = [] metrics_table += self.report_total_times(stats) metrics_table += self.report_merge_part_times(stats) metrics_table += self.report_cpu_usage(stats) metrics_table += self.report_gc_times(stats) metrics_table += self.report_disk_usage(stats) metrics_table += self.report_segment_memory(stats) metrics_table += self.report_segment_counts(stats) for record in stats.op_metrics: operation = record["operation"] metrics_table += self.report_throughput(record, operation) metrics_table += self.report_latency(record, operation) metrics_table += self.report_service_time(record, operation) metrics_table += self.report_error_rate(record, operation) self.add_warnings(warnings, record, operation) meta_info_table += self.report_meta_info() self.write_report(metrics_table, meta_info_table) if warnings: for warning in warnings: console.warn(warning, logger=logger)
def main(): check_python_version() log.install_default_log_config() log.configure_logging() logger = logging.getLogger(__name__) start = time.time() # Early init of console output so we start to show everything consistently. console.init(quiet=False) arg_parser = create_arg_parser() args = arg_parser.parse_args() console.init(quiet=args.quiet) console.println(BANNER) cfg = config.Config(config_name=args.configuration_name) if not cfg.config_present(): cfg.install_default_config() cfg.load_config(auto_upgrade=True) cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.utcnow()) # Local config per node cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root()) cfg.add(config.Scope.application, "node", "rally.cwd", os.getcwd()) logger.info("OS [%s]", str(platform.uname())) logger.info("Python [%s]", str(sys.implementation)) logger.info("Rally version [%s]", version.version()) logger.debug("Command line arguments: %s", args) # Configure networking net.init() if not args.offline: probing_url = cfg.opts("system", "probing.url", default_value="https://github.com", mandatory=False) if not net.has_internet_connection(probing_url): console.warn("No Internet connection detected. Automatic download of track data sets etc. is disabled.", logger=logger) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True) else: logger.info("Detected a working Internet connection.") result = dispatch_sub_command(arg_parser, args, cfg) end = time.time() if result == ExitStatus.SUCCESSFUL: console.println("") console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-") elif result == ExitStatus.INTERRUPTED: console.println("") console.info("ABORTED (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(130) elif result == ExitStatus.ERROR: console.println("") console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(64)
def __init__(self, track, params, **kwargs): super().__init__(track, params, **kwargs) self.request_params = params.get("request-params", {}) self.index_definitions = [] if track.indices: filter_idx = params.get("index") if isinstance(filter_idx, str): filter_idx = [filter_idx] settings = params.get("settings") for idx in track.indices: if not filter_idx or idx.name in filter_idx: body = idx.body if body and settings: if "settings" in body: # merge (and potentially override) body["settings"].update(settings) else: body["settings"] = settings elif not body: # this is just needed because we will output this in the middle of the benchmark and will thus write # this on the same line as the progress message. console.println("") console.warn( "Creating index %s based on deprecated type mappings. Please specify an index body instead. " "For details please see the migration guide in the docs." % idx.name, logger=logger) # TODO #366: Deprecate this syntax. We should only specify all mappings in the body property. # check all types and merge their mappings body = {"mappings": {}} if settings: body["settings"] = settings for t in idx.types: body["mappings"].update(t.mapping) self.index_definitions.append((idx.name, body)) else: try: # only 'index' is mandatory, the body is optional (may be ok to create an index without a body) idx = params["index"] body = params.get("body") if isinstance(idx, str): idx = [idx] for i in idx: self.index_definitions.append((i, body)) except KeyError: raise exceptions.InvalidSyntax( "Please set the property 'index' for the create-index operation" )
def _create_index(self, index_spec, mapping_dir, data_dir): index_name = self._r(index_spec, "name") auto_managed = self._r(index_spec, "auto-managed", mandatory=False, default_value=True) types = [self._create_type(type_spec, mapping_dir, data_dir) for type_spec in self._r(index_spec, "types", mandatory=auto_managed, default_value=[])] valid_document_data = False for type in types: if type.has_valid_document_data(): valid_document_data = True break if not valid_document_data: console.warn("None of the types for index [%s] defines documents. Please check that you either don't want to index data or " "parameter sources are defined for indexing." % index_name, logger=logger) return track.Index(name=index_name, auto_managed=auto_managed, types=types)
def _track_info(self, track_specification, name, mandatory=True): # TODO dm: Remove this backwards compatibility layer v_root = self._r(track_specification, name, mandatory=False) if not v_root: v_meta = self._r(track_specification, ["meta", name], mandatory=False) if not v_meta and mandatory: # issue warning already on the intended level (don't include "meta" here) self._error("Mandatory element '%s' is missing." % name) if v_meta: # do not warn users for tracks in the 'default' repository. It is maintained by us and we will keep the meta # block around for a little while after the release to give users a chance to upgrade. if self.name not in ["geonames", "geopoint", "nyc_taxis", "pmc", "logging", "tiny", "percolator"]: console.warn("Track '%s' uses deprecated syntax: The element '%s' is contained in a meta-block. Please move this " "element to the top-level." % (self.name, name)) return v_meta else: return v_root
def with_actor_system(runnable, cfg): already_running = actor.actor_system_already_running() logger.info("Actor system already running locally? [%s]" % str(already_running)) try: actors = actor.bootstrap_actor_system(try_join=already_running, prefer_local_only=not already_running) # We can only support remote benchmarks if we have a dedicated daemon that is not only bound to 127.0.0.1 cfg.add(config.Scope.application, "system", "remote.benchmarking.supported", already_running) except RuntimeError as e: logger.exception("Could not bootstrap actor system.") if str(e) == "Unable to determine valid external socket address.": console.warn("Could not determine a socket address. Are you running without any network? Switching to degraded mode.", logger=logger) actor.use_offline_actor_system() actors = actor.bootstrap_actor_system(try_join=True) else: raise try: runnable(cfg) finally: # We only shutdown the actor system if it was not already running before if not already_running: shutdown_complete = False times_interrupted = 0 while not shutdown_complete and times_interrupted < 2: try: logger.info("Attempting to shutdown internal actor system.") actors.shutdown() # note that this check will only evaluate to True for a TCP-based actor system. timeout = 15 while actor.actor_system_already_running() and timeout > 0: logger.info("Actor system is still running. Waiting...") time.sleep(1) timeout -= 1 if timeout > 0: shutdown_complete = True logger.info("Shutdown completed.") else: logger.warning("Shutdown timed out. Actor system is still running.") break except KeyboardInterrupt: times_interrupted += 1 logger.warning("User interrupted shutdown of internal actor system.") console.info("Please wait a moment for Rally's internal components to shutdown.") if not shutdown_complete and times_interrupted > 0: logger.warning("Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted) console.println("") console.warn("Terminating now at the risk of leaving child processes behind.") console.println("") console.warn("The next race may fail due to an unclean shutdown.") console.println("") console.println(SKULL) console.println("") elif not shutdown_complete: console.warn("Could not terminate all internal processes within timeout. Please check and force-terminate all Rally processes.")
def with_actor_system(runnable, cfg): already_running = actor.actor_system_already_running() logger.info("Actor system already running locally? [%s]" % str(already_running)) try: actors = actor.bootstrap_actor_system(try_join=already_running, prefer_local_only=not already_running) # We can only support remote benchmarks if we have a dedicated daemon that is not only bound to 127.0.0.1 cfg.add(config.Scope.application, "system", "remote.benchmarking.supported", already_running) except RuntimeError as e: logger.exception("Could not bootstrap actor system.") if str(e) == "Unable to determine valid external socket address.": console.warn("Could not determine a socket address. Are you running without any network? Switching to degraded mode.", logger=logger) actor.use_offline_actor_system() actors = actor.bootstrap_actor_system(try_join=True) else: raise try: runnable(cfg) finally: # We only shutdown the actor system if it was not already running before if not already_running: shutdown_complete = False times_interrupted = 0 while not shutdown_complete and times_interrupted < 2: try: logger.info("Attempting to shutdown internal actor system.") actors.shutdown() # note that this check will only evaluate to True for a TCP-based actor system. timeout = 15 while actor.actor_system_already_running() and timeout > 0: logger.info("Actor system is still running. Waiting...") time.sleep(1) timeout -= 1 if timeout > 0: shutdown_complete = True logger.info("Shutdown completed.") else: logger.warning("Shutdown timed out. Actor system is still running.") break except KeyboardInterrupt: times_interrupted += 1 logger.warning("User interrupted shutdown of internal actor system.") console.info("Please wait a moment for Rally's internal components to shutdown.") if not shutdown_complete and times_interrupted > 0: logger.warning("Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted) console.println("") console.warn("Terminating now at the risk of leaving child processes behind.") console.println("") console.warn("The next race may fail due to an unclean shutdown.") console.println("") console.println(SKULL) console.println("") elif not shutdown_complete: console.warn("Could not terminate all internal processes within timeout. Please check and force-terminate all Rally processes.")
def __init__(self, remote_url, root_dir, repo_name, resource_name, offline, fetch=True): # If no URL is found, we consider this a local only repo (but still require that it is a git repo) self.url = remote_url self.remote = self.url is not None and self.url.strip() != "" self.repo_dir = os.path.join(root_dir, repo_name) self.resource_name = resource_name self.offline = offline if self.remote and not self.offline and fetch: # a normal git repo with a remote if not git.is_working_copy(self.repo_dir): git.clone(src=self.repo_dir, remote=self.url) else: try: git.fetch(src=self.repo_dir) except exceptions.SupplyError: console.warn("Could not update %s. Continuing with your locally available state." % self.resource_name, logger=logger) else: if not git.is_working_copy(self.repo_dir): raise exceptions.SystemSetupError("[{src}] must be a git repository.\n\nPlease run:\ngit -C {src} init" .format(src=self.repo_dir))
def _create_index(self, index_spec, mapping_dir): index_name = self._r(index_spec, "name") if self.override_auto_manage_indices is not None: auto_managed = self.override_auto_manage_indices logger.info("User explicitly forced auto-managed indices to [%s] on the command line." % str(auto_managed)) else: auto_managed = self._r(index_spec, "auto-managed", mandatory=False, default_value=True) logger.info("Using index auto-management setting from track which is set to [%s]." % str(auto_managed)) types = [self._create_type(type_spec, mapping_dir) for type_spec in self._r(index_spec, "types", mandatory=auto_managed, default_value=[])] valid_document_data = False for type in types: if type.has_valid_document_data(): valid_document_data = True break if not valid_document_data: console.warn("None of the types for index [%s] defines documents. Please check that you either don't want to index data or " "parameter sources are defined for indexing." % index_name, logger=logger) return track.Index(name=index_name, auto_managed=auto_managed, types=types)
def receiveMsg_StartEngine(self, msg, sender): self.logger.info("Received signal from race control to start engine.") self.race_control = sender self.cfg = msg.cfg cls = metrics.metrics_store_class(self.cfg) self.metrics_store = cls(self.cfg) self.metrics_store.open(ctx=msg.open_metrics_context) self.car, _ = load_team(self.cfg, msg.external) self.team_revision = self.cfg.opts("mechanic", "repository.revision") # In our startup procedure we first create all mechanics. Only if this succeeds we'll continue. hosts = self.cfg.opts("client", "hosts").default if len(hosts) == 0: raise exceptions.LaunchError("No target hosts are configured.") self.externally_provisioned = msg.external if self.externally_provisioned: self.logger.info("Cluster will not be provisioned by Rally.") # TODO: This needs to be handled later - we should probably disallow this entirely if msg.cluster_settings: pretty_settings = json.dumps(msg.cluster_settings, indent=2) warning = "Ensure that these settings are defined in elasticsearch.yml:\n\n{}\n\nIf they are absent, running this track " \ "will fail or lead to unexpected results.".format(pretty_settings) console.warn(warning, logger=self.logger) self.status = "nodes_started" self.received_responses = [] self.on_all_nodes_started() self.status = "cluster_started" else: console.info("Preparing for race ...", flush=True) self.logger.info( "Cluster consisting of %s will be provisioned by Rally.", hosts) msg.hosts = hosts # Initialize the children array to have the right size to # ensure waiting for all responses self.children = [None] * len(nodes_by_host(to_ip_port(hosts))) self.send(self.createActor(Dispatcher), msg) self.status = "starting" self.received_responses = []
def receiveMsg_StartEngine(self, msg, sender): self.logger.info("Received signal from race control to start engine.") self.race_control = sender self.cfg = msg.cfg cls = metrics.metrics_store_class(self.cfg) self.metrics_store = cls(self.cfg) self.metrics_store.open(ctx=msg.open_metrics_context) name = self.cfg.opts("race", "pipeline") self.car, _ = load_team(self.cfg, msg.external) self.team_revision = self.cfg.opts("mechanic", "repository.revision") # In our startup procedure we first create all mechanics. Only if this succeeds we'll continue. hosts = self.cfg.opts("client", "hosts").default if len(hosts) == 0: raise exceptions.LaunchError("No target hosts are configured.") if msg.external: self.logger.info("Cluster will not be provisioned by Rally.") if msg.cluster_settings: pretty_settings = json.dumps(msg.cluster_settings, indent=2) warning = "Ensure that these settings are defined in elasticsearch.yml:\n\n{}\n\nIf they are absent, running this track " \ "will fail or lead to unexpected results.".format(pretty_settings) console.warn(warning, logger=self.logger) # just create one actor for this special case and run it on the coordinator node (i.e. here) m = self.createActor(NodeMechanicActor, targetActorRequirements={"coordinator": True}) self.children.append(m) self.send(m, msg.for_nodes(ip=hosts)) else: console.info("Preparing for race ...", flush=True) self.logger.info( "Cluster consisting of %s will be provisioned by Rally.", hosts) msg.hosts = hosts # Initialize the children array to have the right size to # ensure waiting for all responses self.children = [None] * len(nodes_by_host(to_ip_port(hosts))) self.send(self.createActor(Dispatcher), msg) self.status = "starting" self.received_responses = []
def run_async(cfg): console.warn("The race-async command is experimental.") logger = logging.getLogger(__name__) # We'll use a special car name for external benchmarks. cfg.add(config.Scope.benchmark, "mechanic", "car.names", ["external"]) coordinator = BenchmarkCoordinator(cfg) try: coordinator.setup() race_driver = driver.AsyncDriver(cfg, coordinator.current_track, coordinator.current_challenge) distribution_flavor, distribution_version, revision = race_driver.setup( ) coordinator.on_preparation_complete(distribution_flavor, distribution_version, revision) new_metrics = race_driver.run() coordinator.on_benchmark_complete(new_metrics) except KeyboardInterrupt: logger.info("User has cancelled the benchmark.") except BaseException as e: tb = sys.exc_info()[2] raise exceptions.RallyError(str(e)).with_traceback(tb)
def main(): check_python_version() log.install_default_log_config() log.configure_logging() logger = logging.getLogger(__name__) start = time.time() # Early init of console output so we start to show everything consistently. console.init(quiet=False) arg_parser = create_arg_parser() args = arg_parser.parse_args() console.init(quiet=args.quiet) console.println(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) if args.effective_start_date: cfg.add(config.Scope.application, "system", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "time.start.user_provided", True) else: cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.utcnow()) cfg.add(config.Scope.application, "system", "time.start.user_provided", False) cfg.add(config.Scope.applicationOverride, "system", "trial.id", str(uuid.uuid4())) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) # Local config per node cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root()) cfg.add(config.Scope.application, "node", "rally.cwd", os.getcwd()) cfg.add(config.Scope.applicationOverride, "mechanic", "source.revision", args.revision) if args.distribution_version: cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "mechanic", "car.names", opts.csv_to_list(args.car)) if args.team_path: cfg.add(config.Scope.applicationOverride, "mechanic", "team.path", os.path.abspath(io.normalize_path(args.team_path))) cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name", None) else: cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name", args.team_repository) cfg.add(config.Scope.applicationOverride, "mechanic", "car.plugins", opts.csv_to_list(args.elasticsearch_plugins)) cfg.add(config.Scope.applicationOverride, "mechanic", "car.params", opts.to_dict(args.car_params)) cfg.add(config.Scope.applicationOverride, "mechanic", "plugin.params", opts.to_dict(args.plugin_params)) if args.keep_cluster_running: cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running", True) # force-preserve the cluster nodes. cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install", True) else: cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running", False) cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "mechanic", "runtime.jdk", args.runtime_jdk) cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.devices", opts.csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.params", opts.to_dict(args.telemetry_params)) cfg.add(config.Scope.applicationOverride, "race", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "race", "laps", args.laps) cfg.add(config.Scope.applicationOverride, "race", "user.tag", args.user_tag) # We can assume here that if a track-path is given, the user did not specify a repository either (although argparse sets it to # its default value) if args.track_path: cfg.add(config.Scope.applicationOverride, "track", "track.path", os.path.abspath(io.normalize_path(args.track_path))) cfg.add(config.Scope.applicationOverride, "track", "repository.name", None) if args.track: # stay as close as possible to argparse errors although we have a custom validation. arg_parser.error( "argument --track not allowed with argument --track-path") # cfg.add(config.Scope.applicationOverride, "track", "track.name", None) else: # cfg.add(config.Scope.applicationOverride, "track", "track.path", None) cfg.add(config.Scope.applicationOverride, "track", "repository.name", args.track_repository) # set the default programmatically because we need to determine whether the user has provided a value chosen_track = args.track if args.track else "geonames" cfg.add(config.Scope.applicationOverride, "track", "track.name", chosen_track) cfg.add(config.Scope.applicationOverride, "track", "params", opts.to_dict(args.track_params)) cfg.add(config.Scope.applicationOverride, "track", "challenge.name", args.challenge) cfg.add(config.Scope.applicationOverride, "track", "include.tasks", opts.csv_to_list(args.include_tasks)) cfg.add(config.Scope.applicationOverride, "track", "test.mode.enabled", args.test_mode) cfg.add(config.Scope.applicationOverride, "reporting", "format", args.report_format) cfg.add(config.Scope.applicationOverride, "reporting", "values", args.show_in_report) cfg.add(config.Scope.applicationOverride, "reporting", "output.path", args.report_file) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "reporting", "baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "reporting", "contender.timestamp", args.contender) if sub_command == "generate": cfg.add(config.Scope.applicationOverride, "generator", "chart.type", args.chart_type) cfg.add(config.Scope.applicationOverride, "generator", "output.path", args.output_path) if args.chart_spec_path and (args.track or args.challenge or args.car or args.node_count): console.println( "You need to specify either --chart-spec-path or --track, --challenge, --car and " "--node-count but not both.") exit(1) if args.chart_spec_path: cfg.add(config.Scope.applicationOverride, "generator", "chart.spec.path", args.chart_spec_path) else: # other options are stored elsewhere already cfg.add(config.Scope.applicationOverride, "generator", "node.count", args.node_count) cfg.add(config.Scope.applicationOverride, "driver", "profiling", args.enable_driver_profiling) cfg.add(config.Scope.applicationOverride, "driver", "on.error", args.on_error) cfg.add(config.Scope.applicationOverride, "driver", "load_driver_hosts", opts.csv_to_list(args.load_driver_hosts)) if sub_command != "list": # Also needed by mechanic (-> telemetry) - duplicate by module? target_hosts = opts.TargetHosts(args.target_hosts) cfg.add(config.Scope.applicationOverride, "client", "hosts", target_hosts) client_options = opts.ClientOptions(args.client_options, target_hosts=target_hosts) cfg.add(config.Scope.applicationOverride, "client", "options", client_options) if "timeout" not in client_options.default: console.info( "You did not provide an explicit timeout in the client options. Assuming default of 10 seconds." ) if list(target_hosts.all_hosts) != list( client_options.all_client_options): console.println( "--target-hosts and --client-options must define the same keys for multi cluster setups." ) exit(1) # split by component? if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) logger.info("OS [%s]", str(os.uname())) logger.info("Python [%s]", str(sys.implementation)) logger.info("Rally version [%s]", version.version()) logger.info("Command line arguments: %s", args) # Configure networking net.init() if not args.offline: if not net.has_internet_connection(): console.warn( "No Internet connection detected. Automatic download of track data sets etc. is disabled.", logger=logger) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True) else: logger.info("Detected a working Internet connection.") success = dispatch_sub_command(cfg, sub_command) end = time.time() if success: console.println("") console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-") else: console.println("") console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(64)
def main(): start = time.time() # Early init of console output so we start to show everything consistently. console.init(quiet=False) pre_configure_logging() args = parse_args() console.init(quiet=args.quiet) console.println(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) # Add global meta info derived by rally itself cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "rally.root", rally_root_path()) cfg.add(config.Scope.application, "system", "rally.cwd", os.getcwd()) cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root()) # Add command line config cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision) cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging) cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "benchmarks", "track", args.track) cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge) cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car) cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health", args.cluster_health) cfg.add(config.Scope.applicationOverride, "benchmarks", "laps", args.laps) cfg.add(config.Scope.applicationOverride, "benchmarks", "test.mode", args.test_mode) cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", convert_hosts(csv_to_list(args.target_hosts))) cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options))) cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format) cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file) if args.override_src_dir is not None: cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir) if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender) configure_logging(cfg) logger.info("Rally version [%s]" % version()) logger.info("Command line arguments: %s" % args) # Configure networking net.init() if not args.offline: if not net.has_internet_connection(): console.warn("No Internet connection detected. Automatic download of track data sets etc. is disabled.", logger=logger) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True) else: logger.info("Detected a working Internet connection.") # Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine # noinspection PyBroadException try: process.kill_running_rally_instances() except BaseException: logger.exception("Could not terminate potentially running Rally instances correctly. Attempting to go on anyway.") try: actors = bootstrap_actor_system(cfg) except RuntimeError as e: logger.exception("Could not bootstrap actor system.") if str(e) == "Unable to determine valid external socket address.": console.warn("Could not determine a socket address. Are you running without any network?", logger=logger) actors = bootstrap_actor_system(cfg, system_base="multiprocQueueBase") else: raise success = False try: success = dispatch_sub_command(cfg, sub_command) finally: shutdown_complete = False times_interrupted = 0 while not shutdown_complete and times_interrupted < 2: try: logger.info("Attempting to shutdown internal actor system.") actors.shutdown() shutdown_complete = True logger.info("Shutdown completed.") except KeyboardInterrupt: times_interrupted += 1 logger.warn("User interrupted shutdown of internal actor system.") console.info("Please wait a moment for Rally's internal components to shutdown.") if not shutdown_complete and times_interrupted > 0: logger.warn("Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted) console.println("") console.warn("Terminating now at the risk of leaving child processes behind.") console.println("") console.warn("The next race may fail due to an unclean shutdown.") console.println("") console.println(SKULL) console.println("") end = time.time() if success: console.println("") console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-") else: console.println("") console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(64)
def migrate(config_file, current_version, target_version, out=print, i=input): prompter = Prompter(i=i, o=out, assume_defaults=False) logger.info("Upgrading configuration from version [%s] to [%s]." % (current_version, target_version)) # Something is really fishy. We don't want to downgrade the configuration. if current_version >= target_version: raise ConfigError( "The existing config file is available in a later version already. Expected version <= [%s] but found [%s]" % (target_version, current_version)) # but first a backup... config_file.backup() config = config_file.load(interpolation=None) if current_version == 0 and target_version > current_version: logger.info("Migrating config from version [0] to [1]") current_version = 1 config["meta"] = {} config["meta"]["config.version"] = str(current_version) # in version 1 we changed some directories from being absolute to being relative config["system"]["log.root.dir"] = "logs" config["provisioning"]["local.install.dir"] = "install" config["reporting"]["report.base.dir"] = "reports" if current_version == 1 and target_version > current_version: logger.info("Migrating config from version [1] to [2]") current_version = 2 config["meta"]["config.version"] = str(current_version) # no need to ask the user now if we are about to upgrade to version 4 config["reporting"]["datastore.type"] = "in-memory" config["reporting"]["datastore.host"] = "" config["reporting"]["datastore.port"] = "" config["reporting"]["datastore.secure"] = "" config["reporting"]["datastore.user"] = "" config["reporting"]["datastore.password"] = "" config["system"]["env.name"] = "local" if current_version == 2 and target_version > current_version: logger.info("Migrating config from version [2] to [3]") current_version = 3 config["meta"]["config.version"] = str(current_version) # Remove obsolete settings config["reporting"].pop("report.base.dir") config["reporting"].pop("output.html.report.filename") if current_version == 3 and target_version > current_version: root_dir = config["system"]["root.dir"] out(""" ***************************************************************************************** You have an old configuration of Rally. Rally has now a much simpler setup routine which will autodetect lots of settings for you and it also does not require you to setup a metrics store anymore. Rally will now migrate your configuration but if you don't need advanced features like a metrics store, then you should delete the configuration directory: rm -rf {0} and then rerun Rally's configuration routine: {1} configure Please also note you have {2:.1f} GB of data in your current benchmark directory at {3} You might want to clean up this directory also. For more details please see {4} ***************************************************************************************** Pausing for 10 seconds to let you consider this message. """.format( config_file.config_dir, PROGRAM_NAME, convert.bytes_to_gb(io.get_size(root_dir)), root_dir, console.format.link( "https://github.com/elastic/rally/blob/master/CHANGELOG.md#030" ))) time.sleep(10) logger.info("Migrating config from version [3] to [4]") current_version = 4 config["meta"]["config.version"] = str(current_version) if len(config["reporting"]["datastore.host"]) > 0: config["reporting"]["datastore.type"] = "elasticsearch" else: config["reporting"]["datastore.type"] = "in-memory" # Remove obsolete settings config["build"].pop("maven.bin") config["benchmarks"].pop("metrics.stats.disk.device") if current_version == 4 and target_version > current_version: config["tracks"] = {} config["tracks"][ "default.url"] = "https://github.com/elastic/rally-tracks" current_version = 5 config["meta"]["config.version"] = str(current_version) if current_version == 5 and target_version > current_version: config["defaults"] = {} config["defaults"]["preserve_benchmark_candidate"] = str(False) current_version = 6 config["meta"]["config.version"] = str(current_version) if current_version == 6 and target_version > current_version: # Remove obsolete settings config.pop("provisioning") config["system"].pop("log.root.dir") current_version = 7 config["meta"]["config.version"] = str(current_version) if current_version == 7 and target_version > current_version: # move [system][root.dir] to [node][root.dir] if "node" not in config: config["node"] = {} config["node"]["root.dir"] = config["system"].pop("root.dir") # also move all references! for section in config: for k, v in config[section].items(): config[section][k] = v.replace("${system:root.dir}", "${node:root.dir}") current_version = 8 config["meta"]["config.version"] = str(current_version) if current_version == 8 and target_version > current_version: config["teams"] = {} config["teams"][ "default.url"] = "https://github.com/elastic/rally-teams" current_version = 9 config["meta"]["config.version"] = str(current_version) if current_version == 9 and target_version > current_version: config["distributions"] = {} config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \ "{{VERSION}}.tar.gz" config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \ "distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz" config["distributions"][ "release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" config["distributions"]["release.cache"] = "true" current_version = 10 config["meta"]["config.version"] = str(current_version) if current_version == 10 and target_version > current_version: config["runtime"]["java.home"] = config["runtime"].pop("java8.home") current_version = 11 config["meta"]["config.version"] = str(current_version) if current_version == 11 and target_version > current_version: # As this is a rather complex migration, we log more than usual to understand potential migration problems better. if "source" in config: if "local.src.dir" in config["source"]: previous_root = config["source"].pop("local.src.dir") logger.info("Set [source][local.src.dir] to [%s]." % previous_root) # if this directory was Rally's default location, then move it on the file system because to allow for checkouts of plugins # in the sibling directory. if previous_root == os.path.join(config["node"]["root.dir"], "src"): new_root_dir_all_sources = previous_root new_es_sub_dir = "elasticsearch" new_root = os.path.join(new_root_dir_all_sources, new_es_sub_dir) # only attempt to move if the directory exists. It may be possible that users never ran a source benchmark although they # have configured it. In that case the source directory will not yet exist. if io.exists(previous_root): logger.info( "Previous source directory was at Rally's default location [%s]. Moving to [%s]." % (previous_root, new_root)) try: # we need to do this in two steps as we need to move the sources to a subdirectory tmp_path = io.normalize_path( os.path.join(new_root_dir_all_sources, os.pardir, "tmp_src_mig")) os.rename(previous_root, tmp_path) io.ensure_dir(new_root) os.rename(tmp_path, new_root) except OSError: logger.exception( "Could not move source directory from [%s] to [%s]." % (previous_root, new_root)) # A warning is sufficient as Rally should just do a fresh checkout if moving did not work. console.warn( "Elasticsearch source directory could not be moved from [%s] to [%s]. Please check the logs." % (previous_root, new_root)) else: logger.info( "Source directory is configured at Rally's default location [%s] but does not exist yet." % previous_root) else: logger.info( "Previous source directory was the custom directory [%s]." % previous_root) new_root_dir_all_sources = io.normalize_path( os.path.join(previous_root, os.path.pardir)) # name of the elasticsearch project directory. new_es_sub_dir = io.basename(previous_root) logger.info("Setting [node][src.root.dir] to [%s]." % new_root_dir_all_sources) config["node"]["src.root.dir"] = new_root_dir_all_sources logger.info( "Setting [source][elasticsearch.src.subdir] to [%s]" % new_es_sub_dir) config["source"]["elasticsearch.src.subdir"] = new_es_sub_dir else: logger.info( "Key [local.src.dir] not found. Advancing without changes." ) else: logger.info( "No section named [source] found in config. Advancing without changes." ) current_version = 12 config["meta"]["config.version"] = str(current_version) if current_version == 12 and target_version > current_version: # the current configuration allows to benchmark from sources if "build" in config and "gradle.bin" in config["build"]: java_9_home = io.guess_java_home(major_version=9) from esrally.utils import jvm if java_9_home and not jvm.is_early_access_release(java_9_home): logger.debug("Autodetected a JDK 9 installation at [%s]" % java_9_home) if "runtime" not in config: config["runtime"] = {} config["runtime"]["java9.home"] = java_9_home else: logger.debug( "Could not autodetect a JDK 9 installation. Checking [java.home] already points to a JDK 9." ) detected = False if "runtime" in config: java_home = config["runtime"]["java.home"] if jvm.major_version( java_home ) == 9 and not jvm.is_early_access_release(java_home): config["runtime"]["java9.home"] = java_home detected = True if not detected: logger.debug( "Could not autodetect a JDK 9 installation. Asking user." ) raw_java_9_home = prompter.ask_property( "Enter the JDK 9 root directory", check_path_exists=True, mandatory=False) if raw_java_9_home and jvm.major_version( raw_java_9_home ) == 9 and not jvm.is_early_access_release( raw_java_9_home): java_9_home = io.normalize_path( raw_java_9_home) if raw_java_9_home else None config["runtime"]["java9.home"] = java_9_home else: out("********************************************************************************" ) out("You don't have a valid JDK 9 installation and cannot benchmark source builds." ) out("") out("You can still benchmark binary distributions with e.g.:" ) out("") out(" %s --distribution-version=6.0.0" % PROGRAM_NAME) out("********************************************************************************" ) out("") current_version = 13 config["meta"]["config.version"] = str(current_version) # all migrations done config_file.store(config) logger.info("Successfully self-upgraded configuration to version [%s]" % target_version)
def main(): check_python_version() start = time.time() # Early init of console output so we start to show everything consistently. console.init(quiet=False) # allow to see a thread-dump on SIGQUIT faulthandler.register(signal.SIGQUIT, file=sys.stderr) pre_configure_logging() args = parse_args() console.init(quiet=args.quiet) console.println(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) if args.effective_start_date: cfg.add(config.Scope.application, "system", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "time.start.user_provided", True) else: cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.utcnow()) cfg.add(config.Scope.application, "system", "time.start.user_provided", False) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) # per node? cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging) # Local config per node cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root()) cfg.add(config.Scope.application, "node", "rally.cwd", os.getcwd()) cfg.add(config.Scope.applicationOverride, "mechanic", "source.revision", args.revision) #TODO dm: Consider renaming this one. It's used by different modules if args.distribution_version: cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name", args.team_repository) cfg.add(config.Scope.applicationOverride, "mechanic", "car.names", csv_to_list(args.car)) cfg.add(config.Scope.applicationOverride, "mechanic", "car.plugins", csv_to_list(args.elasticsearch_plugins)) cfg.add(config.Scope.applicationOverride, "mechanic", "node.datapaths", csv_to_list(args.data_paths)) if args.keep_cluster_running: cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running", True) # force-preserve the cluster nodes. cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install", True) else: cfg.add(config.Scope.applicationOverride, "mechanic", "keep.running", False) cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "race", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "race", "laps", args.laps) cfg.add(config.Scope.applicationOverride, "race", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "track", "repository.name", args.track_repository) cfg.add(config.Scope.applicationOverride, "track", "track.name", args.track) cfg.add(config.Scope.applicationOverride, "track", "challenge.name", args.challenge) cfg.add(config.Scope.applicationOverride, "track", "include.tasks", csv_to_list(args.include_tasks)) cfg.add(config.Scope.applicationOverride, "track", "test.mode.enabled", args.test_mode) cfg.add(config.Scope.applicationOverride, "track", "auto_manage_indices", to_bool(args.auto_manage_indices)) cfg.add(config.Scope.applicationOverride, "reporting", "format", args.report_format) cfg.add(config.Scope.applicationOverride, "reporting", "output.path", args.report_file) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "reporting", "baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "reporting", "contender.timestamp", args.contender) ################################ # new section name: driver ################################ cfg.add(config.Scope.applicationOverride, "driver", "cluster.health", args.cluster_health) cfg.add(config.Scope.applicationOverride, "driver", "profiling", args.enable_driver_profiling) cfg.add(config.Scope.applicationOverride, "driver", "load_driver_hosts", csv_to_list(args.load_driver_hosts)) if sub_command != "list": # Also needed by mechanic (-> telemetry) - duplicate by module? cfg.add(config.Scope.applicationOverride, "client", "hosts", _normalize_hosts(csv_to_list(args.target_hosts))) client_options = kv_to_map(csv_to_list(args.client_options)) cfg.add(config.Scope.applicationOverride, "client", "options", client_options) if "timeout" not in client_options: console.info("You did not provide an explicit timeout in the client options. Assuming default of 10 seconds.") # split by component? if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) configure_logging(cfg) logger.info("OS [%s]" % str(os.uname())) logger.info("Python [%s]" % str(sys.implementation)) logger.info("Rally version [%s]" % version.version()) logger.info("Command line arguments: %s" % args) # Configure networking net.init() if not args.offline: if not net.has_internet_connection(): console.warn("No Internet connection detected. Automatic download of track data sets etc. is disabled.", logger=logger) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True) else: logger.info("Detected a working Internet connection.") # Kill any lingering Rally processes before attempting to continue - the actor system needs to be a singleton on this machine # noinspection PyBroadException try: process.kill_running_rally_instances() except BaseException: logger.exception("Could not terminate potentially running Rally instances correctly. Attempting to go on anyway.") success = dispatch_sub_command(cfg, sub_command) end = time.time() if success: console.println("") console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-") else: console.println("") console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(64)
def main(): start = time.time() # Early init of console output so we start to show everything consistently. console.init(quiet=False) pre_configure_logging() args = parse_args() console.init(quiet=args.quiet) console.println(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) # Add global meta info derived by rally itself cfg.add(config.Scope.application, "meta", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "rally.root", rally_root_path()) cfg.add(config.Scope.application, "system", "invocation.root.dir", paths.Paths(cfg).invocation_root()) # Add command line config cfg.add(config.Scope.applicationOverride, "source", "revision", args.revision) cfg.add(config.Scope.applicationOverride, "source", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "source", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "system", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "system", "track.repository", args.track_repository) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging) cfg.add(config.Scope.applicationOverride, "telemetry", "devices", csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "benchmarks", "track", args.track) cfg.add(config.Scope.applicationOverride, "benchmarks", "challenge", args.challenge) cfg.add(config.Scope.applicationOverride, "benchmarks", "car", args.car) cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health", args.cluster_health) cfg.add(config.Scope.applicationOverride, "benchmarks", "laps", args.laps) cfg.add(config.Scope.applicationOverride, "provisioning", "datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "provisioning", "install.preserve", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "launcher", "external.target.hosts", convert_hosts(csv_to_list(args.target_hosts))) cfg.add(config.Scope.applicationOverride, "launcher", "client.options", kv_to_map(csv_to_list(args.client_options))) cfg.add(config.Scope.applicationOverride, "report", "reportformat", args.report_format) cfg.add(config.Scope.applicationOverride, "report", "reportfile", args.report_file) if args.override_src_dir is not None: cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir) if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "report", "comparison.baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "report", "comparison.contender.timestamp", args.contender) configure_logging(cfg) logger.info("Rally version [%s]" % version()) logger.info("Command line arguments: %s" % args) # Configure networking net.init() if not args.offline: if not net.has_internet_connection(): console.warn( "No Internet connection detected. Automatic download of track data sets etc. is disabled.", logger=logger) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True) else: logger.info("Detected a working Internet connection.") # Kill any lingering Rally processes before attempting to continue - the actor system needs to a singleton on this machine # noinspection PyBroadException try: process.kill_running_rally_instances() except BaseException: logger.exception( "Could not terminate potentially running Rally instances correctly. Attempting to go on anyway." ) try: actors = bootstrap_actor_system(cfg) except RuntimeError as e: logger.exception("Could not bootstrap actor system.") if str(e) == "Unable to determine valid external socket address.": console.warn( "Could not determine a socket address. Are you running without any network?", logger=logger) actors = bootstrap_actor_system(cfg, system_base="multiprocQueueBase") else: raise success = False try: success = dispatch_sub_command(cfg, sub_command) finally: shutdown_complete = False times_interrupted = 0 while not shutdown_complete and times_interrupted < 2: try: logger.info("Attempting to shutdown internal actor system.") actors.shutdown() shutdown_complete = True logger.info("Shutdown completed.") except KeyboardInterrupt: times_interrupted += 1 logger.warn( "User interrupted shutdown of internal actor system.") console.info( "Please wait a moment for Rally's internal components to shutdown." ) if not shutdown_complete and times_interrupted > 0: logger.warn( "Terminating after user has interrupted actor system shutdown explicitly for [%d] times." % times_interrupted) console.println("") console.warn( "Terminating now at the risk of leaving child processes behind." ) console.println("") console.warn("The next race may fail due to an unclean shutdown.") console.println("") console.println(SKULL) console.println("") end = time.time() if success: console.println("") console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-") else: console.println("") console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(64)
def main(): check_python_version() start = time.time() # Early init of console output so we start to show everything consistently. console.init(quiet=False) # allow to see a thread-dump on SIGQUIT faulthandler.register(signal.SIGQUIT, file=sys.stderr) pre_configure_logging() args = parse_args() console.init(quiet=args.quiet) console.println(BANNER) cfg = config.Config(config_name=args.configuration_name) sub_command = derive_sub_command(args, cfg) ensure_configuration_present(cfg, args, sub_command) if args.effective_start_date: cfg.add(config.Scope.application, "system", "time.start", args.effective_start_date) cfg.add(config.Scope.application, "system", "time.start.user_provided", True) else: cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.utcnow()) cfg.add(config.Scope.application, "system", "time.start.user_provided", False) cfg.add(config.Scope.applicationOverride, "system", "quiet.mode", args.quiet) # per node? cfg.add(config.Scope.applicationOverride, "system", "offline.mode", args.offline) cfg.add(config.Scope.applicationOverride, "system", "logging.output", args.logging) # only temporary to ignore unknown actor messages cfg.add(config.Scope.applicationOverride, "system", "ignore.unknown.return", args.ignore_unknown_return_values) # Local config per node cfg.add(config.Scope.application, "node", "rally.root", paths.rally_root()) cfg.add(config.Scope.application, "node", "rally.cwd", os.getcwd()) cfg.add(config.Scope.applicationOverride, "mechanic", "source.revision", args.revision) #TODO dm: Consider renaming this one. It's used by different modules if args.distribution_version: cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "mechanic", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "mechanic", "repository.name", args.team_repository) cfg.add(config.Scope.applicationOverride, "mechanic", "car.name", args.car) cfg.add(config.Scope.applicationOverride, "mechanic", "car.plugins", csv_to_list(args.elasticsearch_plugins)) cfg.add(config.Scope.applicationOverride, "mechanic", "node.datapaths", csv_to_list(args.data_paths)) cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install", convert.to_bool(args.preserve_install)) cfg.add(config.Scope.applicationOverride, "mechanic", "telemetry.devices", csv_to_list(args.telemetry)) if args.override_src_dir is not None: cfg.add(config.Scope.applicationOverride, "source", "local.src.dir", args.override_src_dir) cfg.add(config.Scope.applicationOverride, "race", "pipeline", args.pipeline) cfg.add(config.Scope.applicationOverride, "race", "laps", args.laps) cfg.add(config.Scope.applicationOverride, "race", "user.tag", args.user_tag) cfg.add(config.Scope.applicationOverride, "track", "repository.name", args.track_repository) cfg.add(config.Scope.applicationOverride, "track", "track.name", args.track) cfg.add(config.Scope.applicationOverride, "track", "challenge.name", args.challenge) cfg.add(config.Scope.applicationOverride, "track", "test.mode.enabled", args.test_mode) cfg.add(config.Scope.applicationOverride, "track", "auto_manage_indices", to_bool(args.auto_manage_indices)) cfg.add(config.Scope.applicationOverride, "reporting", "format", args.report_format) cfg.add(config.Scope.applicationOverride, "reporting", "output.path", args.report_file) if sub_command == "compare": cfg.add(config.Scope.applicationOverride, "reporting", "baseline.timestamp", args.baseline) cfg.add(config.Scope.applicationOverride, "reporting", "contender.timestamp", args.contender) ################################ # new section name: driver ################################ cfg.add(config.Scope.applicationOverride, "benchmarks", "cluster.health", args.cluster_health) cfg.add(config.Scope.applicationOverride, "driver", "profiling", args.enable_driver_profiling) if sub_command != "list": # Also needed by mechanic (-> telemetry) - duplicate by module? cfg.add(config.Scope.applicationOverride, "client", "hosts", _normalize_hosts(csv_to_list(args.target_hosts))) client_options = kv_to_map(csv_to_list(args.client_options)) cfg.add(config.Scope.applicationOverride, "client", "options", client_options) if "timeout" not in client_options: console.info( "You did not provide an explicit timeout in the client options. Assuming default of 10 seconds." ) # split by component? if sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit) configure_logging(cfg) logger.info("OS [%s]" % str(os.uname())) logger.info("Python [%s]" % str(sys.implementation)) logger.info("Rally version [%s]" % version.version()) logger.info("Command line arguments: %s" % args) # Configure networking net.init() if not args.offline: if not net.has_internet_connection(): console.warn( "No Internet connection detected. Automatic download of track data sets etc. is disabled.", logger=logger) cfg.add(config.Scope.applicationOverride, "system", "offline.mode", True) else: logger.info("Detected a working Internet connection.") # Kill any lingering Rally processes before attempting to continue - the actor system needs to be a singleton on this machine # noinspection PyBroadException try: process.kill_running_rally_instances() except BaseException: logger.exception( "Could not terminate potentially running Rally instances correctly. Attempting to go on anyway." ) success = dispatch_sub_command(cfg, sub_command) end = time.time() if success: console.println("") console.info("SUCCESS (took %d seconds)" % (end - start), overline="-", underline="-") else: console.println("") console.info("FAILURE (took %d seconds)" % (end - start), overline="-", underline="-") sys.exit(64)