def load_car(self, name, car_params=None): car_config_file = self._car_file(name) if not io.exists(car_config_file): raise exceptions.SystemSetupError("Unknown car [{}]. List the available cars with {} list cars.".format(name, PROGRAM_NAME)) config = self._config_loader(car_config_file) root_paths = [] config_paths = [] config_base_vars = {} description = self._value(config, ["meta", "description"], default="") car_type = self._value(config, ["meta", "type"], default="car") config_bases = self._value(config, ["config", "base"], default="").split(",") for base in config_bases: if base: root_path = os.path.join(self.cars_dir, base) root_paths.append(root_path) config_paths.append(os.path.join(root_path, "templates")) config_file = os.path.join(root_path, "config.ini") if io.exists(config_file): base_config = self._config_loader(config_file) self._copy_section(base_config, "variables", config_base_vars) # it's possible that some cars don't have a config base, e.g. mixins which only override variables if len(config_paths) == 0: self.logger.info("Car [%s] does not define any config paths. Assuming that it is used as a mixin.", name) variables = self._copy_section(config, "variables", {}) # add all car params here to override any defaults if car_params: variables.update(car_params) env = self._copy_section(config, "env", {}) return CarDescriptor(name, description, car_type, root_paths, config_paths, config_base_vars, variables, env)
def load_plugin(self, name, config_names, plugin_params={}): root_path = self._plugin_root_path(name) if not config_names: # maybe we only have a config folder but nothing else (e.g. if there is only an install hook) if io.exists(root_path): return PluginDescriptor(name=name, config=config_names, root_path=root_path) else: core_plugin = self._core_plugin(name) if core_plugin: return core_plugin # If we just have a plugin name then we assume that this is a community plugin and the user has specified a download URL else: logger.info("The plugin [%s] is neither a configured nor an official plugin. Assuming that this is a community " "plugin not requiring any configuration and you have set a proper download URL." % name) return PluginDescriptor(name) else: variables = {} config_paths = [] # used for deduplication known_config_bases = set() # used to determine whether this is a core plugin core_plugin = self._core_plugin(name) for config_name in config_names: config_file = self._plugin_file(name, config_name) # Do we have an explicit configuration for this plugin? if not io.exists(config_file): if core_plugin: raise exceptions.SystemSetupError("Plugin [%s] does not provide configuration [%s]. List the available plugins " "and configurations with %s list elasticsearch-plugins " "--distribution-version=VERSION." % (name, config_name, PROGRAM_NAME)) else: raise exceptions.SystemSetupError("Unknown plugin [%s]. List the available plugins with %s list " "elasticsearch-plugins --distribution-version=VERSION." % (name, PROGRAM_NAME)) config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation()) # Do not modify the case of option keys but read them as is config.optionxform = lambda option: option config.read(config_file) if "config" in config and "base" in config["config"]: config_bases = config["config"]["base"].split(",") for base in config_bases: if base and base not in known_config_bases: config_paths.append(os.path.join(root_path, base)) known_config_bases.add(base) if "variables" in config.sections(): for k, v in config["variables"].items(): variables[k] = v # add all plugin params here to override any defaults variables.update(plugin_params) # maybe one of the configs is really just for providing variables. However, we still require one config base overall. if len(config_paths) == 0: raise exceptions.SystemSetupError("At least one config base is required for plugin [%s]" % name) return PluginDescriptor(name=name, core_plugin=core_plugin is not None, config=config_names, root_path=root_path, config_paths=config_paths, variables=variables)
def load_plugin(self, name, config_names): root_path = self._plugin_root_path(name) if not config_names: # maybe we only have a config folder but nothing else (e.g. if there is only an install hook) if io.exists(root_path): return PluginDescriptor(name=name, config=config_names, root_path=root_path) else: core_plugin = self._core_plugin(name) if core_plugin: return core_plugin # If we just have a plugin name then we assume that this is a community plugin and the user has specified a download URL else: logger.info("The plugin [%s] is neither a configured nor an official plugin. Assuming that this is a community " "plugin not requiring any configuration and you have set a proper download URL." % name) return PluginDescriptor(name) else: variables = {} config_paths = [] # used for deduplication known_config_bases = set() # used to determine whether this is a core plugin core_plugin = self._core_plugin(name) for config_name in config_names: config_file = self._plugin_file(name, config_name) # Do we have an explicit configuration for this plugin? if not io.exists(config_file): if core_plugin: raise exceptions.SystemSetupError("Plugin [%s] does not provide configuration [%s]. List the available plugins " "and configurations with %s list elasticsearch-plugins " "--distribution-version=VERSION." % (name, config_name, PROGRAM_NAME)) else: raise exceptions.SystemSetupError("Unknown plugin [%s]. List the available plugins with %s list " "elasticsearch-plugins --distribution-version=VERSION." % (name, PROGRAM_NAME)) config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation()) # Do not modify the case of option keys but read them as is config.optionxform = lambda option: option config.read(config_file) if "config" in config and "base" in config["config"]: config_bases = config["config"]["base"].split(",") for base in config_bases: if base and base not in known_config_bases: config_paths.append(os.path.join(root_path, base)) known_config_bases.add(base) if "variables" in config.sections(): for k, v in config["variables"].items(): variables[k] = v # maybe one of the configs is really just for providing variables. However, we still require one config base overall. if len(config_paths) == 0: raise exceptions.SystemSetupError("At least one config base is required for plugin [%s]" % name) return PluginDescriptor(name=name, core_plugin=core_plugin is not None, config=config_names, root_path=root_path, config_paths=config_paths, variables=variables)
def load_car(self, name): car_config_file = self._car_file(name) if not io.exists(car_config_file): raise exceptions.SystemSetupError("Unknown car [%s]. List the available cars with %s list cars." % (name, PROGRAM_NAME)) config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation()) # Do not modify the case of option keys but read them as is config.optionxform = lambda option: option config.read(car_config_file) config_paths = [] description = "" car_type = "car" if "meta" in config: description = config["meta"].get("description", description) car_type = config["meta"].get("type", car_type) if "config" in config and "base" in config["config"]: config_bases = config["config"]["base"].split(",") for base in config_bases: if base: config_paths.append(os.path.join(self.cars_dir, base)) # it's possible that some cars don't have a config base, e.g. mixins which only override variables if len(config_paths) == 0: logger.info("Car [%s] does not define any config paths. Assuming that it is used as a mixin." % name) variables = {} if "variables" in config.sections(): for k, v in config["variables"].items(): variables[k] = v env = {} if "env" in config.sections(): for k, v in config["env"].items(): env[k] = v return CarDescriptor(name, description, car_type, config_paths, variables, env)
def __init__(self, remote_url, root_dir, repo_name, resource_name, offline, fetch=True): # If no URL is found, we consider this a local only repo (but still require that it is a git repo) self.url = remote_url self.remote = self.url is not None and self.url.strip() != "" self.repo_dir = os.path.join(root_dir, repo_name) self.resource_name = resource_name self.offline = offline self.logger = logging.getLogger(__name__) if self.remote and not self.offline and fetch: # a normal git repo with a remote if not git.is_working_copy(self.repo_dir): git.clone(src=self.repo_dir, remote=self.url) else: try: git.fetch(src=self.repo_dir) except exceptions.SupplyError: console.warn( "Could not update %s. Continuing with your locally available state." % self.resource_name) else: if not git.is_working_copy(self.repo_dir): if io.exists(self.repo_dir): raise exceptions.SystemSetupError( "[{src}] must be a git repository.\n\nPlease run:\ngit -C {src} init" .format(src=self.repo_dir)) else: raise exceptions.SystemSetupError( "Expected a git repository at [{src}] but the directory does not exist." .format(src=self.repo_dir))
def load_car(self, name): car_config_file = self._car_file(name) if not io.exists(car_config_file): raise exceptions.SystemSetupError("Unknown car [%s]. List the available cars with %s list cars." % (name, PROGRAM_NAME)) config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation()) # Do not modify the case of option keys but read them as is config.optionxform = lambda option: option config.read(car_config_file) config_paths = [] description = "" car_type = "car" if "meta" in config: description = config["meta"].get("description", description) car_type = config["meta"].get("type", car_type) if "config" in config and "base" in config["config"]: config_bases = config["config"]["base"].split(",") for base in config_bases: if base: config_paths.append(os.path.join(self.cars_dir, base)) # it's possible that some cars don't have a config base, e.g. mixins which only override variables if len(config_paths) == 0: logger.info("Car [%s] does not define any config paths. Assuming that it is used as a mixin." % name) variables = {} if "variables" in config.sections(): for k, v in config["variables"].items(): variables[k] = v env = {} if "env" in config.sections(): for k, v in config["env"].items(): env[k] = v return CarDescriptor(name, description, car_type, config_paths, variables, env)
def load_car(self, name): car_config_file = self._car_file(name) if not io.exists(car_config_file): raise exceptions.SystemSetupError( "Unknown car [%s]. List the available cars with %s list cars." % (name, PROGRAM_NAME)) config = configparser.ConfigParser( interpolation=configparser.ExtendedInterpolation()) # Do not modify the case of option keys but read them as is config.optionxform = lambda option: option config.read(car_config_file) config_paths = [] if "config" in config and "base" in config["config"]: config_bases = config["config"]["base"].split(",") for base in config_bases: if base: config_paths.append(os.path.join(self.cars_dir, base)) if len(config_paths) == 0: raise exceptions.SystemSetupError( "At least one config base is required for car [%s]" % name) variables = {} if "variables" in config.sections(): for k, v in config["variables"].items(): variables[k] = v env = {} if "env" in config.sections(): for k, v in config["env"].items(): env[k] = v return Car(name, config_paths, variables, env)
def ask_property(self, prompt, mandatory=True, check_path_exists=False, check_pattern=None, choices=None, sensitive=False, default_value=None): if default_value is not None: final_prompt = "%s (default: %s): " % (prompt, default_value) elif not mandatory: final_prompt = "%s (Press Enter to skip): " % prompt else: final_prompt = "%s: " % prompt while True: if self.assume_defaults and (default_value is not None or not mandatory): self.o(final_prompt) value = None elif sensitive: value = self.sec_i(final_prompt) else: value = self.i(final_prompt) if not value or value.strip() == "": if mandatory and default_value is None: self.o(" Value is required. Please retry.") continue else: # suppress output when the default is empty if default_value: self.o(" Using default value '%s'" % default_value) # this way, we can still check the path... value = default_value if mandatory or value is not None: if check_path_exists and not io.exists(value): self.o("'%s' does not exist. Please check and retry." % value) continue if check_pattern is not None and not check_pattern.match( str(value)): self.o( "Input does not match pattern [%s]. Please check and retry." % check_pattern.pattern) continue if choices is not None and str(value) not in choices: self.o( "Input is not one of the valid choices %s. Please check and retry." % choices) continue self.o("") # user entered a valid value return value
def install_default_log_config(): """ Ensures a log configuration file is present on this machine. The default log configuration is based on the template in resources/logging.json. It also ensures that the default log path has been created so log files can be successfully opened in that directory. """ log_config = log_config_path() if not io.exists(log_config): io.ensure_dir(io.dirname(log_config)) source_path = io.normalize_path(os.path.join(os.path.dirname(__file__), "resources", "logging.json")) with open(log_config, "w", encoding="UTF-8") as target: with open(source_path, "r", encoding="UTF-8") as src: contents = src.read().replace("${LOG_PATH}", default_log_path()) target.write(contents) io.ensure_dir(default_log_path())
def remove_obsolete_default_log_config(): """ Log rotation is problematic because Rally uses multiple processes and there is a lurking race condition when rolling log files. Hence, we do not rotate logs from within Rally and leverage established tools like logrotate for that. Checks whether the user has a problematic out-of-the-box logging configuration delivered with Rally 1.0.0 which used log rotation and removes it so it can be replaced by a new one in a later step. """ log_config = log_config_path() if io.exists(log_config): source_path = io.normalize_path(os.path.join(os.path.dirname(__file__), "resources", "logging_1_0_0.json")) with open(source_path, "r", encoding="UTF-8") as src: contents = src.read().replace("${LOG_PATH}", default_log_path()) source_hash = hashlib.sha512(contents.encode()).hexdigest() with open(log_config, "r", encoding="UTF-8") as target: target_hash = hashlib.sha512(target.read().encode()).hexdigest() if source_hash == target_hash: os.rename(log_config, "{}.bak".format(log_config))
def install_default_log_config(): """ Ensures a log configuration file is present on this machine. The default log configuration is based on the template in resources/logging.json. It also ensures that the default log path has been created so log files can be successfully opened in that directory. """ log_config = log_config_path() if not io.exists(log_config): io.ensure_dir(io.dirname(log_config)) source_path = io.normalize_path( os.path.join(os.path.dirname(__file__), "resources", "logging.json")) with open(log_config, "w", encoding="UTF-8") as target: with open(source_path, "r", encoding="UTF-8") as src: # Ensure we have a trailing path separator as after LOG_PATH there will only be the file name log_path = os.path.join(paths.logs(), "") # the logging path might contain backslashes that we need to escape log_path = io.escape_path(log_path) contents = src.read().replace("${LOG_PATH}", log_path) target.write(contents) io.ensure_dir(paths.logs())
def migrate(config_file, current_version, target_version, out=print, i=input): prompter = Prompter(i=i, o=out, assume_defaults=False) logger.info("Upgrading configuration from version [%s] to [%s]." % (current_version, target_version)) # Something is really fishy. We don't want to downgrade the configuration. if current_version >= target_version: raise ConfigError( "The existing config file is available in a later version already. Expected version <= [%s] but found [%s]" % (target_version, current_version)) # but first a backup... config_file.backup() config = config_file.load(interpolation=None) if current_version == 0 and target_version > current_version: logger.info("Migrating config from version [0] to [1]") current_version = 1 config["meta"] = {} config["meta"]["config.version"] = str(current_version) # in version 1 we changed some directories from being absolute to being relative config["system"]["log.root.dir"] = "logs" config["provisioning"]["local.install.dir"] = "install" config["reporting"]["report.base.dir"] = "reports" if current_version == 1 and target_version > current_version: logger.info("Migrating config from version [1] to [2]") current_version = 2 config["meta"]["config.version"] = str(current_version) # no need to ask the user now if we are about to upgrade to version 4 config["reporting"]["datastore.type"] = "in-memory" config["reporting"]["datastore.host"] = "" config["reporting"]["datastore.port"] = "" config["reporting"]["datastore.secure"] = "" config["reporting"]["datastore.user"] = "" config["reporting"]["datastore.password"] = "" config["system"]["env.name"] = "local" if current_version == 2 and target_version > current_version: logger.info("Migrating config from version [2] to [3]") current_version = 3 config["meta"]["config.version"] = str(current_version) # Remove obsolete settings config["reporting"].pop("report.base.dir") config["reporting"].pop("output.html.report.filename") if current_version == 3 and target_version > current_version: root_dir = config["system"]["root.dir"] out(""" ***************************************************************************************** You have an old configuration of Rally. Rally has now a much simpler setup routine which will autodetect lots of settings for you and it also does not require you to setup a metrics store anymore. Rally will now migrate your configuration but if you don't need advanced features like a metrics store, then you should delete the configuration directory: rm -rf {0} and then rerun Rally's configuration routine: {1} configure Please also note you have {2:.1f} GB of data in your current benchmark directory at {3} You might want to clean up this directory also. For more details please see {4} ***************************************************************************************** Pausing for 10 seconds to let you consider this message. """.format( config_file.config_dir, PROGRAM_NAME, convert.bytes_to_gb(io.get_size(root_dir)), root_dir, console.format.link( "https://github.com/elastic/rally/blob/master/CHANGELOG.md#030" ))) time.sleep(10) logger.info("Migrating config from version [3] to [4]") current_version = 4 config["meta"]["config.version"] = str(current_version) if len(config["reporting"]["datastore.host"]) > 0: config["reporting"]["datastore.type"] = "elasticsearch" else: config["reporting"]["datastore.type"] = "in-memory" # Remove obsolete settings config["build"].pop("maven.bin") config["benchmarks"].pop("metrics.stats.disk.device") if current_version == 4 and target_version > current_version: config["tracks"] = {} config["tracks"][ "default.url"] = "https://github.com/elastic/rally-tracks" current_version = 5 config["meta"]["config.version"] = str(current_version) if current_version == 5 and target_version > current_version: config["defaults"] = {} config["defaults"]["preserve_benchmark_candidate"] = str(False) current_version = 6 config["meta"]["config.version"] = str(current_version) if current_version == 6 and target_version > current_version: # Remove obsolete settings config.pop("provisioning") config["system"].pop("log.root.dir") current_version = 7 config["meta"]["config.version"] = str(current_version) if current_version == 7 and target_version > current_version: # move [system][root.dir] to [node][root.dir] if "node" not in config: config["node"] = {} config["node"]["root.dir"] = config["system"].pop("root.dir") # also move all references! for section in config: for k, v in config[section].items(): config[section][k] = v.replace("${system:root.dir}", "${node:root.dir}") current_version = 8 config["meta"]["config.version"] = str(current_version) if current_version == 8 and target_version > current_version: config["teams"] = {} config["teams"][ "default.url"] = "https://github.com/elastic/rally-teams" current_version = 9 config["meta"]["config.version"] = str(current_version) if current_version == 9 and target_version > current_version: config["distributions"] = {} config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \ "{{VERSION}}.tar.gz" config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \ "distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz" config["distributions"][ "release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" config["distributions"]["release.cache"] = "true" current_version = 10 config["meta"]["config.version"] = str(current_version) if current_version == 10 and target_version > current_version: config["runtime"]["java.home"] = config["runtime"].pop("java8.home") current_version = 11 config["meta"]["config.version"] = str(current_version) if current_version == 11 and target_version > current_version: # As this is a rather complex migration, we log more than usual to understand potential migration problems better. if "source" in config: if "local.src.dir" in config["source"]: previous_root = config["source"].pop("local.src.dir") logger.info("Set [source][local.src.dir] to [%s]." % previous_root) # if this directory was Rally's default location, then move it on the file system because to allow for checkouts of plugins # in the sibling directory. if previous_root == os.path.join(config["node"]["root.dir"], "src"): new_root_dir_all_sources = previous_root new_es_sub_dir = "elasticsearch" new_root = os.path.join(new_root_dir_all_sources, new_es_sub_dir) # only attempt to move if the directory exists. It may be possible that users never ran a source benchmark although they # have configured it. In that case the source directory will not yet exist. if io.exists(previous_root): logger.info( "Previous source directory was at Rally's default location [%s]. Moving to [%s]." % (previous_root, new_root)) try: # we need to do this in two steps as we need to move the sources to a subdirectory tmp_path = io.normalize_path( os.path.join(new_root_dir_all_sources, os.pardir, "tmp_src_mig")) os.rename(previous_root, tmp_path) io.ensure_dir(new_root) os.rename(tmp_path, new_root) except OSError: logger.exception( "Could not move source directory from [%s] to [%s]." % (previous_root, new_root)) # A warning is sufficient as Rally should just do a fresh checkout if moving did not work. console.warn( "Elasticsearch source directory could not be moved from [%s] to [%s]. Please check the logs." % (previous_root, new_root)) else: logger.info( "Source directory is configured at Rally's default location [%s] but does not exist yet." % previous_root) else: logger.info( "Previous source directory was the custom directory [%s]." % previous_root) new_root_dir_all_sources = io.normalize_path( os.path.join(previous_root, os.path.pardir)) # name of the elasticsearch project directory. new_es_sub_dir = io.basename(previous_root) logger.info("Setting [node][src.root.dir] to [%s]." % new_root_dir_all_sources) config["node"]["src.root.dir"] = new_root_dir_all_sources logger.info( "Setting [source][elasticsearch.src.subdir] to [%s]" % new_es_sub_dir) config["source"]["elasticsearch.src.subdir"] = new_es_sub_dir else: logger.info( "Key [local.src.dir] not found. Advancing without changes." ) else: logger.info( "No section named [source] found in config. Advancing without changes." ) current_version = 12 config["meta"]["config.version"] = str(current_version) if current_version == 12 and target_version > current_version: # the current configuration allows to benchmark from sources if "build" in config and "gradle.bin" in config["build"]: java_9_home = io.guess_java_home(major_version=9) from esrally.utils import jvm if java_9_home and not jvm.is_early_access_release(java_9_home): logger.debug("Autodetected a JDK 9 installation at [%s]" % java_9_home) if "runtime" not in config: config["runtime"] = {} config["runtime"]["java9.home"] = java_9_home else: logger.debug( "Could not autodetect a JDK 9 installation. Checking [java.home] already points to a JDK 9." ) detected = False if "runtime" in config: java_home = config["runtime"]["java.home"] if jvm.major_version( java_home ) == 9 and not jvm.is_early_access_release(java_home): config["runtime"]["java9.home"] = java_home detected = True if not detected: logger.debug( "Could not autodetect a JDK 9 installation. Asking user." ) raw_java_9_home = prompter.ask_property( "Enter the JDK 9 root directory", check_path_exists=True, mandatory=False) if raw_java_9_home and jvm.major_version( raw_java_9_home ) == 9 and not jvm.is_early_access_release( raw_java_9_home): java_9_home = io.normalize_path( raw_java_9_home) if raw_java_9_home else None config["runtime"]["java9.home"] = java_9_home else: out("********************************************************************************" ) out("You don't have a valid JDK 9 installation and cannot benchmark source builds." ) out("") out("You can still benchmark binary distributions with e.g.:" ) out("") out(" %s --distribution-version=6.0.0" % PROGRAM_NAME) out("********************************************************************************" ) out("") current_version = 13 config["meta"]["config.version"] = str(current_version) # all migrations done config_file.store(config) logger.info("Successfully self-upgraded configuration to version [%s]" % target_version)