def detach_from_cluster(self, cluster): data_paths = self.config.opts("provisioning", "local.data.paths", mandatory=False) if data_paths is not None: data_path = data_paths[0] index_size_bytes = io.get_size(data_path) self.metrics_store.put_count_cluster_level("final_index_size_bytes", index_size_bytes, "byte") process.run_subprocess_with_logging("find %s -ls" % data_path, header="index files:")
def detach_from_cluster(self, cluster): if self.attached and self.data_paths: self.attached = False data_path = self.data_paths[0] index_size_bytes = io.get_size(data_path) self.metrics_store.put_count_cluster_level( "final_index_size_bytes", index_size_bytes, "byte") process.run_subprocess_with_logging("find %s -ls" % data_path, header="index files:")
def detach_from_node(self, node, running): # we need to gather the file size after the node has terminated so we can be sure that it has written all its buffers. if not running and self.attached and self.data_paths: self.attached = False index_size_bytes = 0 for data_path in self.data_paths: index_size_bytes += io.get_size(data_path) process.run_subprocess_with_logging("find %s -ls" % data_path, header="index files:") self.metrics_store.put_count_node_level(node.node_name, "final_index_size_bytes", index_size_bytes, "byte")
def detach_from_cluster(self, cluster): data_paths = self.config.opts("provisioning", "local.data.paths", mandatory=False) if data_paths is not None: data_path = data_paths[0] index_size_bytes = io.get_size(data_path) self.metrics_store.put_count_cluster_level( "final_index_size_bytes", index_size_bytes, "byte") process.run_subprocess_with_logging("find %s -ls" % data_path, header="index files:")
def migrate(config_file, current_version, target_version, out=print, i=input): prompter = Prompter(i=i, o=out, assume_defaults=False) logger.info("Upgrading configuration from version [%s] to [%s]." % (current_version, target_version)) # Something is really fishy. We don't want to downgrade the configuration. if current_version >= target_version: raise ConfigError( "The existing config file is available in a later version already. Expected version <= [%s] but found [%s]" % (target_version, current_version)) # but first a backup... config_file.backup() config = config_file.load(interpolation=None) if current_version == 0 and target_version > current_version: logger.info("Migrating config from version [0] to [1]") current_version = 1 config["meta"] = {} config["meta"]["config.version"] = str(current_version) # in version 1 we changed some directories from being absolute to being relative config["system"]["log.root.dir"] = "logs" config["provisioning"]["local.install.dir"] = "install" config["reporting"]["report.base.dir"] = "reports" if current_version == 1 and target_version > current_version: logger.info("Migrating config from version [1] to [2]") current_version = 2 config["meta"]["config.version"] = str(current_version) # no need to ask the user now if we are about to upgrade to version 4 config["reporting"]["datastore.type"] = "in-memory" config["reporting"]["datastore.host"] = "" config["reporting"]["datastore.port"] = "" config["reporting"]["datastore.secure"] = "" config["reporting"]["datastore.user"] = "" config["reporting"]["datastore.password"] = "" config["system"]["env.name"] = "local" if current_version == 2 and target_version > current_version: logger.info("Migrating config from version [2] to [3]") current_version = 3 config["meta"]["config.version"] = str(current_version) # Remove obsolete settings config["reporting"].pop("report.base.dir") config["reporting"].pop("output.html.report.filename") if current_version == 3 and target_version > current_version: root_dir = config["system"]["root.dir"] out(""" ***************************************************************************************** You have an old configuration of Rally. Rally has now a much simpler setup routine which will autodetect lots of settings for you and it also does not require you to setup a metrics store anymore. Rally will now migrate your configuration but if you don't need advanced features like a metrics store, then you should delete the configuration directory: rm -rf {0} and then rerun Rally's configuration routine: {1} configure Please also note you have {2:.1f} GB of data in your current benchmark directory at {3} You might want to clean up this directory also. For more details please see {4} ***************************************************************************************** Pausing for 10 seconds to let you consider this message. """.format( config_file.config_dir, PROGRAM_NAME, convert.bytes_to_gb(io.get_size(root_dir)), root_dir, console.format.link( "https://github.com/elastic/rally/blob/master/CHANGELOG.md#030" ))) time.sleep(10) logger.info("Migrating config from version [3] to [4]") current_version = 4 config["meta"]["config.version"] = str(current_version) if len(config["reporting"]["datastore.host"]) > 0: config["reporting"]["datastore.type"] = "elasticsearch" else: config["reporting"]["datastore.type"] = "in-memory" # Remove obsolete settings config["build"].pop("maven.bin") config["benchmarks"].pop("metrics.stats.disk.device") if current_version == 4 and target_version > current_version: config["tracks"] = {} config["tracks"][ "default.url"] = "https://github.com/elastic/rally-tracks" current_version = 5 config["meta"]["config.version"] = str(current_version) if current_version == 5 and target_version > current_version: config["defaults"] = {} config["defaults"]["preserve_benchmark_candidate"] = str(False) current_version = 6 config["meta"]["config.version"] = str(current_version) if current_version == 6 and target_version > current_version: # Remove obsolete settings config.pop("provisioning") config["system"].pop("log.root.dir") current_version = 7 config["meta"]["config.version"] = str(current_version) if current_version == 7 and target_version > current_version: # move [system][root.dir] to [node][root.dir] if "node" not in config: config["node"] = {} config["node"]["root.dir"] = config["system"].pop("root.dir") # also move all references! for section in config: for k, v in config[section].items(): config[section][k] = v.replace("${system:root.dir}", "${node:root.dir}") current_version = 8 config["meta"]["config.version"] = str(current_version) if current_version == 8 and target_version > current_version: config["teams"] = {} config["teams"][ "default.url"] = "https://github.com/elastic/rally-teams" current_version = 9 config["meta"]["config.version"] = str(current_version) if current_version == 9 and target_version > current_version: config["distributions"] = {} config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \ "{{VERSION}}.tar.gz" config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \ "distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz" config["distributions"][ "release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" config["distributions"]["release.cache"] = "true" current_version = 10 config["meta"]["config.version"] = str(current_version) if current_version == 10 and target_version > current_version: config["runtime"]["java.home"] = config["runtime"].pop("java8.home") current_version = 11 config["meta"]["config.version"] = str(current_version) if current_version == 11 and target_version > current_version: # As this is a rather complex migration, we log more than usual to understand potential migration problems better. if "source" in config: if "local.src.dir" in config["source"]: previous_root = config["source"].pop("local.src.dir") logger.info("Set [source][local.src.dir] to [%s]." % previous_root) # if this directory was Rally's default location, then move it on the file system because to allow for checkouts of plugins # in the sibling directory. if previous_root == os.path.join(config["node"]["root.dir"], "src"): new_root_dir_all_sources = previous_root new_es_sub_dir = "elasticsearch" new_root = os.path.join(new_root_dir_all_sources, new_es_sub_dir) # only attempt to move if the directory exists. It may be possible that users never ran a source benchmark although they # have configured it. In that case the source directory will not yet exist. if io.exists(previous_root): logger.info( "Previous source directory was at Rally's default location [%s]. Moving to [%s]." % (previous_root, new_root)) try: # we need to do this in two steps as we need to move the sources to a subdirectory tmp_path = io.normalize_path( os.path.join(new_root_dir_all_sources, os.pardir, "tmp_src_mig")) os.rename(previous_root, tmp_path) io.ensure_dir(new_root) os.rename(tmp_path, new_root) except OSError: logger.exception( "Could not move source directory from [%s] to [%s]." % (previous_root, new_root)) # A warning is sufficient as Rally should just do a fresh checkout if moving did not work. console.warn( "Elasticsearch source directory could not be moved from [%s] to [%s]. Please check the logs." % (previous_root, new_root)) else: logger.info( "Source directory is configured at Rally's default location [%s] but does not exist yet." % previous_root) else: logger.info( "Previous source directory was the custom directory [%s]." % previous_root) new_root_dir_all_sources = io.normalize_path( os.path.join(previous_root, os.path.pardir)) # name of the elasticsearch project directory. new_es_sub_dir = io.basename(previous_root) logger.info("Setting [node][src.root.dir] to [%s]." % new_root_dir_all_sources) config["node"]["src.root.dir"] = new_root_dir_all_sources logger.info( "Setting [source][elasticsearch.src.subdir] to [%s]" % new_es_sub_dir) config["source"]["elasticsearch.src.subdir"] = new_es_sub_dir else: logger.info( "Key [local.src.dir] not found. Advancing without changes." ) else: logger.info( "No section named [source] found in config. Advancing without changes." ) current_version = 12 config["meta"]["config.version"] = str(current_version) if current_version == 12 and target_version > current_version: # the current configuration allows to benchmark from sources if "build" in config and "gradle.bin" in config["build"]: java_9_home = io.guess_java_home(major_version=9) from esrally.utils import jvm if java_9_home and not jvm.is_early_access_release(java_9_home): logger.debug("Autodetected a JDK 9 installation at [%s]" % java_9_home) if "runtime" not in config: config["runtime"] = {} config["runtime"]["java9.home"] = java_9_home else: logger.debug( "Could not autodetect a JDK 9 installation. Checking [java.home] already points to a JDK 9." ) detected = False if "runtime" in config: java_home = config["runtime"]["java.home"] if jvm.major_version( java_home ) == 9 and not jvm.is_early_access_release(java_home): config["runtime"]["java9.home"] = java_home detected = True if not detected: logger.debug( "Could not autodetect a JDK 9 installation. Asking user." ) raw_java_9_home = prompter.ask_property( "Enter the JDK 9 root directory", check_path_exists=True, mandatory=False) if raw_java_9_home and jvm.major_version( raw_java_9_home ) == 9 and not jvm.is_early_access_release( raw_java_9_home): java_9_home = io.normalize_path( raw_java_9_home) if raw_java_9_home else None config["runtime"]["java9.home"] = java_9_home else: out("********************************************************************************" ) out("You don't have a valid JDK 9 installation and cannot benchmark source builds." ) out("") out("You can still benchmark binary distributions with e.g.:" ) out("") out(" %s --distribution-version=6.0.0" % PROGRAM_NAME) out("********************************************************************************" ) out("") current_version = 13 config["meta"]["config.version"] = str(current_version) # all migrations done config_file.store(config) logger.info("Successfully self-upgraded configuration to version [%s]" % target_version)
def migrate(config_file, current_version, target_version, out=print): logger.info("Upgrading configuration from version [%s] to [%s]." % (current_version, target_version)) # Something is really fishy. We don't want to downgrade the configuration. if current_version >= target_version: raise ConfigError( "The existing config file is available in a later version already. Expected version <= [%s] but found [%s]" % (target_version, current_version)) # but first a backup... config_file.backup() config = config_file.load(interpolation=None) if current_version == 0 and target_version > current_version: logger.info("Migrating config from version [0] to [1]") current_version = 1 config["meta"] = {} config["meta"]["config.version"] = str(current_version) # in version 1 we changed some directories from being absolute to being relative config["system"]["log.root.dir"] = "logs" config["provisioning"]["local.install.dir"] = "install" config["reporting"]["report.base.dir"] = "reports" if current_version == 1 and target_version > current_version: logger.info("Migrating config from version [1] to [2]") current_version = 2 config["meta"]["config.version"] = str(current_version) # no need to ask the user now if we are about to upgrade to version 4 config["reporting"]["datastore.type"] = "in-memory" config["reporting"]["datastore.host"] = "" config["reporting"]["datastore.port"] = "" config["reporting"]["datastore.secure"] = "" config["reporting"]["datastore.user"] = "" config["reporting"]["datastore.password"] = "" config["system"]["env.name"] = "local" if current_version == 2 and target_version > current_version: logger.info("Migrating config from version [2] to [3]") current_version = 3 config["meta"]["config.version"] = str(current_version) # Remove obsolete settings config["reporting"].pop("report.base.dir") config["reporting"].pop("output.html.report.filename") if current_version == 3 and target_version > current_version: root_dir = config["system"]["root.dir"] out("*****************************************************************************************" ) out("") out("You have an old configuration of Rally. Rally has now a much simpler setup" ) out("routine which will autodetect lots of settings for you and it also does not" ) out("require you to setup a metrics store anymore.") out("") out("Rally will now migrate your configuration but if you don't need advanced features" ) out("like a metrics store, then you should delete the configuration directory:" ) out("") out(" rm -rf %s" % config_file.config_dir) out("") out("and then rerun Rally's configuration routine:") out("") out(" %s configure" % PROGRAM_NAME) out("") out("Please also note you have %.1f GB of data in your current benchmark directory at" % convert.bytes_to_gb(io.get_size(root_dir))) out() out(" %s" % root_dir) out("") out("You might want to clean up this directory also.") out() out("For more details please see %s" % console.format.link( "https://github.com/elastic/rally/blob/master/CHANGELOG.md#030")) out("") out("*****************************************************************************************" ) out("") out("Pausing for 10 seconds to let you consider this message.") time.sleep(10) logger.info("Migrating config from version [3] to [4]") current_version = 4 config["meta"]["config.version"] = str(current_version) if len(config["reporting"]["datastore.host"]) > 0: config["reporting"]["datastore.type"] = "elasticsearch" else: config["reporting"]["datastore.type"] = "in-memory" # Remove obsolete settings config["build"].pop("maven.bin") config["benchmarks"].pop("metrics.stats.disk.device") if current_version == 4 and target_version > current_version: config["tracks"] = {} config["tracks"][ "default.url"] = "https://github.com/elastic/rally-tracks" current_version = 5 config["meta"]["config.version"] = str(current_version) if current_version == 5 and target_version > current_version: config["defaults"] = {} config["defaults"]["preserve_benchmark_candidate"] = str(False) current_version = 6 config["meta"]["config.version"] = str(current_version) if current_version == 6 and target_version > current_version: # Remove obsolete settings config.pop("provisioning") config["system"].pop("log.root.dir") current_version = 7 config["meta"]["config.version"] = str(current_version) if current_version == 7 and target_version > current_version: # move [system][root.dir] to [node][root.dir] if "node" not in config: config["node"] = {} config["node"]["root.dir"] = config["system"].pop("root.dir") # also move all references! for section in config: for k, v in config[section].items(): config[section][k] = v.replace("${system:root.dir}", "${node:root.dir}") current_version = 8 config["meta"]["config.version"] = str(current_version) if current_version == 8 and target_version > current_version: config["teams"] = {} config["teams"][ "default.url"] = "https://github.com/elastic/rally-teams" current_version = 9 config["meta"]["config.version"] = str(current_version) if current_version == 9 and target_version > current_version: config["distributions"] = {} config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \ "{{VERSION}}.tar.gz" config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \ "distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz" config["distributions"][ "release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz" config["distributions"]["release.cache"] = "true" current_version = 10 config["meta"]["config.version"] = str(current_version) if current_version == 10 and target_version > current_version: config["runtime"]["java.home"] = config["runtime"].pop("java8.home") current_version = 11 config["meta"]["config.version"] = str(current_version) # all migrations done config_file.store(config) logger.info("Successfully self-upgraded configuration to version [%s]" % target_version)
def migrate(config_file, current_version, target_version, out=print): logger.info("Upgrading configuration from version [%s] to [%s]." % (current_version, target_version)) # Something is really fishy. We don't want to downgrade the configuration. if current_version >= target_version: raise ConfigError("The existing config file is available in a later version already. Expected version <= [%s] but found [%s]" % (target_version, current_version)) # but first a backup... config_file.backup() config = config_file.load(interpolation=None) if current_version == 0 and target_version > current_version: logger.info("Migrating config from version [0] to [1]") current_version = 1 config["meta"] = {} config["meta"]["config.version"] = str(current_version) # in version 1 we changed some directories from being absolute to being relative config["system"]["log.root.dir"] = "logs" config["provisioning"]["local.install.dir"] = "install" config["reporting"]["report.base.dir"] = "reports" if current_version == 1 and target_version > current_version: logger.info("Migrating config from version [1] to [2]") current_version = 2 config["meta"]["config.version"] = str(current_version) # no need to ask the user now if we are about to upgrade to version 4 config["reporting"]["datastore.type"] = "in-memory" config["reporting"]["datastore.host"] = "" config["reporting"]["datastore.port"] = "" config["reporting"]["datastore.secure"] = "" config["reporting"]["datastore.user"] = "" config["reporting"]["datastore.password"] = "" config["system"]["env.name"] = "local" if current_version == 2 and target_version > current_version: logger.info("Migrating config from version [2] to [3]") current_version = 3 config["meta"]["config.version"] = str(current_version) # Remove obsolete settings config["reporting"].pop("report.base.dir") config["reporting"].pop("output.html.report.filename") if current_version == 3 and target_version > current_version: root_dir = config["system"]["root.dir"] out("*****************************************************************************************") out("") out("You have an old configuration of Rally. Rally has now a much simpler setup") out("routine which will autodetect lots of settings for you and it also does not") out("require you to setup a metrics store anymore.") out("") out("Rally will now migrate your configuration but if you don't need advanced features") out("like a metrics store, then you should delete the configuration directory:") out("") out(" rm -rf %s" % config_file.config_dir) out("") out("and then rerun Rally's configuration routine:") out("") out(" %s configure" % PROGRAM_NAME) out("") out("Please also note you have %.1f GB of data in your current benchmark directory at" % convert.bytes_to_gb(io.get_size(root_dir))) out() out(" %s" % root_dir) out("") out("You might want to clean up this directory also.") out() out("For more details please see %s" % console.format.link("https://github.com/elastic/rally/blob/master/CHANGELOG.md#030")) out("") out("*****************************************************************************************") out("") out("Pausing for 10 seconds to let you consider this message.") time.sleep(10) logger.info("Migrating config from version [3] to [4]") current_version = 4 config["meta"]["config.version"] = str(current_version) if len(config["reporting"]["datastore.host"]) > 0: config["reporting"]["datastore.type"] = "elasticsearch" else: config["reporting"]["datastore.type"] = "in-memory" # Remove obsolete settings config["build"].pop("maven.bin") config["benchmarks"].pop("metrics.stats.disk.device") if current_version == 4 and target_version > current_version: config["tracks"] = {} config["tracks"]["default.url"] = "https://github.com/elastic/rally-tracks" current_version = 5 config["meta"]["config.version"] = str(current_version) if current_version == 5 and target_version > current_version: config["defaults"] = {} config["defaults"]["preserve_benchmark_candidate"] = str(False) current_version = 6 config["meta"]["config.version"] = str(current_version) # all migrations done config_file.store(config) logger.info("Successfully self-upgraded configuration to version [%s]" % target_version)
def print_index_stats(self, data_dir): index_size_bytes = io.get_size(data_dir) self.metrics_store.put_count_cluster_level("final_index_size_bytes", index_size_bytes, "byte") process.run_subprocess_with_logging("find %s -ls" % data_dir, header="index files:")