def print_help_on_errors(): heading = "Getting further help:" console.println(console.format.bold(heading)) console.println(console.format.underline_for(heading)) console.println("* Check the log files in {} for errors.".format( paths.logs())) console.println("* Read the documentation at {}".format( console.format.link(doc_link()))) console.println("* Ask a question on the forum at {}".format( console.format.link( "https://discuss.elastic.co/c/elasticsearch/rally"))) console.println( "* Raise an issue at {} and include the log files in {}.".format( console.format.link("https://github.com/elastic/rally/issues"), paths.logs()))
def install_default_log_config(): """ Ensures a log configuration file is present on this machine. The default log configuration is based on the template in resources/logging.json. It also ensures that the default log path has been created so log files can be successfully opened in that directory. """ log_config = log_config_path() if not io.exists(log_config): io.ensure_dir(io.dirname(log_config)) source_path = io.normalize_path( os.path.join(os.path.dirname(__file__), "resources", "logging.json")) with open(log_config, "w", encoding="UTF-8") as target: with open(source_path, "r", encoding="UTF-8") as src: contents = src.read().replace("${LOG_PATH}", paths.logs()) target.write(contents) io.ensure_dir(paths.logs())
def install_default_log_config(): """ Ensures a log configuration file is present on this machine. The default log configuration is based on the template in resources/logging.json. It also ensures that the default log path has been created so log files can be successfully opened in that directory. """ log_config = log_config_path() if not io.exists(log_config): io.ensure_dir(io.dirname(log_config)) source_path = io.normalize_path( os.path.join(os.path.dirname(__file__), "resources", "logging.json")) with open(log_config, "w", encoding="UTF-8") as target: with open(source_path, "r", encoding="UTF-8") as src: # Ensure we have a trailing path separator as after LOG_PATH there will only be the file name log_path = os.path.join(paths.logs(), "") # the logging path might contain backslashes that we need to escape log_path = io.escape_path(log_path) contents = src.read().replace("${LOG_PATH}", log_path) target.write(contents) io.ensure_dir(paths.logs())
def remove_obsolete_default_log_config(): """ Log rotation is problematic because Rally uses multiple processes and there is a lurking race condition when rolling log files. Hence, we do not rotate logs from within Rally and leverage established tools like logrotate for that. Checks whether the user has a problematic out-of-the-box logging configuration delivered with Rally 1.0.0 which used log rotation and removes it so it can be replaced by a new one in a later step. """ log_config = log_config_path() if io.exists(log_config): source_path = io.normalize_path( os.path.join(os.path.dirname(__file__), "resources", "logging_1_0_0.json")) with open(source_path, "r", encoding="UTF-8") as src: contents = src.read().replace("${LOG_PATH}", paths.logs()) source_hash = hashlib.sha512(contents.encode()).hexdigest() with open(log_config, "r", encoding="UTF-8") as target: target_hash = hashlib.sha512(target.read().encode()).hexdigest() if source_hash == target_hash: os.rename(log_config, "{}.bak".format(log_config))
def create(cfg, sources, distribution, build, car, plugins=None): logger = logging.getLogger(__name__) if plugins is None: plugins = [] revisions = _extract_revisions(cfg.opts("mechanic", "source.revision")) distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False) supply_requirements = _supply_requirements(sources, distribution, build, plugins, revisions, distribution_version) build_needed = any([build for _, _, build in supply_requirements.values()]) src_config = cfg.all_opts("source") suppliers = [] if build_needed: java_home = _java_home(car) es_src_dir = os.path.join( _src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir")) builder = Builder(es_src_dir, java_home, paths.logs()) else: builder = None es_supplier_type, es_version, es_build = supply_requirements[ "elasticsearch"] if es_supplier_type == "source": es_src_dir = os.path.join( _src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir")) suppliers.append( ElasticsearchSourceSupplier(es_version, es_src_dir, remote_url=cfg.opts( "source", "remote.repo.url"), car=car, builder=builder)) repo = None else: es_src_dir = None distributions_root = os.path.join( cfg.opts("node", "root.dir"), cfg.opts("source", "distribution.dir")) dist_cfg = {} # car / plugin defines defaults... dist_cfg.update(car.variables) for plugin in plugins: for k, v in plugin.variables.items(): dist_cfg["plugin_{}_{}".format(plugin.name, k)] = v # ... but the user can override it in rally.ini dist_cfg.update(cfg.all_opts("distributions")) repo = DistributionRepository(name=cfg.opts("mechanic", "distribution.repository"), distribution_config=dist_cfg, version=es_version) suppliers.append( ElasticsearchDistributionSupplier(repo, distributions_root)) for plugin in plugins: supplier_type, plugin_version, build_plugin = supply_requirements[ plugin.name] if supplier_type == "source": if CorePluginSourceSupplier.can_handle(plugin): logger.info("Adding core plugin source supplier for [%s].", plugin.name) assert es_src_dir is not None, "Cannot build core plugin %s when Elasticsearch is not built from source." % plugin.name suppliers.append( CorePluginSourceSupplier(plugin, es_src_dir, builder)) elif ExternalPluginSourceSupplier.can_handle(plugin): logger.info("Adding external plugin source supplier for [%s].", plugin.name) suppliers.append( ExternalPluginSourceSupplier( plugin, plugin_version, _src_dir(cfg, mandatory=False), src_config, builder)) else: raise exceptions.RallyError( "Plugin %s can neither be treated as core nor as external plugin. Requirements: %s" % (plugin.name, supply_requirements[plugin.name])) else: logger.info("Adding plugin distribution supplier for [%s].", plugin.name) assert repo is not None, "Cannot benchmark plugin %s from a distribution version but Elasticsearch from sources" % plugin.name suppliers.append(PluginDistributionSupplier(repo, plugin)) return CompositeSupplier(suppliers)
def create(cfg, sources, distribution, car, plugins=None): logger = logging.getLogger(__name__) if plugins is None: plugins = [] caching_enabled = cfg.opts("source", "cache", mandatory=False, default_value=True) revisions = _extract_revisions( cfg.opts("mechanic", "source.revision", mandatory=sources)) distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False) supply_requirements = _supply_requirements(sources, distribution, plugins, revisions, distribution_version) build_needed = any([build for _, _, build in supply_requirements.values()]) es_supplier_type, es_version, _ = supply_requirements["elasticsearch"] src_config = cfg.all_opts("source") suppliers = [] target_os = cfg.opts("mechanic", "target.os", mandatory=False) target_arch = cfg.opts("mechanic", "target.arch", mandatory=False) template_renderer = TemplateRenderer(version=es_version, os_name=target_os, arch=target_arch) if build_needed: raw_build_jdk = car.mandatory_var("build.jdk") try: build_jdk = int(raw_build_jdk) except ValueError: raise exceptions.SystemSetupError( f"Car config key [build.jdk] is invalid: [{raw_build_jdk}] (must be int)" ) es_src_dir = os.path.join( _src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir")) builder = Builder(es_src_dir, build_jdk, paths.logs()) else: builder = None distributions_root = os.path.join(cfg.opts("node", "root.dir"), cfg.opts("source", "distribution.dir")) dist_cfg = {} # car / plugin defines defaults... dist_cfg.update(car.variables) for plugin in plugins: for k, v in plugin.variables.items(): dist_cfg["plugin_{}_{}".format(plugin.name, k)] = v # ... but the user can override it in rally.ini dist_cfg.update(cfg.all_opts("distributions")) if caching_enabled: logger.info("Enabling source artifact caching.") max_age_days = int( cfg.opts("source", "cache.days", mandatory=False, default_value=7)) if max_age_days <= 0: raise exceptions.SystemSetupError( f"cache.days must be a positive number but is {max_age_days}") source_distributions_root = os.path.join(distributions_root, "src") _prune(source_distributions_root, max_age_days) else: logger.info("Disabling source artifact caching.") source_distributions_root = None if es_supplier_type == "source": es_src_dir = os.path.join( _src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir")) source_supplier = ElasticsearchSourceSupplier( es_version, es_src_dir, remote_url=cfg.opts("source", "remote.repo.url"), car=car, builder=builder, template_renderer=template_renderer) if caching_enabled: es_file_resolver = ElasticsearchFileNameResolver( dist_cfg, template_renderer) source_supplier = CachedSourceSupplier(source_distributions_root, source_supplier, es_file_resolver) suppliers.append(source_supplier) repo = None else: es_src_dir = None repo = DistributionRepository(name=cfg.opts("mechanic", "distribution.repository"), distribution_config=dist_cfg, template_renderer=template_renderer) suppliers.append( ElasticsearchDistributionSupplier(repo, es_version, distributions_root)) for plugin in plugins: supplier_type, plugin_version, _ = supply_requirements[plugin.name] if supplier_type == "source": if CorePluginSourceSupplier.can_handle(plugin): logger.info("Adding core plugin source supplier for [%s].", plugin.name) assert es_src_dir is not None, f"Cannot build core plugin {plugin.name} when Elasticsearch is not built from source." plugin_supplier = CorePluginSourceSupplier( plugin, es_src_dir, builder) elif ExternalPluginSourceSupplier.can_handle(plugin): logger.info("Adding external plugin source supplier for [%s].", plugin.name) plugin_supplier = ExternalPluginSourceSupplier( plugin, plugin_version, _src_dir(cfg, mandatory=False), src_config, builder) else: raise exceptions.RallyError( "Plugin %s can neither be treated as core nor as external plugin. Requirements: %s" % (plugin.name, supply_requirements[plugin.name])) if caching_enabled: plugin_file_resolver = PluginFileNameResolver( plugin.name, plugin_version) plugin_supplier = CachedSourceSupplier( source_distributions_root, plugin_supplier, plugin_file_resolver) suppliers.append(plugin_supplier) else: logger.info("Adding plugin distribution supplier for [%s].", plugin.name) assert repo is not None, "Cannot benchmark plugin %s from a distribution version but Elasticsearch from sources" % plugin.name suppliers.append(PluginDistributionSupplier(repo, plugin)) return CompositeSupplier(suppliers)