def check_config(): # check configures # Ensure mandatory configures are available for _, prop_name in HiBenchEnvPropMappingMandatory.items(): assert HibenchConf.get(prop_name, None) is not None, "Mandatory configure missing: %s" % prop_name # Ensure all ref values in configure has been expanded for _, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items(): assert "${" not in HibenchConf.get(prop_name, ""), "Unsolved ref key: %s. \n Defined at %s:\n Unsolved value:%s\n" % (prop_name, HibenchConfRef.get(prop_name, "unknown"), HibenchConf.get(prop_name, "unknown"))
def check_config(): # check configures # Ensure mandatory configures are available for _, prop_name in HiBenchEnvPropMappingMandatory.items(): assert HibenchConf.get( prop_name, None) is not None, "Mandatory configure missing: %s" % prop_name # Ensure all ref values in configure has been expanded for _, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items(): assert "${" not in HibenchConf.get(prop_name, ""), "Unsolved ref key: %s. \n Defined at %s:\n Unsolved value:%s\n" % ( prop_name, HibenchConfRef.get(prop_name, "unknown"), HibenchConf.get(prop_name, "unknown"))
def export_config(workload_name, framework_name): join = os.path.join report_dir = HibenchConf['hibench.report.dir'] conf_dir = join(report_dir, workload_name, framework_name, 'conf') conf_filename = join(conf_dir, "%s.conf" % workload_name) spark_conf_dir = join(conf_dir, "sparkbench") spark_prop_conf_filename = join(spark_conf_dir, "spark.conf") sparkbench_prop_conf_filename = join(spark_conf_dir, "sparkbench.conf") if not os.path.exists(spark_conf_dir): os.makedirs(spark_conf_dir) if not os.path.exists(conf_dir): os.makedirs(conf_dir) # generate configure for hibench sources = defaultdict(list) for env_name, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items(): source = HibenchConfRef.get(prop_name, 'None') sources[source].append('%s=%s' % (env_name, HibenchConf.get(prop_name, ''))) with open(conf_filename, 'w') as f: for source in sorted(sources.keys()): f.write("# Source: %s\n" % source) f.write("\n".join(sorted(sources[source]))) f.write("\n\n") f.write("#Source: add for internal usage\n") f.write("SPARKBENCH_PROPERTIES_FILES=%s\n" % sparkbench_prop_conf_filename) f.write("SPARK_PROP_CONF=%s\n" % spark_prop_conf_filename) f.write("WORKLOAD_RESULT_FOLDER=%s\n" % join(conf_dir, "..")) f.write("HIBENCH_WORKLOAD_CONF=%s\n" % conf_filename) f.write("export HADOOP_EXECUTABLE\n") f.write("export HADOOP_CONF_DIR\n") # generate properties for spark & sparkbench sources = defaultdict(list) for prop_name, prop_value in HibenchConf.items(): source = HibenchConfRef.get(prop_name, 'None') sources[source].append('%s\t%s' % (prop_name, prop_value)) # generate configure for sparkbench with open(spark_prop_conf_filename, 'w') as f: for source in sorted(sources.keys()): items = [x for x in sources[source] if x.startswith("spark.")] if items: f.write("# Source: %s\n" % source) f.write("\n".join(sorted(items))) f.write("\n\n") # generate configure for spark with open(sparkbench_prop_conf_filename, 'w') as f: for source in sorted(sources.keys()): items = [x for x in sources[source] if x.startswith( "sparkbench.") or x.startswith("hibench.")] if items: f.write("# Source: %s\n" % source) f.write("\n".join(sorted(items))) f.write("\n\n") return conf_filename
def load_config(conf_root, workload_root, workload_folder, patching_config=""): abspath = os.path.abspath conf_root = abspath(conf_root) workload_root = abspath(workload_root) workload_folder = abspath(workload_folder) workload_tail = workload_folder[len(workload_root) :][1:] workload_api = os.path.dirname(workload_tail) if os.path.dirname(workload_tail) else workload_tail workload_name = os.path.basename(workload_root) conf_files = ( sorted(glob.glob(conf_root + "/*.conf")) + sorted(glob.glob("%s/conf/*.conf" % (workload_root,))) + sorted(glob.glob("%s/%s/*.conf" % (workload_root, workload_api))) ) # load values from conf files for filename in conf_files: log("Parsing conf: %s" % filename) with open(filename) as f: for line in f.readlines(): line = line.strip() if not line: continue # skip empty lines if line[0] == "#": continue # skip comments try: key, value = re.split("\s", line, 1) except ValueError: key = line.strip() value = "" HibenchConf[key] = value.strip() HibenchConfRef[key] = filename # override values from os environment variable settings for env_name, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items(): if env_name in os.environ: env_value = os.getenv(env_name) HibenchConf[prop_name] = env_value HibenchConfRef[prop_name] = "OS environment variable:%s" % env_name # override values by patching config for item in [x for x in patching_config.split(",") if x]: key, value = re.split("=", item, 1) HibenchConf[key] = value.strip() HibenchConfRef[key] = "Overrided by parent script during calling: " + item # generate ref values waterfall_config() # generate auto probe values generate_optional_value() # generate ref values again to ensure all values can be found waterfall_config(force=True) # check check_config() # Export config to file, let bash script to import as local variables. print export_config(workload_name, workload_api)
def load_config(conf_root, workload_root, workload_folder, patching_config=""): abspath = os.path.abspath conf_root = abspath(conf_root) workload_root = abspath(workload_root) workload_folder = abspath(workload_folder) workload_tail = workload_folder[len(workload_root):][1:] workload_api = os.path.dirname(workload_tail) if os.path.dirname( workload_tail) else workload_tail workload_name = os.path.basename(workload_root) conf_files = sorted(glob.glob(conf_root+"/*.conf")) + \ sorted(glob.glob("%s/conf/*.conf" % (workload_root,))) + \ sorted(glob.glob("%s/%s/*.conf" % (workload_root, workload_api))) # load values from conf files for filename in conf_files: log("Parsing conf: %s" % filename) with open(filename) as f: for line in f.readlines(): line = line.strip() if not line: continue # skip empty lines if line[0] == '#': continue # skip comments try: key, value = re.split("\s", line, 1) except ValueError: key = line.strip() value = "" HibenchConf[key] = value.strip() HibenchConfRef[key] = filename # override values from os environment variable settings for env_name, prop_name in HiBenchEnvPropMappingMandatory.items( ) + HiBenchEnvPropMapping.items(): if env_name in os.environ: env_value = os.getenv(env_name) HibenchConf[prop_name] = env_value HibenchConfRef[prop_name] = "OS environment variable:%s" % env_name # override values by patching config for item in [x for x in patching_config.split(',') if x]: key, value = re.split('=', item, 1) HibenchConf[key] = value.strip() HibenchConfRef[ key] = "Overrided by parent script during calling: " + item # generate ref values waterfall_config() # generate auto probe values generate_optional_value() # generate ref values again to ensure all values can be found waterfall_config(force=True) # check check_config() # Export config to file, let bash script to import as local variables. print export_config(workload_name, workload_api)
def override_conf_from_environment(): # override values from os environment variable settings for env_name, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items(): # The overrides from environments has 2 premises, the second one is either # the prop_name is not set in advance by config files or the conf line # itself set an env variable to a hibench conf if env_name in os.environ and (not HibenchConf.get( prop_name) or HibenchConf.get(prop_name) == "$" + env_name): env_value = os.getenv(env_name) HibenchConf[prop_name] = env_value HibenchConfRef[prop_name] = "OS environment variable:%s" % env_name