def check_config(): # check configures
# Ensure mandatory configures are available
for _, prop_name in HiBenchEnvPropMappingMandatory.items():
assert HibenchConf.get(prop_name, None) is not None, "Mandatory configure missing: %s" % prop_name
# Ensure all ref values in configure has been expanded
for _, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items():
assert "${" not in HibenchConf.get(prop_name, ""), "Unsolved ref key: %s. \n Defined at %s:\n Unsolved value:%s\n" % (prop_name,
HibenchConfRef.get(prop_name, "unknown"),
HibenchConf.get(prop_name, "unknown"))
def check_config(): # check configures
# Ensure mandatory configures are available
for _, prop_name in HiBenchEnvPropMappingMandatory.items():
assert HibenchConf.get(
prop_name, None) is not None, "Mandatory configure missing: %s" % prop_name
# Ensure all ref values in configure has been expanded
for _, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items():
assert "${" not in HibenchConf.get(prop_name, ""), "Unsolved ref key: %s. \n Defined at %s:\n Unsolved value:%s\n" % (
prop_name, HibenchConfRef.get(prop_name, "unknown"), HibenchConf.get(prop_name, "unknown"))
def export_config(workload_name, framework_name):
join = os.path.join
report_dir = HibenchConf['hibench.report.dir']
conf_dir = join(report_dir, workload_name, framework_name, 'conf')
conf_filename = join(conf_dir, "%s.conf" % workload_name)
spark_conf_dir = join(conf_dir, "sparkbench")
spark_prop_conf_filename = join(spark_conf_dir, "spark.conf")
sparkbench_prop_conf_filename = join(spark_conf_dir, "sparkbench.conf")
if not os.path.exists(spark_conf_dir):
os.makedirs(spark_conf_dir)
if not os.path.exists(conf_dir):
os.makedirs(conf_dir)
# generate configure for hibench
sources = defaultdict(list)
for env_name, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items():
source = HibenchConfRef.get(prop_name, 'None')
sources[source].append('%s=%s' % (env_name, HibenchConf.get(prop_name, '')))
with open(conf_filename, 'w') as f:
for source in sorted(sources.keys()):
f.write("# Source: %s\n" % source)
f.write("\n".join(sorted(sources[source])))
f.write("\n\n")
f.write("#Source: add for internal usage\n")
f.write("SPARKBENCH_PROPERTIES_FILES=%s\n" % sparkbench_prop_conf_filename)
f.write("SPARK_PROP_CONF=%s\n" % spark_prop_conf_filename)
f.write("WORKLOAD_RESULT_FOLDER=%s\n" % join(conf_dir, ".."))
f.write("HIBENCH_WORKLOAD_CONF=%s\n" % conf_filename)
f.write("export HADOOP_EXECUTABLE\n")
f.write("export HADOOP_CONF_DIR\n")
# generate properties for spark & sparkbench
sources = defaultdict(list)
for prop_name, prop_value in HibenchConf.items():
source = HibenchConfRef.get(prop_name, 'None')
sources[source].append('%s\t%s' % (prop_name, prop_value))
# generate configure for sparkbench
with open(spark_prop_conf_filename, 'w') as f:
for source in sorted(sources.keys()):
items = [x for x in sources[source] if x.startswith("spark.")]
if items:
f.write("# Source: %s\n" % source)
f.write("\n".join(sorted(items)))
f.write("\n\n")
# generate configure for spark
with open(sparkbench_prop_conf_filename, 'w') as f:
for source in sorted(sources.keys()):
items = [x for x in sources[source] if x.startswith(
"sparkbench.") or x.startswith("hibench.")]
if items:
f.write("# Source: %s\n" % source)
f.write("\n".join(sorted(items)))
f.write("\n\n")
return conf_filename
def export_config(workload_name, framework_name):
join = os.path.join
report_dir = HibenchConf['hibench.report.dir']
conf_dir = join(report_dir, workload_name, framework_name, 'conf')
conf_filename = join(conf_dir, "%s.conf" % workload_name)
spark_conf_dir = join(conf_dir, "sparkbench")
spark_prop_conf_filename = join(spark_conf_dir, "spark.conf")
sparkbench_prop_conf_filename = join(spark_conf_dir, "sparkbench.conf")
if not os.path.exists(spark_conf_dir):
os.makedirs(spark_conf_dir)
if not os.path.exists(conf_dir):
os.makedirs(conf_dir)
# generate configure for hibench
sources = defaultdict(list)
for env_name, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items():
source = HibenchConfRef.get(prop_name, 'None')
sources[source].append('%s=%s' % (env_name, HibenchConf.get(prop_name, '')))
with open(conf_filename, 'w') as f:
for source in sorted(sources.keys()):
f.write("# Source: %s\n" % source)
f.write("\n".join(sorted(sources[source])))
f.write("\n\n")
f.write("#Source: add for internal usage\n")
f.write("SPARKBENCH_PROPERTIES_FILES=%s\n" % sparkbench_prop_conf_filename)
f.write("SPARK_PROP_CONF=%s\n" % spark_prop_conf_filename)
f.write("WORKLOAD_RESULT_FOLDER=%s\n" % join(conf_dir, ".."))
f.write("HIBENCH_WORKLOAD_CONF=%s\n" % conf_filename)
f.write("export HADOOP_EXECUTABLE\n")
f.write("export HADOOP_CONF_DIR\n")
# generate properties for spark & sparkbench
sources = defaultdict(list)
for prop_name, prop_value in HibenchConf.items():
source = HibenchConfRef.get(prop_name, 'None')
sources[source].append('%s\t%s' % (prop_name, prop_value))
# generate configure for sparkbench
with open(spark_prop_conf_filename, 'w') as f:
for source in sorted(sources.keys()):
items = [x for x in sources[source] if x.startswith("spark.")]
if items:
f.write("# Source: %s\n" % source)
f.write("\n".join(sorted(items)))
f.write("\n\n")
# generate configure for spark
with open(sparkbench_prop_conf_filename, 'w') as f:
for source in sorted(sources.keys()):
items = [x for x in sources[source] if x.startswith(
"sparkbench.") or x.startswith("hibench.")]
if items:
f.write("# Source: %s\n" % source)
f.write("\n".join(sorted(items)))
f.write("\n\n")
return conf_filename
def load_config(conf_root, workload_root, workload_folder, patching_config=""):
abspath = os.path.abspath
conf_root = abspath(conf_root)
workload_root = abspath(workload_root)
workload_folder = abspath(workload_folder)
workload_tail = workload_folder[len(workload_root) :][1:]
workload_api = os.path.dirname(workload_tail) if os.path.dirname(workload_tail) else workload_tail
workload_name = os.path.basename(workload_root)
conf_files = (
sorted(glob.glob(conf_root + "/*.conf"))
+ sorted(glob.glob("%s/conf/*.conf" % (workload_root,)))
+ sorted(glob.glob("%s/%s/*.conf" % (workload_root, workload_api)))
)
# load values from conf files
for filename in conf_files:
log("Parsing conf: %s" % filename)
with open(filename) as f:
for line in f.readlines():
line = line.strip()
if not line:
continue # skip empty lines
if line[0] == "#":
continue # skip comments
try:
key, value = re.split("\s", line, 1)
except ValueError:
key = line.strip()
value = ""
HibenchConf[key] = value.strip()
HibenchConfRef[key] = filename
# override values from os environment variable settings
for env_name, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items():
if env_name in os.environ:
env_value = os.getenv(env_name)
HibenchConf[prop_name] = env_value
HibenchConfRef[prop_name] = "OS environment variable:%s" % env_name
# override values by patching config
for item in [x for x in patching_config.split(",") if x]:
key, value = re.split("=", item, 1)
HibenchConf[key] = value.strip()
HibenchConfRef[key] = "Overrided by parent script during calling: " + item
# generate ref values
waterfall_config()
# generate auto probe values
generate_optional_value()
# generate ref values again to ensure all values can be found
waterfall_config(force=True)
# check
check_config()
# Export config to file, let bash script to import as local variables.
print export_config(workload_name, workload_api)
def load_config(conf_root, workload_root, workload_folder, patching_config=""):
abspath = os.path.abspath
conf_root = abspath(conf_root)
workload_root = abspath(workload_root)
workload_folder = abspath(workload_folder)
workload_tail = workload_folder[len(workload_root):][1:]
workload_api = os.path.dirname(workload_tail) if os.path.dirname(
workload_tail) else workload_tail
workload_name = os.path.basename(workload_root)
conf_files = sorted(glob.glob(conf_root+"/*.conf")) + \
sorted(glob.glob("%s/conf/*.conf" % (workload_root,))) + \
sorted(glob.glob("%s/%s/*.conf" % (workload_root, workload_api)))
# load values from conf files
for filename in conf_files:
log("Parsing conf: %s" % filename)
with open(filename) as f:
for line in f.readlines():
line = line.strip()
if not line: continue # skip empty lines
if line[0] == '#': continue # skip comments
try:
key, value = re.split("\s", line, 1)
except ValueError:
key = line.strip()
value = ""
HibenchConf[key] = value.strip()
HibenchConfRef[key] = filename
# override values from os environment variable settings
for env_name, prop_name in HiBenchEnvPropMappingMandatory.items(
) + HiBenchEnvPropMapping.items():
if env_name in os.environ:
env_value = os.getenv(env_name)
HibenchConf[prop_name] = env_value
HibenchConfRef[prop_name] = "OS environment variable:%s" % env_name
# override values by patching config
for item in [x for x in patching_config.split(',') if x]:
key, value = re.split('=', item, 1)
HibenchConf[key] = value.strip()
HibenchConfRef[
key] = "Overrided by parent script during calling: " + item
# generate ref values
waterfall_config()
# generate auto probe values
generate_optional_value()
# generate ref values again to ensure all values can be found
waterfall_config(force=True)
# check
check_config()
# Export config to file, let bash script to import as local variables.
print export_config(workload_name, workload_api)
def override_conf_from_environment():
# override values from os environment variable settings
for env_name, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items():
# The overrides from environments has 2 premises, the second one is either
# the prop_name is not set in advance by config files or the conf line
# itself set an env variable to a hibench conf
if env_name in os.environ and (not HibenchConf.get(
prop_name) or HibenchConf.get(prop_name) == "$" + env_name):
env_value = os.getenv(env_name)
HibenchConf[prop_name] = env_value
HibenchConfRef[prop_name] = "OS environment variable:%s" % env_name
def override_conf_from_environment():
# override values from os environment variable settings
for env_name, prop_name in HiBenchEnvPropMappingMandatory.items() + HiBenchEnvPropMapping.items():
# The overrides from environments has 2 premises, the second one is either
# the prop_name is not set in advance by config files or the conf line
# itself set an env variable to a hibench conf
if env_name in os.environ and (not HibenchConf.get(
prop_name) or HibenchConf.get(prop_name) == "$" + env_name):
env_value = os.getenv(env_name)
HibenchConf[prop_name] = env_value
HibenchConfRef[prop_name] = "OS environment variable:%s" % env_name
