def process_base_dir(base_dir,
view_dir="",
default_val="-",
short_len=25,
ignore_keys=IGNORE_KEYS):
"""Create an overview table of all experiments in the given directory.
Args:
directory (str): A directory containing experiment folders.
default_val (str): Default value if an entry is missing.
short_len (int): Cut strings to this length. Full string in alt-text.
Returns:
dict: {"ccols": Columns for config entries,
"rcols": Columns for result entries,
"rows": The actual data}
"""
full_dir = os.path.join(base_dir, view_dir)
config_keys = set()
result_keys = set()
exps = []
non_exps = []
### Load Experiments with keys / different param values
for sub_dir in sorted(os.listdir(full_dir)):
dir_path = os.path.join(full_dir, sub_dir)
if os.path.isdir(dir_path):
try:
exp = ExperimentReader(full_dir, sub_dir)
if exp.ignore:
continue
config_keys.update(list(exp.config.flat().keys()))
result_keys.update(list(exp.get_results().keys()))
exps.append(exp)
except Exception as e:
print("Could not load experiment: ", dir_path)
print(e)
print("-" * 20)
non_exps.append(os.path.join(view_dir, sub_dir))
### Get not common val keys
diff_keys = list(
Config.difference_config_static(*[xp.config for xp in exps]).flat())
### Remove unwanted keys
config_keys -= set(ignore_keys)
result_keys -= set(ignore_keys)
### Generate table rows
sorted_c_keys1 = sorted([c for c in config_keys if c in diff_keys],
key=lambda x: str(x).lower())
sorted_c_keys2 = sorted([c for c in config_keys if c not in diff_keys],
key=lambda x: str(x).lower())
sorted_r_keys = sorted(result_keys, key=lambda x: str(x).lower())
rows = []
for exp in exps:
config_row = []
for key in sorted_c_keys1:
attr_strng = str(exp.config.flat().get(key, default_val))
config_row.append((attr_strng, attr_strng[:short_len]))
for key in sorted_c_keys2:
attr_strng = str(exp.config.flat().get(key, default_val))
config_row.append((attr_strng, attr_strng[:short_len]))
result_row = []
for key in sorted_r_keys:
attr_strng = str(exp.get_results().get(key, default_val))
result_row.append((attr_strng, attr_strng[:short_len]))
name = exp.exp_name
time = exp.exp_info.get(
"time", default_val) if "time" in exp.exp_info else exp.config.get(
"time", default_val)
state = exp.exp_info.get(
"state",
default_val) if "state" in exp.exp_info else exp.config.get(
"state", default_val)
epoch = exp.exp_info.get(
"epoch",
default_val) if "epoch" in exp.exp_info else exp.config.get(
"epoch", default_val)
rows.append((os.path.relpath(exp.work_dir,
base_dir), exp.star, str(name), str(time),
str(state), str(epoch), config_row, result_row))
return {
"ccols1": sorted_c_keys1,
"ccols2": sorted_c_keys2,
"rcols": sorted_r_keys,
"rows": rows,
"noexp": non_exps
}
def experiment(base_dir):
experiment_paths = request.args.getlist('exp')
name = request.args.get('name', "")
do_save = request.args.get('save', "")
combi = request.args.get('combi', 'false')
experiments = []
if combi == "true":
combi_exp = CombiExperimentReader(base_dir,
experiment_paths,
name=name)
if do_save == "true":
combi_exp.save()
experiments = [combi_exp]
else:
# Get all Experiments
for experiment_path in sorted(experiment_paths):
exp = ExperimentReader(base_dir, experiment_path)
experiments.append(exp)
# Assign unique names
exp_names = [exp.exp_name for exp in experiments]
if len(exp_names) > len(set(exp_names)):
for i, exp in enumerate(experiments):
exp.exp_name += str(i)
exp_names = [exp.exp_name for exp in experiments]
# Site Content
content = {}
# Get config
default_val = "-"
combi_config = {}
exp_configs = [exp.config.flat(False) for exp in experiments]
diff_config_keys = list(
Config.difference_config_static(*exp_configs).keys())
config_keys = set([k for c in exp_configs for k in c.keys()])
for k in sorted(config_keys):
combi_config[k] = []
for conf in exp_configs:
combi_config[k].append(conf.get(k, default_val))
config_keys = list(sorted(list(config_keys)))
# Get results
default_val = "-"
combi_results = {}
exp_results = [exp.get_results() for exp in experiments]
result_keys = set([k for r in exp_results for k in r.keys()])
for k in sorted(result_keys):
combi_results[k] = []
for res in exp_results:
combi_results[k].append(res.get(k, default_val))
result_keys = list(sorted(list(result_keys)))
# Get images
images = OrderedDict({})
image_keys = set()
image_path = {}
for exp in experiments:
exp_images = exp.get_images()
img_groups = group_images(exp_images)
images[exp.exp_name] = img_groups
image_path[exp.exp_name] = exp.img_dir
image_keys.update(list(img_groups.keys()))
image_keys = list(image_keys)
image_keys.sort()
# Get Plots
plots = OrderedDict({})
for exp in experiments:
exp_plots = exp.get_plots()
plots[exp.exp_name] = exp_plots
# Get logs
logs_dict = OrderedDict({})
for exp in experiments:
exp_logs = [(os.path.basename(l), exp.exp_dir) for l in exp.get_logs()]
logs_dict[exp.exp_name] = exp_logs
content["title"] = experiments
content["images"] = {
"img_path": image_path,
"imgs": images,
"img_keys": image_keys
}
content["plots"] = {"plots": plots}
content["config"] = {
"exps": experiments,
"configs": combi_config,
"keys": config_keys,
"diff_keys": diff_config_keys
}
content["results"] = {
"exps": exp_names,
"results": combi_results,
"keys": result_keys
}
content["logs"] = {"logs_dict": logs_dict}
return render_template('experiment.html', **content)