def get_config(self):
combi_config = copy.deepcopy(self.experiments[0].config)
config_diff = Config.difference_config_static(
*[e.config for e in self.experiments], only_set=True)
combi_config.update(config_diff)
return combi_config
def group_experiments_by(exps, group_by_list):
configs_flat = [e.config.flat()
for e in exps] ### Exclude already combined experiments
config_diff = Config.difference_config_static(*configs_flat)
group_diff_key_list = []
group_diff_val_list = []
for diff_key in group_by_list:
if diff_key in config_diff:
group_diff_key_list.append(diff_key)
group_diff_val_list.append(set(config_diff[diff_key]))
val_combis = itertools.product(*group_diff_val_list)
group_dict = defaultdict(list)
for val_combi in val_combis:
for e in exps:
e_config = e.config.flat()
is_match = True
for key, val in zip(group_diff_key_list, val_combi):
if e_config[key] != val:
is_match = False
if is_match:
group_dict[val_combi].append(e)
return list(group_dict.values())
def run_experiment(experiment, configs, args, mods=None, **kwargs):
# set a few defaults
if "explogger_kwargs" not in kwargs:
kwargs["explogger_kwargs"] = dict(
folder_format="{experiment_name}_%Y%m%d-%H%M%S")
if "explogger_freq" not in kwargs:
kwargs["explogger_freq"] = 1
if "resume_save_types" not in kwargs:
kwargs["resume_save_types"] = ("model", "simple", "th_vars", "results")
config = Config(file_=args.config) if args.config is not None else Config()
config.update_missing(configs[args.default_config].deepcopy())
if args.mods is not None and mods is not None:
for mod in args.mods:
config.update(mods[mod])
config = Config(config=config, update_from_argv=True)
# GET EXISTING EXPERIMENTS TO BE ABLE TO SKIP CERTAIN CONFIGS
if args.skip_existing:
existing_configs = []
for exp in os.listdir(args.base_dir):
try:
existing_configs.append(
Config(file_=os.path.join(args.base_dir, exp, "config",
"config.json")))
except Exception as e:
pass
if args.grid is not None:
grid = GridSearch().read(args.grid)
else:
grid = [{}]
for combi in grid:
config.update(combi)
if args.skip_existing:
skip_this = False
for existing_config in existing_configs:
if existing_config.contains(config):
skip_this = True
break
if skip_this:
continue
if "backup_every" in config:
kwargs["save_checkpoint_every_epoch"] = config["backup_every"]
loggers = {}
if args.visdomlogger:
loggers["v"] = ("visdom", {}, 1)
if args.tensorboardxlogger is not None:
if args.tensorboardxlogger == "same":
loggers["tx"] = ("tensorboard", {}, 1)
else:
loggers["tx"] = ("tensorboard", {
"target_dir": args.tensorboardxlogger
}, 1)
if args.telegramlogger:
kwargs["use_telegram"] = True
if args.automatic_description:
difference_to_default = Config.difference_config_static(
config, configs["DEFAULTS"]).flat(keep_lists=True,
max_split_size=0,
flatten_int=True)
description_str = ""
for key, val in difference_to_default.items():
val = val[0]
description_str = "{} = {}\n{}".format(key, val,
description_str)
config.description = description_str
exp = experiment(config=config,
base_dir=args.base_dir,
resume=args.resume,
ignore_resume_config=args.ignore_resume_config,
loggers=loggers,
**kwargs)
trained = False
if args.resume is None or args.test is False:
exp.run()
trained = True
if args.test:
exp.run_test(setup=not trained)
if isinstance(args.resume,
str) and exp.elog is not None and args.copy_test:
for f in glob.glob(os.path.join(exp.elog.save_dir, "test*")):
if os.path.isdir(f):
shutil.copytree(
f,
os.path.join(args.resume, "save",
os.path.basename(f)))
else:
shutil.copy(f, os.path.join(args.resume, "save"))
def process_base_dir(base_dir,
view_dir="",
default_val="-",
short_len=25,
ignore_keys=IGNORE_KEYS):
"""Create an overview table of all experiments in the given directory.
Args:
directory (str): A directory containing experiment folders.
default_val (str): Default value if an entry is missing.
short_len (int): Cut strings to this length. Full string in alt-text.
Returns:
dict: {"ccols": Columns for config entries,
"rcols": Columns for result entries,
"rows": The actual data}
"""
full_dir = os.path.join(base_dir, view_dir)
config_keys = set()
result_keys = set()
exps = []
non_exps = []
### Load Experiments with keys / different param values
for sub_dir in sorted(os.listdir(full_dir)):
dir_path = os.path.join(full_dir, sub_dir)
if os.path.isdir(dir_path):
try:
exp = ExperimentReader(full_dir, sub_dir)
if exp.ignore:
continue
config_keys.update(list(exp.config.flat().keys()))
result_keys.update(list(exp.get_results().keys()))
exps.append(exp)
except Exception as e:
print("Could not load experiment: ", dir_path)
print(e)
print("-" * 20)
non_exps.append(os.path.join(view_dir, sub_dir))
### Get not common val keys
diff_keys = list(
Config.difference_config_static(*[xp.config for xp in exps]).flat())
### Remove unwanted keys
config_keys -= set(ignore_keys)
result_keys -= set(ignore_keys)
### Generate table rows
sorted_c_keys1 = sorted([c for c in config_keys if c in diff_keys],
key=lambda x: str(x).lower())
sorted_c_keys2 = sorted([c for c in config_keys if c not in diff_keys],
key=lambda x: str(x).lower())
sorted_r_keys = sorted(result_keys, key=lambda x: str(x).lower())
rows = []
for exp in exps:
config_row = []
for key in sorted_c_keys1:
attr_strng = str(exp.config.flat().get(key, default_val))
config_row.append((attr_strng, attr_strng[:short_len]))
for key in sorted_c_keys2:
attr_strng = str(exp.config.flat().get(key, default_val))
config_row.append((attr_strng, attr_strng[:short_len]))
result_row = []
for key in sorted_r_keys:
attr_strng = str(exp.get_results().get(key, default_val))
result_row.append((attr_strng, attr_strng[:short_len]))
name = exp.exp_name
time = exp.exp_info.get(
"time", default_val) if "time" in exp.exp_info else exp.config.get(
"time", default_val)
state = exp.exp_info.get(
"state",
default_val) if "state" in exp.exp_info else exp.config.get(
"state", default_val)
epoch = exp.exp_info.get(
"epoch",
default_val) if "epoch" in exp.exp_info else exp.config.get(
"epoch", default_val)
rows.append((os.path.relpath(exp.work_dir,
base_dir), exp.star, str(name), str(time),
str(state), str(epoch), config_row, result_row))
return {
"ccols1": sorted_c_keys1,
"ccols2": sorted_c_keys2,
"rcols": sorted_r_keys,
"rows": rows,
"noexp": non_exps
}
def experiment(base_dir):
experiment_paths = request.args.getlist('exp')
name = request.args.get('name', "")
do_save = request.args.get('save', "")
combi = request.args.get('combi', 'false')
experiments = []
if combi == "true":
combi_exp = CombiExperimentReader(base_dir,
experiment_paths,
name=name)
if do_save == "true":
combi_exp.save()
experiments = [combi_exp]
else:
# Get all Experiments
for experiment_path in sorted(experiment_paths):
exp = ExperimentReader(base_dir, experiment_path)
experiments.append(exp)
# Assign unique names
exp_names = [exp.exp_name for exp in experiments]
if len(exp_names) > len(set(exp_names)):
for i, exp in enumerate(experiments):
exp.exp_name += str(i)
exp_names = [exp.exp_name for exp in experiments]
# Site Content
content = {}
# Get config
default_val = "-"
combi_config = {}
exp_configs = [exp.config.flat(False) for exp in experiments]
diff_config_keys = list(
Config.difference_config_static(*exp_configs).keys())
config_keys = set([k for c in exp_configs for k in c.keys()])
for k in sorted(config_keys):
combi_config[k] = []
for conf in exp_configs:
combi_config[k].append(conf.get(k, default_val))
config_keys = list(sorted(list(config_keys)))
# Get results
default_val = "-"
combi_results = {}
exp_results = [exp.get_results() for exp in experiments]
result_keys = set([k for r in exp_results for k in r.keys()])
for k in sorted(result_keys):
combi_results[k] = []
for res in exp_results:
combi_results[k].append(res.get(k, default_val))
result_keys = list(sorted(list(result_keys)))
# Get images
images = OrderedDict({})
image_keys = set()
image_path = {}
for exp in experiments:
exp_images = exp.get_images()
img_groups = group_images(exp_images)
images[exp.exp_name] = img_groups
image_path[exp.exp_name] = exp.img_dir
image_keys.update(list(img_groups.keys()))
image_keys = list(image_keys)
image_keys.sort()
# Get Plots
plots = OrderedDict({})
for exp in experiments:
exp_plots = exp.get_plots()
plots[exp.exp_name] = exp_plots
# Get logs
logs_dict = OrderedDict({})
for exp in experiments:
exp_logs = [(os.path.basename(l), exp.exp_dir) for l in exp.get_logs()]
logs_dict[exp.exp_name] = exp_logs
content["title"] = experiments
content["images"] = {
"img_path": image_path,
"imgs": images,
"img_keys": image_keys
}
content["plots"] = {"plots": plots}
content["config"] = {
"exps": experiments,
"configs": combi_config,
"keys": config_keys,
"diff_keys": diff_config_keys
}
content["results"] = {
"exps": exp_names,
"results": combi_results,
"keys": result_keys
}
content["logs"] = {"logs_dict": logs_dict}
return render_template('experiment.html', **content)