def get_config(self): combi_config = copy.deepcopy(self.experiments[0].config) config_diff = Config.difference_config_static( *[e.config for e in self.experiments], only_set=True) combi_config.update(config_diff) return combi_config
def group_experiments_by(exps, group_by_list): configs_flat = [e.config.flat() for e in exps] ### Exclude already combined experiments config_diff = Config.difference_config_static(*configs_flat) group_diff_key_list = [] group_diff_val_list = [] for diff_key in group_by_list: if diff_key in config_diff: group_diff_key_list.append(diff_key) group_diff_val_list.append(set(config_diff[diff_key])) val_combis = itertools.product(*group_diff_val_list) group_dict = defaultdict(list) for val_combi in val_combis: for e in exps: e_config = e.config.flat() is_match = True for key, val in zip(group_diff_key_list, val_combi): if e_config[key] != val: is_match = False if is_match: group_dict[val_combi].append(e) return list(group_dict.values())
def run_experiment(experiment, configs, args, mods=None, **kwargs): # set a few defaults if "explogger_kwargs" not in kwargs: kwargs["explogger_kwargs"] = dict( folder_format="{experiment_name}_%Y%m%d-%H%M%S") if "explogger_freq" not in kwargs: kwargs["explogger_freq"] = 1 if "resume_save_types" not in kwargs: kwargs["resume_save_types"] = ("model", "simple", "th_vars", "results") config = Config(file_=args.config) if args.config is not None else Config() config.update_missing(configs[args.default_config].deepcopy()) if args.mods is not None and mods is not None: for mod in args.mods: config.update(mods[mod]) config = Config(config=config, update_from_argv=True) # GET EXISTING EXPERIMENTS TO BE ABLE TO SKIP CERTAIN CONFIGS if args.skip_existing: existing_configs = [] for exp in os.listdir(args.base_dir): try: existing_configs.append( Config(file_=os.path.join(args.base_dir, exp, "config", "config.json"))) except Exception as e: pass if args.grid is not None: grid = GridSearch().read(args.grid) else: grid = [{}] for combi in grid: config.update(combi) if args.skip_existing: skip_this = False for existing_config in existing_configs: if existing_config.contains(config): skip_this = True break if skip_this: continue if "backup_every" in config: kwargs["save_checkpoint_every_epoch"] = config["backup_every"] loggers = {} if args.visdomlogger: loggers["v"] = ("visdom", {}, 1) if args.tensorboardxlogger is not None: if args.tensorboardxlogger == "same": loggers["tx"] = ("tensorboard", {}, 1) else: loggers["tx"] = ("tensorboard", { "target_dir": args.tensorboardxlogger }, 1) if args.telegramlogger: kwargs["use_telegram"] = True if args.automatic_description: difference_to_default = Config.difference_config_static( config, configs["DEFAULTS"]).flat(keep_lists=True, max_split_size=0, flatten_int=True) description_str = "" for key, val in difference_to_default.items(): val = val[0] description_str = "{} = {}\n{}".format(key, val, description_str) config.description = description_str exp = experiment(config=config, base_dir=args.base_dir, resume=args.resume, ignore_resume_config=args.ignore_resume_config, loggers=loggers, **kwargs) trained = False if args.resume is None or args.test is False: exp.run() trained = True if args.test: exp.run_test(setup=not trained) if isinstance(args.resume, str) and exp.elog is not None and args.copy_test: for f in glob.glob(os.path.join(exp.elog.save_dir, "test*")): if os.path.isdir(f): shutil.copytree( f, os.path.join(args.resume, "save", os.path.basename(f))) else: shutil.copy(f, os.path.join(args.resume, "save"))
def process_base_dir(base_dir, view_dir="", default_val="-", short_len=25, ignore_keys=IGNORE_KEYS): """Create an overview table of all experiments in the given directory. Args: directory (str): A directory containing experiment folders. default_val (str): Default value if an entry is missing. short_len (int): Cut strings to this length. Full string in alt-text. Returns: dict: {"ccols": Columns for config entries, "rcols": Columns for result entries, "rows": The actual data} """ full_dir = os.path.join(base_dir, view_dir) config_keys = set() result_keys = set() exps = [] non_exps = [] ### Load Experiments with keys / different param values for sub_dir in sorted(os.listdir(full_dir)): dir_path = os.path.join(full_dir, sub_dir) if os.path.isdir(dir_path): try: exp = ExperimentReader(full_dir, sub_dir) if exp.ignore: continue config_keys.update(list(exp.config.flat().keys())) result_keys.update(list(exp.get_results().keys())) exps.append(exp) except Exception as e: print("Could not load experiment: ", dir_path) print(e) print("-" * 20) non_exps.append(os.path.join(view_dir, sub_dir)) ### Get not common val keys diff_keys = list( Config.difference_config_static(*[xp.config for xp in exps]).flat()) ### Remove unwanted keys config_keys -= set(ignore_keys) result_keys -= set(ignore_keys) ### Generate table rows sorted_c_keys1 = sorted([c for c in config_keys if c in diff_keys], key=lambda x: str(x).lower()) sorted_c_keys2 = sorted([c for c in config_keys if c not in diff_keys], key=lambda x: str(x).lower()) sorted_r_keys = sorted(result_keys, key=lambda x: str(x).lower()) rows = [] for exp in exps: config_row = [] for key in sorted_c_keys1: attr_strng = str(exp.config.flat().get(key, default_val)) config_row.append((attr_strng, attr_strng[:short_len])) for key in sorted_c_keys2: attr_strng = str(exp.config.flat().get(key, default_val)) config_row.append((attr_strng, attr_strng[:short_len])) result_row = [] for key in sorted_r_keys: attr_strng = str(exp.get_results().get(key, default_val)) result_row.append((attr_strng, attr_strng[:short_len])) name = exp.exp_name time = exp.exp_info.get( "time", default_val) if "time" in exp.exp_info else exp.config.get( "time", default_val) state = exp.exp_info.get( "state", default_val) if "state" in exp.exp_info else exp.config.get( "state", default_val) epoch = exp.exp_info.get( "epoch", default_val) if "epoch" in exp.exp_info else exp.config.get( "epoch", default_val) rows.append((os.path.relpath(exp.work_dir, base_dir), exp.star, str(name), str(time), str(state), str(epoch), config_row, result_row)) return { "ccols1": sorted_c_keys1, "ccols2": sorted_c_keys2, "rcols": sorted_r_keys, "rows": rows, "noexp": non_exps }
def experiment(base_dir): experiment_paths = request.args.getlist('exp') name = request.args.get('name', "") do_save = request.args.get('save', "") combi = request.args.get('combi', 'false') experiments = [] if combi == "true": combi_exp = CombiExperimentReader(base_dir, experiment_paths, name=name) if do_save == "true": combi_exp.save() experiments = [combi_exp] else: # Get all Experiments for experiment_path in sorted(experiment_paths): exp = ExperimentReader(base_dir, experiment_path) experiments.append(exp) # Assign unique names exp_names = [exp.exp_name for exp in experiments] if len(exp_names) > len(set(exp_names)): for i, exp in enumerate(experiments): exp.exp_name += str(i) exp_names = [exp.exp_name for exp in experiments] # Site Content content = {} # Get config default_val = "-" combi_config = {} exp_configs = [exp.config.flat(False) for exp in experiments] diff_config_keys = list( Config.difference_config_static(*exp_configs).keys()) config_keys = set([k for c in exp_configs for k in c.keys()]) for k in sorted(config_keys): combi_config[k] = [] for conf in exp_configs: combi_config[k].append(conf.get(k, default_val)) config_keys = list(sorted(list(config_keys))) # Get results default_val = "-" combi_results = {} exp_results = [exp.get_results() for exp in experiments] result_keys = set([k for r in exp_results for k in r.keys()]) for k in sorted(result_keys): combi_results[k] = [] for res in exp_results: combi_results[k].append(res.get(k, default_val)) result_keys = list(sorted(list(result_keys))) # Get images images = OrderedDict({}) image_keys = set() image_path = {} for exp in experiments: exp_images = exp.get_images() img_groups = group_images(exp_images) images[exp.exp_name] = img_groups image_path[exp.exp_name] = exp.img_dir image_keys.update(list(img_groups.keys())) image_keys = list(image_keys) image_keys.sort() # Get Plots plots = OrderedDict({}) for exp in experiments: exp_plots = exp.get_plots() plots[exp.exp_name] = exp_plots # Get logs logs_dict = OrderedDict({}) for exp in experiments: exp_logs = [(os.path.basename(l), exp.exp_dir) for l in exp.get_logs()] logs_dict[exp.exp_name] = exp_logs content["title"] = experiments content["images"] = { "img_path": image_path, "imgs": images, "img_keys": image_keys } content["plots"] = {"plots": plots} content["config"] = { "exps": experiments, "configs": combi_config, "keys": config_keys, "diff_keys": diff_config_keys } content["results"] = { "exps": exp_names, "results": combi_results, "keys": result_keys } content["logs"] = {"logs_dict": logs_dict} return render_template('experiment.html', **content)