def make_args_list(n_trials, dataset_names, algorithms, n_hparams_from, n_hparams, steps, data_dir, task, holdout_fraction, single_test_envs, hparams): args_list = [] for trial_seed in range(n_trials): for dataset in dataset_names: for algorithm in algorithms: if single_test_envs: all_test_envs = [ [i] for i in range(datasets.num_environments(dataset)) ] else: all_test_envs = all_test_env_combinations( datasets.num_environments(dataset)) for test_envs in all_test_envs: for hparams_seed in range(n_hparams_from, n_hparams): train_args = {} train_args['dataset'] = dataset train_args['algorithm'] = algorithm train_args['test_envs'] = test_envs train_args['holdout_fraction'] = holdout_fraction train_args['hparams_seed'] = hparams_seed train_args['data_dir'] = data_dir train_args['task'] = task train_args['trial_seed'] = trial_seed train_args['seed'] = misc.seed_hash( dataset, algorithm, test_envs, hparams_seed, trial_seed) if steps is not None: train_args['steps'] = steps if hparams is not None: train_args['hparams'] = hparams args_list.append(train_args) return args_list
def test_dataset_erm(self, dataset_name): """ Test that ERM can complete one step on a given dataset without raising an error. Also test that num_environments() works correctly. """ batch_size = 8 hparams = hparams_registry.default_hparams('ERM', dataset_name) dataset = datasets.get_dataset_class(dataset_name)( os.environ['DATA_DIR'], [], hparams) self.assertEqual(datasets.num_environments(dataset_name), len(dataset)) algorithm = algorithms.get_algorithm_class('ERM')(dataset.input_shape, dataset.num_classes, len(dataset), hparams).cuda() minibatches = helpers.make_minibatches(dataset, batch_size) algorithm.update(minibatches)
def make_args_list(n_trials, dataset_names, algorithms, n_hparams_from, n_hparams, steps, data_dir, task, holdout_fraction, hparams, best_hparams_by_method): with open('domainbed/best_hparams.json', 'r') as f: best_hparams = json.load(f) args_list = [] for trial_seed in range(n_trials): for dataset in dataset_names: for algorithm in algorithms: all_test_envs = [ i for i in range(datasets.num_environments(dataset)) ] for test_env in all_test_envs: if best_hparams_by_method is not None: key = f"{dataset},{algorithm},{test_env},{best_hparams_by_method}" hparams = json.dumps(best_hparams[key]['hparams']) n_hparams_from = 0 n_hparams = 1 for hparams_seed in range(n_hparams_from, n_hparams): train_args = {} train_args['dataset'] = dataset train_args['algorithm'] = algorithm train_args['test_envs'] = [test_env] train_args['holdout_fraction'] = holdout_fraction train_args['hparams_seed'] = hparams_seed train_args['data_dir'] = data_dir train_args['task'] = task train_args['trial_seed'] = trial_seed if steps is not None: train_args['steps'] = steps if hparams is not None: train_args['hparams'] = hparams args_list.append(train_args) return args_list
def todo_rename(records, selection_method, latex): grouped_records = reporting.get_grouped_records(records).map(lambda group: { **group, "sweep_acc": selection_method.sweep_acc(group["records"]) } ).filter(lambda g: g["sweep_acc"] is not None) # read algorithm names and sort (predefined order) alg_names = Q(records).select("args.algorithm").unique() alg_names = ([n for n in algorithms.ALGORITHMS if n in alg_names] + [n for n in alg_names if n not in algorithms.ALGORITHMS]) # read dataset names and sort (lexicographic order) dataset_names = Q(records).select("args.dataset").unique().sorted() dataset_names = [d for d in datasets.DATASETS if d in dataset_names] for dataset in dataset_names: if latex: print() print("\\subsubsection{{{}}}".format(dataset)) test_envs = range(datasets.num_environments(dataset)) table = [[None for _ in [*test_envs, "Avg"]] for _ in alg_names] for i, algorithm in enumerate(alg_names): means = [] for j, test_env in enumerate(test_envs): trial_accs = (grouped_records .filter_equals( "dataset, algorithm, test_env", (dataset, algorithm, test_env) ).select("sweep_acc")) mean, err, table[i][j] = format_mean(trial_accs, latex) means.append(mean) if None in means: table[i][-1] = "X" else: table[i][-1] = "{:.1f}".format(sum(means) / len(means)) col_labels = [ "Algorithm", *datasets.get_dataset_class(dataset).ENVIRONMENTS, "Avg" ] header_text = (f"Dataset: {dataset}, " f"model selection method: {selection_method.name}") print_table(table, header_text, alg_names, list(col_labels), colwidth=20, latex=latex) # Print an "averages" table if latex: print() print("\\subsubsection{Averages}") table = [[None for _ in [*dataset_names, "Avg"]] for _ in alg_names] for i, algorithm in enumerate(alg_names): means = [] for j, dataset in enumerate(dataset_names): trial_averages = (grouped_records .filter_equals("algorithm, dataset", (algorithm, dataset)) .group("trial_seed") .map(lambda trial_seed, group: group.select("sweep_acc").mean() ) ) mean, err, table[i][j] = format_mean(trial_averages, latex) means.append(mean) if None in means: table[i][-1] = "X" else: table[i][-1] = "{:.1f}".format(sum(means) / len(means)) col_labels = ["Algorithm", *dataset_names, "Avg"] header_text = f"Averages, model selection method: {selection_method.name}" print_table(table, header_text, alg_names, col_labels, colwidth=25, latex=latex)
os.makedirs(args.output_dir, exist_ok=True) datasets_to_save = [ "OfficeHome", "TerraIncognita", "DomainNet", "RotatedMNIST", "ColoredMNIST", "SVIRO", ] for dataset_name in tqdm(datasets_to_save): hparams = hparams_registry.default_hparams("ERM", dataset_name) dataset = datasets.get_dataset_class(dataset_name)( args.data_dir, list(range(datasets.num_environments(dataset_name))), hparams) for env_idx, env in enumerate(tqdm(dataset)): for i in tqdm(range(50)): idx = random.choice(list(range(len(env)))) x, y = env[idx] while y > 10: idx = random.choice(list(range(len(env)))) x, y = env[idx] if x.shape[0] == 2: x = torch.cat([x, torch.zeros_like(x)], dim=0)[:3, :, :] if x.min() < 0: mean = torch.tensor([0.485, 0.456, 0.406])[:, None, None] std = torch.tensor([0.229, 0.224, 0.225])[:, None, None] x = (x * std) + mean assert x.min() >= 0 assert x.max() <= 1
def print_results_tables(records, selection_method, latex): """Given all records, print a results table for each dataset.""" grouped_records = reporting.get_grouped_records(records).map(lambda group: { **group, "sweep_accs": selection_method.sweep_accs(group["records"]) } ) # read algorithm names and sort (predefined order) alg_names = Q(records).select("args.algorithm").unique() alg_names = ([n for n in algorithms.ALGORITHMS if n in alg_names] + [n for n in alg_names if n not in algorithms.ALGORITHMS]) # read dataset names and sort (lexicographic order) dataset_names = Q(records).select("args.dataset").unique().sorted() dataset_names = [d for d in datasets.DATASETS if d in dataset_names] for dataset in dataset_names: if latex: print() print("\\subsubsection{{{}}}".format(dataset)) test_envs = range(datasets.num_environments(dataset)) table = [[None for _ in [*test_envs, "Avg"]] for _ in alg_names] for i, algorithm in enumerate(alg_names): means = [] stdevs = [] for j, test_env in enumerate(test_envs): try: acc = grouped_records.filter_equals( "dataset, algorithm, test_env", (dataset, algorithm, test_env) )[0]['sweep_accs'][0] mean = acc['test_acc'] stdev = acc['test_acc_std'] except: mean = float('nan') stdev = float('nan') means.append(mean) stdevs.append(stdev) _, _, table[i][j] = format_mean(mean, stdev, latex) avg_mean = np.mean(means) avg_stdev = np.sqrt(np.sum(np.array(stdevs)**2)) / len(stdevs) _, _, table[i][-1] = format_mean(avg_mean, avg_stdev, latex) col_labels = [ "Algorithm", *datasets.get_dataset_class(dataset).ENVIRONMENTS, "Avg" ] header_text = (f"Dataset: {dataset}, " f"model selection method: {selection_method.name}") print_table(table, header_text, alg_names, list(col_labels), colwidth=20, latex=latex) # Print an "averages" table if latex: print() print("\\subsubsection{Averages}") table = [[None for _ in [*dataset_names, "Avg"]] for _ in alg_names] for i, algorithm in enumerate(alg_names): means = [] for j, dataset in enumerate(dataset_names): try: mean = (grouped_records .filter_equals("algorithm, dataset", (algorithm, dataset)) .select(lambda x: x['sweep_accs'][0]['test_acc']) .mean() ) except: mean = float('nan') mean *= 100. table[i][j] = "{:.1f}".format(mean) means.append(mean) table[i][-1] = "{:.1f}".format(sum(means) / len(means)) col_labels = ["Algorithm", *dataset_names, "Avg"] header_text = f"Averages, model selection method: {selection_method.name}" print_table(table, header_text, alg_names, col_labels, colwidth=25, latex=latex)