def make_args_list(n_trials, dataset_names, algorithms, n_hparams_from,
n_hparams, steps, data_dir, task, holdout_fraction,
single_test_envs, hparams):
args_list = []
for trial_seed in range(n_trials):
for dataset in dataset_names:
for algorithm in algorithms:
if single_test_envs:
all_test_envs = [
[i] for i in range(datasets.num_environments(dataset))
]
else:
all_test_envs = all_test_env_combinations(
datasets.num_environments(dataset))
for test_envs in all_test_envs:
for hparams_seed in range(n_hparams_from, n_hparams):
train_args = {}
train_args['dataset'] = dataset
train_args['algorithm'] = algorithm
train_args['test_envs'] = test_envs
train_args['holdout_fraction'] = holdout_fraction
train_args['hparams_seed'] = hparams_seed
train_args['data_dir'] = data_dir
train_args['task'] = task
train_args['trial_seed'] = trial_seed
train_args['seed'] = misc.seed_hash(
dataset, algorithm, test_envs, hparams_seed,
trial_seed)
if steps is not None:
train_args['steps'] = steps
if hparams is not None:
train_args['hparams'] = hparams
args_list.append(train_args)
return args_list
def test_dataset_erm(self, dataset_name):
"""
Test that ERM can complete one step on a given dataset without raising
an error.
Also test that num_environments() works correctly.
"""
batch_size = 8
hparams = hparams_registry.default_hparams('ERM', dataset_name)
dataset = datasets.get_dataset_class(dataset_name)(
os.environ['DATA_DIR'], [], hparams)
self.assertEqual(datasets.num_environments(dataset_name), len(dataset))
algorithm = algorithms.get_algorithm_class('ERM')(dataset.input_shape,
dataset.num_classes,
len(dataset),
hparams).cuda()
minibatches = helpers.make_minibatches(dataset, batch_size)
algorithm.update(minibatches)
def make_args_list(n_trials, dataset_names, algorithms, n_hparams_from,
n_hparams, steps, data_dir, task, holdout_fraction, hparams,
best_hparams_by_method):
with open('domainbed/best_hparams.json', 'r') as f:
best_hparams = json.load(f)
args_list = []
for trial_seed in range(n_trials):
for dataset in dataset_names:
for algorithm in algorithms:
all_test_envs = [
i for i in range(datasets.num_environments(dataset))
]
for test_env in all_test_envs:
if best_hparams_by_method is not None:
key = f"{dataset},{algorithm},{test_env},{best_hparams_by_method}"
hparams = json.dumps(best_hparams[key]['hparams'])
n_hparams_from = 0
n_hparams = 1
for hparams_seed in range(n_hparams_from, n_hparams):
train_args = {}
train_args['dataset'] = dataset
train_args['algorithm'] = algorithm
train_args['test_envs'] = [test_env]
train_args['holdout_fraction'] = holdout_fraction
train_args['hparams_seed'] = hparams_seed
train_args['data_dir'] = data_dir
train_args['task'] = task
train_args['trial_seed'] = trial_seed
if steps is not None:
train_args['steps'] = steps
if hparams is not None:
train_args['hparams'] = hparams
args_list.append(train_args)
return args_list
def todo_rename(records, selection_method, latex):
grouped_records = reporting.get_grouped_records(records).map(lambda group:
{ **group, "sweep_acc": selection_method.sweep_acc(group["records"]) }
).filter(lambda g: g["sweep_acc"] is not None)
# read algorithm names and sort (predefined order)
alg_names = Q(records).select("args.algorithm").unique()
alg_names = ([n for n in algorithms.ALGORITHMS if n in alg_names] +
[n for n in alg_names if n not in algorithms.ALGORITHMS])
# read dataset names and sort (lexicographic order)
dataset_names = Q(records).select("args.dataset").unique().sorted()
dataset_names = [d for d in datasets.DATASETS if d in dataset_names]
for dataset in dataset_names:
if latex:
print()
print("\\subsubsection{{{}}}".format(dataset))
test_envs = range(datasets.num_environments(dataset))
table = [[None for _ in [*test_envs, "Avg"]] for _ in alg_names]
for i, algorithm in enumerate(alg_names):
means = []
for j, test_env in enumerate(test_envs):
trial_accs = (grouped_records
.filter_equals(
"dataset, algorithm, test_env",
(dataset, algorithm, test_env)
).select("sweep_acc"))
mean, err, table[i][j] = format_mean(trial_accs, latex)
means.append(mean)
if None in means:
table[i][-1] = "X"
else:
table[i][-1] = "{:.1f}".format(sum(means) / len(means))
col_labels = [
"Algorithm",
*datasets.get_dataset_class(dataset).ENVIRONMENTS,
"Avg"
]
header_text = (f"Dataset: {dataset}, "
f"model selection method: {selection_method.name}")
print_table(table, header_text, alg_names, list(col_labels),
colwidth=20, latex=latex)
# Print an "averages" table
if latex:
print()
print("\\subsubsection{Averages}")
table = [[None for _ in [*dataset_names, "Avg"]] for _ in alg_names]
for i, algorithm in enumerate(alg_names):
means = []
for j, dataset in enumerate(dataset_names):
trial_averages = (grouped_records
.filter_equals("algorithm, dataset", (algorithm, dataset))
.group("trial_seed")
.map(lambda trial_seed, group:
group.select("sweep_acc").mean()
)
)
mean, err, table[i][j] = format_mean(trial_averages, latex)
means.append(mean)
if None in means:
table[i][-1] = "X"
else:
table[i][-1] = "{:.1f}".format(sum(means) / len(means))
col_labels = ["Algorithm", *dataset_names, "Avg"]
header_text = f"Averages, model selection method: {selection_method.name}"
print_table(table, header_text, alg_names, col_labels, colwidth=25,
latex=latex)
os.makedirs(args.output_dir, exist_ok=True)
datasets_to_save = [
"OfficeHome",
"TerraIncognita",
"DomainNet",
"RotatedMNIST",
"ColoredMNIST",
"SVIRO",
]
for dataset_name in tqdm(datasets_to_save):
hparams = hparams_registry.default_hparams("ERM", dataset_name)
dataset = datasets.get_dataset_class(dataset_name)(
args.data_dir,
list(range(datasets.num_environments(dataset_name))), hparams)
for env_idx, env in enumerate(tqdm(dataset)):
for i in tqdm(range(50)):
idx = random.choice(list(range(len(env))))
x, y = env[idx]
while y > 10:
idx = random.choice(list(range(len(env))))
x, y = env[idx]
if x.shape[0] == 2:
x = torch.cat([x, torch.zeros_like(x)], dim=0)[:3, :, :]
if x.min() < 0:
mean = torch.tensor([0.485, 0.456, 0.406])[:, None, None]
std = torch.tensor([0.229, 0.224, 0.225])[:, None, None]
x = (x * std) + mean
assert x.min() >= 0
assert x.max() <= 1
def print_results_tables(records, selection_method, latex):
"""Given all records, print a results table for each dataset."""
grouped_records = reporting.get_grouped_records(records).map(lambda group:
{ **group, "sweep_accs": selection_method.sweep_accs(group["records"]) }
)
# read algorithm names and sort (predefined order)
alg_names = Q(records).select("args.algorithm").unique()
alg_names = ([n for n in algorithms.ALGORITHMS if n in alg_names] +
[n for n in alg_names if n not in algorithms.ALGORITHMS])
# read dataset names and sort (lexicographic order)
dataset_names = Q(records).select("args.dataset").unique().sorted()
dataset_names = [d for d in datasets.DATASETS if d in dataset_names]
for dataset in dataset_names:
if latex:
print()
print("\\subsubsection{{{}}}".format(dataset))
test_envs = range(datasets.num_environments(dataset))
table = [[None for _ in [*test_envs, "Avg"]] for _ in alg_names]
for i, algorithm in enumerate(alg_names):
means = []
stdevs = []
for j, test_env in enumerate(test_envs):
try:
acc = grouped_records.filter_equals(
"dataset, algorithm, test_env",
(dataset, algorithm, test_env)
)[0]['sweep_accs'][0]
mean = acc['test_acc']
stdev = acc['test_acc_std']
except:
mean = float('nan')
stdev = float('nan')
means.append(mean)
stdevs.append(stdev)
_, _, table[i][j] = format_mean(mean, stdev, latex)
avg_mean = np.mean(means)
avg_stdev = np.sqrt(np.sum(np.array(stdevs)**2)) / len(stdevs)
_, _, table[i][-1] = format_mean(avg_mean, avg_stdev, latex)
col_labels = [
"Algorithm",
*datasets.get_dataset_class(dataset).ENVIRONMENTS,
"Avg"
]
header_text = (f"Dataset: {dataset}, "
f"model selection method: {selection_method.name}")
print_table(table, header_text, alg_names, list(col_labels),
colwidth=20, latex=latex)
# Print an "averages" table
if latex:
print()
print("\\subsubsection{Averages}")
table = [[None for _ in [*dataset_names, "Avg"]] for _ in alg_names]
for i, algorithm in enumerate(alg_names):
means = []
for j, dataset in enumerate(dataset_names):
try:
mean = (grouped_records
.filter_equals("algorithm, dataset", (algorithm, dataset))
.select(lambda x: x['sweep_accs'][0]['test_acc'])
.mean()
)
except:
mean = float('nan')
mean *= 100.
table[i][j] = "{:.1f}".format(mean)
means.append(mean)
table[i][-1] = "{:.1f}".format(sum(means) / len(means))
col_labels = ["Algorithm", *dataset_names, "Avg"]
header_text = f"Averages, model selection method: {selection_method.name}"
print_table(table, header_text, alg_names, col_labels, colwidth=25,
latex=latex)
