def main():
np.set_printoptions(suppress=True)
parser = argparse.ArgumentParser(
description="Domain generalization testbed")
parser.add_argument("--input_dir", type=str, default="")
parser.add_argument("--latex", action="store_true")
args = parser.parse_args()
results_file = "results.tex" if args.latex else "results.txt"
sys.stdout = misc.Tee(os.path.join(args.input_dir, results_file), "w")
records = reporting.load_records(args.input_dir)
if args.latex:
print("\\documentclass{article}")
print("\\usepackage{booktabs}")
print("\\usepackage{adjustbox}")
print("\\begin{document}")
print("\\section{Full DomainBed results}")
print("% Total records:", len(records))
else:
print("Total records:", len(records))
SELECTION_METHODS = [
model_selection.IIDAccuracySelectionMethod,
model_selection.LeaveOneOutSelectionMethod,
model_selection.OracleSelectionMethod,
]
for selection_method in SELECTION_METHODS:
if args.latex:
print()
print("\\subsection{{Model selection: {}}}".format(
selection_method.name))
print_results_tables(records, selection_method, args.latex)
if args.latex:
print("\\end{document}")
help='Checkpoint every N steps. Default is dataset-dependent.')
parser.add_argument('--test_envs', type=int, nargs='+', default=[0])
parser.add_argument('--output_dir', type=str, default="train_output")
parser.add_argument('--holdout_fraction', type=float, default=0.2)
parser.add_argument('--uda_holdout_fraction', type=float, default=0)
parser.add_argument('--skip_model_save', action='store_true')
parser.add_argument('--save_model_every_checkpoint', action='store_true')
args = parser.parse_args()
# If we ever want to implement checkpointing, just persist these values
# every once in a while, and then load them from disk here.
start_step = 0
algorithm_dict = None
os.makedirs(args.output_dir, exist_ok=True)
sys.stdout = misc.Tee(os.path.join(args.output_dir, 'out.txt'))
sys.stderr = misc.Tee(os.path.join(args.output_dir, 'err.txt'))
print("Environment:")
print("\tPython: {}".format(sys.version.split(" ")[0]))
print("\tPyTorch: {}".format(torch.__version__))
print("\tTorchvision: {}".format(torchvision.__version__))
print("\tCUDA: {}".format(torch.version.cuda))
print("\tCUDNN: {}".format(torch.backends.cudnn.version()))
print("\tNumPy: {}".format(np.__version__))
print("\tPIL: {}".format(PIL.__version__))
print('Args:')
for k, v in sorted(vars(args).items()):
print('\t{}: {}'.format(k, v))
colwidth=25,
latex=latex)
if __name__ == "__main__":
np.set_printoptions(suppress=True)
parser = argparse.ArgumentParser(
description="Domain generalization testbed")
parser.add_argument("--input_dir", type=str, required=True)
parser.add_argument("--latex", action="store_true")
args = parser.parse_args()
results_file = "results.tex" if args.latex else "results.txt"
sys.stdout = misc.Tee(os.path.join(args.input_dir, results_file), "w")
records = reporting.load_records(args.input_dir)
if args.latex:
print("\\documentclass{article}")
print("\\usepackage{booktabs}")
print("\\usepackage{adjustbox}")
print("\\begin{document}")
print("\\section{Full DomainBed results}")
print("% Total records:", len(records))
else:
print("Total records:", len(records))
SELECTION_METHODS = [
model_selection.IIDAccuracySelectionMethod,
'--checkpoint_freq',
type=int,
default=None,
help='Checkpoint every N steps. Default is dataset-dependent.')
parser.add_argument('--test_envs', type=int, nargs='+', default=[0])
parser.add_argument('--output_dir', type=str, default="train_output")
parser.add_argument('--holdout_fraction', type=float, default=0.2)
args = parser.parse_args()
# If we ever want to implement checkpointing, just persist these values
# every once in a while, and then load them from disk here.
start_step = 0
algorithm_dict = None
os.makedirs(args.output_dir, exist_ok=True)
sys.stdout = misc.Tee(os.path.join(args.output_dir, 'log.txt'))
print('Args:')
for k, v in sorted(vars(args).items()):
print('\t{}: {}'.format(k, v))
if args.hparams_seed == 0:
hparams = hparams_registry.default_hparams(args.algorithm,
args.dataset)
else:
hparams = hparams_registry.random_hparams(
args.algorithm, args.dataset,
misc.seed_hash(args.hparams_seed, args.trial_seed))
if args.hparams:
hparams.update(json.loads(args.hparams))
)
parser.add_argument("--test_envs", type=int, nargs="+", default=[0])
parser.add_argument("--output_dir", type=str, default="train_output")
parser.add_argument("--holdout_fraction", type=float, default=0.2)
parser.add_argument("--uda_holdout_fraction", type=float, default=0)
parser.add_argument("--skip_model_save", action="store_true")
parser.add_argument("--save_model_every_checkpoint", action="store_true")
args = parser.parse_args()
# If we ever want to implement checkpointing, just persist these values
# every once in a while, and then load them from disk here.
start_step = 0
algorithm_dict = None
os.makedirs(args.output_dir, exist_ok=True)
sys.stdout = misc.Tee(os.path.join(args.output_dir, "out.txt"))
sys.stderr = misc.Tee(os.path.join(args.output_dir, "err.txt"))
print("Environment:")
print("\tPython: {}".format(sys.version.split(" ")[0]))
print("\tPyTorch: {}".format(torch.__version__))
print("\tTorchvision: {}".format(torchvision.__version__))
print("\tCUDA: {}".format(torch.version.cuda))
print("\tCUDNN: {}".format(torch.backends.cudnn.version()))
print("\tNumPy: {}".format(np.__version__))
print("\tPIL: {}".format(PIL.__version__))
print("Args:")
for k, v in sorted(vars(args).items()):
print("\t{}: {}".format(k, v))
except IOError:
pass
return Q(records)
if __name__ == "__main__":
np.set_printoptions(suppress=True)
parser = argparse.ArgumentParser(
description='Domain generalization testbed')
parser.add_argument('--input_dir', type=str, default="")
parser.add_argument('--latex', action='store_true')
args = parser.parse_args()
sys.stdout = misc.Tee(os.path.join(args.input_dir, 'results.txt'), "w")
records = load_records(args.input_dir)
if args.latex:
print("\\documentclass{article}")
print("\\usepackage{booktabs}")
print("\\begin{document}")
print('% Total records:', len(records))
else:
print('Total records:', len(records))
SELECTION_METHODS = [
model_selection.IIDAccuracySelectionMethod,
model_selection.LeaveOneOutSelectionMethod,
model_selection.OracleSelectionMethod,
col_labels = ["Algorithm", *dataset_names, "Avg"]
header_text = f"Averages, model selection method: {selection_method.name}"
print_table(table, header_text, alg_names, col_labels, colwidth=25,
latex=latex)
if __name__ == "__main__":
np.set_printoptions(suppress=True)
parser = argparse.ArgumentParser(
description="Domain generalization testbed")
parser.add_argument("--input_dir", required=True)
parser.add_argument('--dataset', required=True)
parser.add_argument('--algorithm', required=True)
parser.add_argument('--test_env', type=int, required=True)
args = parser.parse_args()
sys.stdout = misc.Tee(os.path.join(args.input_dir, 'hparams_out.txt'), "w")
sys.stderr = misc.Tee(os.path.join(args.input_dir, 'hparams_err.txt'), "w")
records = reporting.load_records(args.input_dir)
print("Total records:", len(records))
records = reporting.get_grouped_records(records)
records = records.filter(
lambda r:
r['dataset'] == args.dataset and
r['algorithm'] == args.algorithm and
r['test_env'] == args.test_env
)
SELECTION_METHODS = [
model_selection.IIDAccuracySelectionMethod,
