def main(): np.set_printoptions(suppress=True) parser = argparse.ArgumentParser( description="Domain generalization testbed") parser.add_argument("--input_dir", type=str, default="") parser.add_argument("--latex", action="store_true") args = parser.parse_args() results_file = "results.tex" if args.latex else "results.txt" sys.stdout = misc.Tee(os.path.join(args.input_dir, results_file), "w") records = reporting.load_records(args.input_dir) if args.latex: print("\\documentclass{article}") print("\\usepackage{booktabs}") print("\\usepackage{adjustbox}") print("\\begin{document}") print("\\section{Full DomainBed results}") print("% Total records:", len(records)) else: print("Total records:", len(records)) SELECTION_METHODS = [ model_selection.IIDAccuracySelectionMethod, model_selection.LeaveOneOutSelectionMethod, model_selection.OracleSelectionMethod, ] for selection_method in SELECTION_METHODS: if args.latex: print() print("\\subsection{{Model selection: {}}}".format( selection_method.name)) print_results_tables(records, selection_method, args.latex) if args.latex: print("\\end{document}")
help='Checkpoint every N steps. Default is dataset-dependent.') parser.add_argument('--test_envs', type=int, nargs='+', default=[0]) parser.add_argument('--output_dir', type=str, default="train_output") parser.add_argument('--holdout_fraction', type=float, default=0.2) parser.add_argument('--uda_holdout_fraction', type=float, default=0) parser.add_argument('--skip_model_save', action='store_true') parser.add_argument('--save_model_every_checkpoint', action='store_true') args = parser.parse_args() # If we ever want to implement checkpointing, just persist these values # every once in a while, and then load them from disk here. start_step = 0 algorithm_dict = None os.makedirs(args.output_dir, exist_ok=True) sys.stdout = misc.Tee(os.path.join(args.output_dir, 'out.txt')) sys.stderr = misc.Tee(os.path.join(args.output_dir, 'err.txt')) print("Environment:") print("\tPython: {}".format(sys.version.split(" ")[0])) print("\tPyTorch: {}".format(torch.__version__)) print("\tTorchvision: {}".format(torchvision.__version__)) print("\tCUDA: {}".format(torch.version.cuda)) print("\tCUDNN: {}".format(torch.backends.cudnn.version())) print("\tNumPy: {}".format(np.__version__)) print("\tPIL: {}".format(PIL.__version__)) print('Args:') for k, v in sorted(vars(args).items()): print('\t{}: {}'.format(k, v))
colwidth=25, latex=latex) if __name__ == "__main__": np.set_printoptions(suppress=True) parser = argparse.ArgumentParser( description="Domain generalization testbed") parser.add_argument("--input_dir", type=str, required=True) parser.add_argument("--latex", action="store_true") args = parser.parse_args() results_file = "results.tex" if args.latex else "results.txt" sys.stdout = misc.Tee(os.path.join(args.input_dir, results_file), "w") records = reporting.load_records(args.input_dir) if args.latex: print("\\documentclass{article}") print("\\usepackage{booktabs}") print("\\usepackage{adjustbox}") print("\\begin{document}") print("\\section{Full DomainBed results}") print("% Total records:", len(records)) else: print("Total records:", len(records)) SELECTION_METHODS = [ model_selection.IIDAccuracySelectionMethod,
'--checkpoint_freq', type=int, default=None, help='Checkpoint every N steps. Default is dataset-dependent.') parser.add_argument('--test_envs', type=int, nargs='+', default=[0]) parser.add_argument('--output_dir', type=str, default="train_output") parser.add_argument('--holdout_fraction', type=float, default=0.2) args = parser.parse_args() # If we ever want to implement checkpointing, just persist these values # every once in a while, and then load them from disk here. start_step = 0 algorithm_dict = None os.makedirs(args.output_dir, exist_ok=True) sys.stdout = misc.Tee(os.path.join(args.output_dir, 'log.txt')) print('Args:') for k, v in sorted(vars(args).items()): print('\t{}: {}'.format(k, v)) if args.hparams_seed == 0: hparams = hparams_registry.default_hparams(args.algorithm, args.dataset) else: hparams = hparams_registry.random_hparams( args.algorithm, args.dataset, misc.seed_hash(args.hparams_seed, args.trial_seed)) if args.hparams: hparams.update(json.loads(args.hparams))
) parser.add_argument("--test_envs", type=int, nargs="+", default=[0]) parser.add_argument("--output_dir", type=str, default="train_output") parser.add_argument("--holdout_fraction", type=float, default=0.2) parser.add_argument("--uda_holdout_fraction", type=float, default=0) parser.add_argument("--skip_model_save", action="store_true") parser.add_argument("--save_model_every_checkpoint", action="store_true") args = parser.parse_args() # If we ever want to implement checkpointing, just persist these values # every once in a while, and then load them from disk here. start_step = 0 algorithm_dict = None os.makedirs(args.output_dir, exist_ok=True) sys.stdout = misc.Tee(os.path.join(args.output_dir, "out.txt")) sys.stderr = misc.Tee(os.path.join(args.output_dir, "err.txt")) print("Environment:") print("\tPython: {}".format(sys.version.split(" ")[0])) print("\tPyTorch: {}".format(torch.__version__)) print("\tTorchvision: {}".format(torchvision.__version__)) print("\tCUDA: {}".format(torch.version.cuda)) print("\tCUDNN: {}".format(torch.backends.cudnn.version())) print("\tNumPy: {}".format(np.__version__)) print("\tPIL: {}".format(PIL.__version__)) print("Args:") for k, v in sorted(vars(args).items()): print("\t{}: {}".format(k, v))
except IOError: pass return Q(records) if __name__ == "__main__": np.set_printoptions(suppress=True) parser = argparse.ArgumentParser( description='Domain generalization testbed') parser.add_argument('--input_dir', type=str, default="") parser.add_argument('--latex', action='store_true') args = parser.parse_args() sys.stdout = misc.Tee(os.path.join(args.input_dir, 'results.txt'), "w") records = load_records(args.input_dir) if args.latex: print("\\documentclass{article}") print("\\usepackage{booktabs}") print("\\begin{document}") print('% Total records:', len(records)) else: print('Total records:', len(records)) SELECTION_METHODS = [ model_selection.IIDAccuracySelectionMethod, model_selection.LeaveOneOutSelectionMethod, model_selection.OracleSelectionMethod,
col_labels = ["Algorithm", *dataset_names, "Avg"] header_text = f"Averages, model selection method: {selection_method.name}" print_table(table, header_text, alg_names, col_labels, colwidth=25, latex=latex) if __name__ == "__main__": np.set_printoptions(suppress=True) parser = argparse.ArgumentParser( description="Domain generalization testbed") parser.add_argument("--input_dir", required=True) parser.add_argument('--dataset', required=True) parser.add_argument('--algorithm', required=True) parser.add_argument('--test_env', type=int, required=True) args = parser.parse_args() sys.stdout = misc.Tee(os.path.join(args.input_dir, 'hparams_out.txt'), "w") sys.stderr = misc.Tee(os.path.join(args.input_dir, 'hparams_err.txt'), "w") records = reporting.load_records(args.input_dir) print("Total records:", len(records)) records = reporting.get_grouped_records(records) records = records.filter( lambda r: r['dataset'] == args.dataset and r['algorithm'] == args.algorithm and r['test_env'] == args.test_env ) SELECTION_METHODS = [ model_selection.IIDAccuracySelectionMethod,