def save(self, path): savedir = smartutils.create_folder(pjoin(path, type(self).__name__)) smartutils.save_dict_to_json_file(pjoin(savedir, "hyperparams.json"), self.hyperparameters) params = {param.name: param.get_value() for param in self.parameters} assert len(self.parameters) == len(params) # Implies names are all unique. np.savez(pjoin(savedir, "params.npz"), **params)
def save(self, path): savedir = utils.create_folder(pjoin(path, "model")) hyperparameters = {'input_size': self.input_size, 'output_size': self.output_size} utils.save_dict_to_json_file(pjoin(savedir, "meta.json"), {"name": self.__class__.__name__}) utils.save_dict_to_json_file(pjoin(savedir, "hyperparams.json"), hyperparameters) params = {param.name: param.get_value() for param in self.parameters} np.savez(pjoin(savedir, "params.npz"), **params)
def maybe_create_experiment_folder(args, exclude=[], retrocompatibility_defaults={}): # Extract experiments hyperparameters hyperparams = OrderedDict(sorted(vars(args).items())) # Remove hyperparams that should not be part of the hash for name in exclude: if name in hyperparams: del hyperparams[name] # Get/generate experiment name experiment_name = args.name if experiment_name is None: experiment_name = generate_uid_from_string(repr(hyperparams)) # Create experiment folder experiment_path = pjoin(".", "experiments", experiment_name) resuming = False if os.path.isdir(experiment_path) and not args.force: resuming = True print("### Resuming experiment ({0}). ###\n".format(experiment_name)) # Check if provided hyperparams match those in the experiment folder hyperparams_loaded = smartutils.load_dict_from_json_file( pjoin(experiment_path, "hyperparams.json")) for name in exclude: if name in hyperparams_loaded: del hyperparams_loaded[name] for new_hyperparam, default_value in retrocompatibility_defaults.items( ): if new_hyperparam in hyperparams and new_hyperparam not in hyperparams_loaded: hyperparams_loaded[new_hyperparam] = default_value if hyperparams != hyperparams_loaded: print("{\n" + "\n".join([ "{}: {}".format(k, hyperparams[k]) for k in sorted(hyperparams.keys()) ]) + "\n}") print("{\n" + "\n".join([ "{}: {}".format(k, hyperparams_loaded[k]) for k in sorted(hyperparams_loaded.keys()) ]) + "\n}") print( "The arguments provided are different than the one saved. Use --force if you are certain.\nQuitting." ) sys.exit(1) else: if os.path.isdir(experiment_path): shutil.rmtree(experiment_path) os.makedirs(experiment_path) smartutils.save_dict_to_json_file( pjoin(experiment_path, "hyperparams.json"), hyperparams) return experiment_path, hyperparams, resuming
def save(self, path): savedir = utils.create_folder(pjoin(path, "model")) hyperparameters = { 'input_size': self.input_size, 'output_size': self.output_size } utils.save_dict_to_json_file(pjoin(savedir, "meta.json"), {"name": self.__class__.__name__}) utils.save_dict_to_json_file(pjoin(savedir, "hyperparams.json"), hyperparameters) params = {param.name: param.get_value() for param in self.parameters} np.savez(pjoin(savedir, "params.npz"), **params)
def maybe_create_experiment_folder(args, exclude=[], retrocompatibility_defaults={}): # Extract experiments hyperparameters hyperparams = OrderedDict(sorted(vars(args).items())) # Remove hyperparams that should not be part of the hash for name in exclude: if name in hyperparams: del hyperparams[name] # Get/generate experiment name experiment_name = args.name if experiment_name is None: experiment_name = generate_uid_from_string(repr(hyperparams)) # Create experiment folder experiment_path = pjoin(".", "experiments", experiment_name) resuming = False if os.path.isdir(experiment_path) and not args.force: resuming = True print("### Resuming experiment ({0}). ###\n".format(experiment_name)) # Check if provided hyperparams match those in the experiment folder hyperparams_loaded = smartutils.load_dict_from_json_file(pjoin(experiment_path, "hyperparams.json")) for name in exclude: if name in hyperparams_loaded: del hyperparams_loaded[name] for new_hyperparams, default_value in retrocompatibility_defaults.items(): if new_hyperparams not in hyperparams_loaded: hyperparams_loaded[new_hyperparams] = default_value if hyperparams != hyperparams_loaded: print("{\n" + "\n".join(["{}: {}".format(k, hyperparams[k]) for k in sorted(hyperparams.keys())]) + "\n}") print("{\n" + "\n".join(["{}: {}".format(k, hyperparams_loaded[k]) for k in sorted(hyperparams_loaded.keys())]) + "\n}") print("The arguments provided are different than the one saved. Use --force if you are certain.\nQuitting.") sys.exit(1) else: if os.path.isdir(experiment_path): shutil.rmtree(experiment_path) os.makedirs(experiment_path) smartutils.save_dict_to_json_file(pjoin(experiment_path, "hyperparams.json"), hyperparams) return experiment_path, hyperparams, resuming
def save(self, path): savedir = smartutils.create_folder(pjoin(path, type(self).__name__)) smartutils.save_dict_to_json_file(pjoin(savedir, "hyperparams.json"), self.hyperparameters) params = {param.name: param.get_value() for param in self.parameters} assert len(self.parameters) == len( params) # Implies names are all unique. np.savez(pjoin(savedir, "params.npz"), **params) state = { "version": 1, "_srng_rstate": self.srng.rstate, "_srng_state_updates": [ state_update[0].get_value() for state_update in self.srng.state_updates ] } np.savez(pjoin(savedir, "state.npz"), **state)
def main(): parser = build_argparser() args = parser.parse_args() # Get experiment folder experiment_path = args.name if not os.path.isdir(experiment_path): # If not a directory, it must be the name of the experiment. experiment_path = pjoin(".", "experiments", args.name) if not os.path.isdir(experiment_path): parser.error('Cannot find experiment: {0}!'.format(args.name)) if not os.path.isdir(pjoin(experiment_path, "evaluation")): parser.error('Cannot find evaluations for experiment: {0}!'.format( experiment_path)) results_file = pjoin(experiment_path, "results.json") if not os.path.isfile(results_file) or args.force: with Timer("Merging NLL evaluations"): results = merge_NLL_evaluations( evaluation_dir=pjoin(experiment_path, "evaluation")) smartutils.save_dict_to_json_file(results_file, {"NLL": results}) else: print("Loading saved losses... (use --force to re-run evaluation)") results = smartutils.load_dict_from_json_file(results_file)["NLL"] nb_orderings = results['nb_orderings'] for dataset in ['validset', 'testset']: print("NLL estimate on {} ({} orderings): {:.2f} ± {:.2f}".format( dataset, nb_orderings, results[dataset]['mean'], results[dataset]['stderror'])) if results[dataset]['incomplete']: print( "** Warning **: {} evaluation is incomplete. Missing some orderings or batches." .format(dataset))
def save(self, path): state = {"version": 1, "var": self.var.get_value()} smartutils.save_dict_to_json_file(pjoin(path, type(self).__name__ + ".json"), state)
def main(): parser = build_parser() args = parser.parse_args() print(args) # Get experiment folder experiment_path = args.name if not os.path.isdir(experiment_path): # If not a directory, it must be the name of the experiment. experiment_path = pjoin(".", "experiments", args.name) if not os.path.isdir(experiment_path): parser.error('Cannot find experiment: {0}!'.format(args.name)) # Load experiments hyperparameters try: hyperparams = smartutils.load_dict_from_json_file(pjoin(experiment_path, "hyperparams.json")) except FileNotFoundError: hyperparams = smartutils.load_dict_from_json_file(pjoin(experiment_path, "..", "hyperparams.json")) with Timer("Loading dataset", newline=True): volume_manager = VolumeManager() dataset = datasets.load_tractography_dataset(args.subjects, volume_manager, name="dataset", use_sh_coeffs=hyperparams['use_sh_coeffs']) print("Dataset size:", len(dataset)) with Timer("Loading model"): model = None if hyperparams['model'] == 'gru_regression': from learn2track.models import GRU_Regression model = GRU_Regression.create(experiment_path, volume_manager=volume_manager) elif hyperparams['model'] == 'gru_mixture': from learn2track.models import GRU_Mixture model = GRU_Mixture.create(experiment_path, volume_manager=volume_manager) elif hyperparams['model'] == 'gru_multistep': from learn2track.models import GRU_Multistep_Gaussian model = GRU_Multistep_Gaussian.create(experiment_path, volume_manager=volume_manager) model.k = 1 model.m = 1 elif hyperparams['model'] == 'ffnn_regression': from learn2track.models import FFNN_Regression model = FFNN_Regression.create(experiment_path, volume_manager=volume_manager) else: raise NameError("Unknown model: {}".format(hyperparams['model'])) with Timer("Building evaluation function"): # Override K for gru_multistep if 'k' in hyperparams: hyperparams['k'] = 1 batch_scheduler = batch_scheduler_factory(hyperparams, dataset, train_mode=False, batch_size_override=args.batch_size) loss = loss_factory(hyperparams, model, dataset, loss_type=args.loss_type) l2_error = views.LossView(loss=loss, batch_scheduler=batch_scheduler) with Timer("Evaluating...", newline=True): results_file = pjoin(experiment_path, "results.json") results = {} if os.path.isfile(results_file) and not args.force: print("Loading saved results... (use --force to re-run evaluation)") results = smartutils.load_dict_from_json_file(results_file) tag = "" if args.loss_type == 'expected_value' or hyperparams['model'] == 'gru_regression': tag = "_EV_L2_error" elif args.loss_type == 'maximum_component': tag = "_MC_L2_error" elif hyperparams['model'] == 'gru_mixture' or hyperparams['model'] == 'gru_multistep': tag = "_NLL" entry = args.dataset_name + tag if entry not in results or args.force: with Timer("Evaluating {}".format(entry)): dummy_status = Status() # Forces recomputing results results[entry] = {'mean': float(l2_error.mean.view(dummy_status)), 'stderror': float(l2_error.stderror.view(dummy_status))} smartutils.save_dict_to_json_file(results_file, results) # Update results file. print("{}: {:.4f} ± {:.4f}".format(entry, results[entry]['mean'], results[entry]['stderror']))
def main(): parser = build_parser() args = parser.parse_args() print(args) # Get experiment folder experiment_path = args.name if not os.path.isdir(experiment_path): # If not a directory, it must be the name of the experiment. experiment_path = pjoin(".", "experiments", args.name) if not os.path.isdir(experiment_path): parser.error('Cannot find experiment: {0}!'.format(args.name)) # Load experiments hyperparameters try: hyperparams = smartutils.load_dict_from_json_file( pjoin(experiment_path, "hyperparams.json")) except FileNotFoundError: hyperparams = smartutils.load_dict_from_json_file( pjoin(experiment_path, "..", "hyperparams.json")) # Use this for hyperparams added in a new version, but nonexistent from older versions retrocompatibility_defaults = { 'feed_previous_direction': False, 'predict_offset': False, 'normalize': False, 'keep_step_size': False, 'sort_streamlines': False, 'use_layer_normalization': False, 'drop_prob': 0., 'use_zoneout': False } for new_hyperparams, default_value in retrocompatibility_defaults.items(): if new_hyperparams not in hyperparams: hyperparams[new_hyperparams] = default_value with Timer("Loading dataset", newline=True): volume_manager = VolumeManager() dataset = datasets.load_tractography_dataset( args.subjects, volume_manager, name="dataset", use_sh_coeffs=hyperparams['use_sh_coeffs']) print("Dataset size:", len(dataset)) with Timer("Loading model"): model = None if hyperparams['model'] == 'gru_regression': from learn2track.models import GRU_Regression model = GRU_Regression.create(experiment_path, volume_manager=volume_manager) elif hyperparams['model'] == 'gru_gaussian': from learn2track.models import GRU_Gaussian model = GRU_Gaussian.create(experiment_path, volume_manager=volume_manager) elif hyperparams['model'] == 'gru_mixture': from learn2track.models import GRU_Mixture model = GRU_Mixture.create(experiment_path, volume_manager=volume_manager) elif hyperparams['model'] == 'gru_multistep': from learn2track.models import GRU_Multistep_Gaussian model = GRU_Multistep_Gaussian.create( experiment_path, volume_manager=volume_manager) model.k = 1 model.m = 1 elif hyperparams['model'] == 'ffnn_regression': from learn2track.models import FFNN_Regression model = FFNN_Regression.create(experiment_path, volume_manager=volume_manager) else: raise NameError("Unknown model: {}".format(hyperparams['model'])) model.drop_prob = 0. # Make sure dropout/zoneout is not used when testing with Timer("Building evaluation function"): # Override K for gru_multistep if 'k' in hyperparams: hyperparams['k'] = 1 batch_scheduler = batch_scheduler_factory( hyperparams, dataset, train_mode=False, batch_size_override=args.batch_size) loss = loss_factory(hyperparams, model, dataset, loss_type=args.loss_type) l2_error = views.LossView(loss=loss, batch_scheduler=batch_scheduler) with Timer("Evaluating...", newline=True): results_file = pjoin(experiment_path, "results.json") results = {} if os.path.isfile(results_file) and not args.force: print( "Loading saved results... (use --force to re-run evaluation)") results = smartutils.load_dict_from_json_file(results_file) tag = "" if args.loss_type == 'expected_value' or hyperparams[ 'model'] == 'gru_regression': tag = "_EV_L2_error" elif args.loss_type == 'maximum_component': tag = "_MC_L2_error" elif hyperparams['model'] in [ 'gru_gaussian', 'gru_mixture', 'gru_multistep' ]: tag = "_NLL" entry = args.dataset_name + tag if entry not in results or args.force: with Timer("Evaluating {}".format(entry)): dummy_status = Status() # Forces recomputing results results[entry] = { 'mean': float(l2_error.mean.view(dummy_status)), 'stderror': float(l2_error.stderror.view(dummy_status)) } smartutils.save_dict_to_json_file( results_file, results) # Update results file. print("{}: {:.4f} ± {:.4f}".format(entry, results[entry]['mean'], results[entry]['stderror']))