def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False): """ Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval :param model_dir: The folder to retrieve the model :param eval_data_all: The switch to turn on if you want to put all data in evaluation data :return: None """ # Retrieve the flag object print("Retrieving flag object for parameters") if (model_dir.startswith("models")): model_dir = model_dir[7:] print("after removing prefix models/, now model_dir is:", model_dir) flags = helper_functions.load_flags(os.path.join("models", model_dir)) flags.eval_model = model_dir # Reset the eval mode # Set up the test_ratio if flags.data_set == 'ballistics': flags.test_ratio = 0.1 elif flags.data_set == 'sine_wave': flags.test_ratio = 0.1 elif flags.data_set == 'robotic_arm': flags.test_ratio = 0.1 # Get the data train_loader, test_loader = data_reader.read_data(flags, eval_data_all=eval_data_all) print("Making network now") # Make Network ntwk = Network(INN, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model) print(ntwk.ckpt_dir) print("number of trainable parameters is :") pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters() if p.requires_grad) print(pytorch_total_params) # Evaluation process print("Start eval now:") if multi_flag: ntwk.evaluate_multiple_time() else: pred_file, truth_file = ntwk.evaluate() # Plot the MSE distribution if flags.data_set != 'meta_material' and not multi_flag: plotMSELossDistrib(pred_file, truth_file, flags) print("Evaluation finished") # If gaussian, plot the scatter plot if flags.data_set == 'gaussian_mixture': Xpred = helper_functions.get_Xpred(path='data/', name=flags.eval_model) Ypred = helper_functions.get_Ypred(path='data/', name=flags.eval_model) # Plot the points scatter generate_Gaussian.plotData(Xpred, Ypred, save_dir='data/' + flags.eval_model.replace('/','_') + 'generation plot.png', eval_mode=True)
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False): """ Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval :param model_dir: The folder to retrieve the model :param eval_data_all: The switch to turn on if you want to put all data in evaluation data :return: None """ # Retrieve the flag object print("Retrieving flag object for parameters") if (model_dir.startswith("models")): model_dir = model_dir[7:] print("after removing prefix models/, now model_dir is:", model_dir) if model_dir.startswith('/'): # It is a absolute path flags = helper_functions.load_flags(model_dir) else: flags = helper_functions.load_flags(os.path.join("models", model_dir)) flags.eval_model = model_dir # Reset the eval mode flags.test_ratio = get_test_ratio_helper(flags) # 2020.10.10 only, delete afterward flags.test_ratio *= 2 # Get the data train_loader, test_loader = data_reader.read_data( flags, eval_data_all=eval_data_all) print("Making network now") # Make Network ntwk = Network(MDN, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model) print(model_dir) print("number of trainable parameters is :") pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters() if p.requires_grad) print(pytorch_total_params) # Evaluation process print("Start eval now:") if multi_flag: ntwk.evaluate_multiple_time() else: pred_file, truth_file = ntwk.evaluate() # Plot the MSE distribution if flags.data_set != 'meta_material' and not multi_flag: plotMSELossDistrib(pred_file, truth_file, flags) print("Evaluation finished")
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False, test_ratio=None): """ Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval :param model_dir: The folder to retrieve the model :param eval_data_all: The switch to turn on if you want to put all data in evaluation data :return: None """ # Retrieve the flag object print("Retrieving flag object for parameters") if (model_dir.startswith("models")): model_dir = model_dir[7:] print("after removing prefix models/, now model_dir is:", model_dir) flags = load_flags(os.path.join("models", model_dir)) flags.eval_model = model_dir # Reset the eval mode if test_ratio is None: flags.test_ratio = get_test_ratio_helper(flags) else: # To make the test ratio swipe with respect to inference time # also making the batch size large enough flags.test_ratio = test_ratio flags.batch_size = 2000 # Get the data train_loader, test_loader = data_reader.read_data(flags, eval_data_all=eval_data_all) print("Making network now") # Make Network ntwk = Network(Forward, Backward, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model) print("number of trainable parameters is :") pytorch_total_params = sum(p.numel() for p in ntwk.model_f.parameters() if p.requires_grad) +\ sum(p.numel() for p in ntwk.model_b.parameters() if p.requires_grad) print(pytorch_total_params) # Evaluation process print("Start eval now:") if multi_flag: ntwk.evaluate_multiple_time() else: pred_file, truth_file = ntwk.evaluate() # Plot the MSE distribution if flags.data_set != 'meta_material' and not multi_flag: plotMSELossDistrib(pred_file, truth_file, flags) print("Evaluation finished")
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False, modulized_flag=False): """ Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval :param model_dir: The folder to retrieve the model :param eval_data_all: The switch to turn on if you want to put all data in evaluation data :return: None """ # Retrieve the flag object print("Retrieving flag object for parameters") if (model_dir.startswith("models")): model_dir = model_dir[7:] print("after removing prefix models/, now model_dir is:", model_dir) flags = helper_functions.load_flags(os.path.join("models", model_dir)) flags.eval_model = model_dir # Reset the eval mode flags.test_ratio = get_test_ratio_helper(flags) # Get the data train_loader, test_loader = data_reader.read_data( flags, eval_data_all=eval_data_all) print("Making network now") # Make Network ntwk = Network(INN, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model) print(ntwk.ckpt_dir) print("number of trainable parameters is :") pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters() if p.requires_grad) print(pytorch_total_params) # Evaluation process print("Start eval now:") if modulized_flag: ntwk.evaluate_modulized_multi_time() elif multi_flag: ntwk.evaluate_multiple_time() else: pred_file, truth_file = ntwk.evaluate() # Plot the MSE distribution if flags.data_set != 'Yang_sim' and not multi_flag and not modulized_flag: # meta-material does not have simulator, hence no Ypred given MSE = plotMSELossDistrib(pred_file, truth_file, flags) # Add this MSE back to the folder flags.best_validation_loss = MSE helper_functions.save_flags(flags, os.path.join("models", model_dir)) elif flags.data_set == 'Yang_sim' and not multi_flag and not modulized_flag: # Save the current path for getting back in the future cwd = os.getcwd() abs_path_Xpred = os.path.abspath(pred_file.replace('Ypred', 'Xpred')) # Change to NA dictory to do prediction os.chdir('../NA/') MSE = predict.ensemble_predict_master('../Data/Yang_sim/state_dicts/', abs_path_Xpred, no_plot=False) # Add this MSE back to the folder flags.best_validation_loss = MSE os.chdir(cwd) helper_functions.save_flags(flags, os.path.join("models", model_dir)) print("Evaluation finished")