def ensemble_predict(model_list, Xpred_file, model_dir=None, no_plot=True, remove_extra_files=True):
"""
This predicts the output from an ensemble of models
:param model_list: The list of model names to aggregate
:param Xpred_file: The Xpred_file that you want to predict
:param model_dir: The directory to plot the plot
:param no_plot: If True, do not plot (For multi_eval)
:param remove_extra_files: Remove all the files generated except for the ensemble one
:return: The prediction Ypred_file
"""
print("this is doing ensemble prediction for models :", model_list)
pred_list = []
# Get the predictions into a list of np array
for pre_trained_model in model_list:
pred_file, truth_file, flags = predict_from_model(pre_trained_model, Xpred_file)
#pred = np.loadtxt(pred_file, delimiter=' ')
#if remove_extra_files: # Remove the generated files
# os.remove(pred_file)
pred_list.append(np.copy(np.expand_dims(pred_file, axis=2)))
# Take the mean of the predictions
pred_all = np.concatenate(pred_list, axis=2)
pred_mean = np.mean(pred_all, axis=2)
save_name = Xpred_file.replace('Xpred', 'Ypred')
np.savetxt(save_name, pred_mean)
# If no_plot, then return
if no_plot:
return
# saving the plot down
flags.eval_model = 'ensemble_plot' + Xpred_file.replace('/', '')
if model_dir is None:
plotMSELossDistrib(save_name, truth_file, flags)
else:
plotMSELossDistrib(save_name, truth_file, flags, save_dir=model_dir)
def ensemble_predict(model_list, Xpred_file, plot_mode=True):
"""
This predicts the output from an ensemble of models
:param model_list: The list of model names to aggregate
:param Xpred_file: The Xpred_file that you want to predict
:return: The prediction Ypred_file
"""
print("this is doing ensemble prediction for models :", model_list)
pred_list = []
# Get the predictions into a list of np array
for pre_trained_model in model_list:
pred_file, truth_file, flags = predict_from_model(pre_trained_model,
Xpred_file,
save_mode=False)
#pred = np.loadtxt(pred_file, delimiter=' ')
pred = pred_file
pred_list.append(np.copy(np.expand_dims(pred, axis=2)))
# Take the mean of the predictions
pred_all = np.concatenate(pred_list, axis=2)
pred_mean = np.mean(pred_all, axis=2)
save_name = Xpred_file.replace('Xpred', 'Ypred_ensemble')
np.savetxt(save_name, pred_mean)
# saving the plot down
save_name_ensemble = Xpred_file.split('/')[-1][:-4]
print('save name of emsemble = ', save_name_ensemble)
flags.eval_model = 'ensemble_model' + save_name_ensemble
if plot_mode:
plotMSELossDistrib(save_name, truth_file, flags)
def predict(model_dir, Ytruth_file, multi_flag=False):
"""
Predict the output from given spectra
"""
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
if model_dir.startswith('/'): # It is a absolute path
flags = helper_functions.load_flags(model_dir)
else:
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
ntwk = Network(INN,
flags,
train_loader=None,
test_loader=None,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
pred_file, truth_file = ntwk.predict(Ytruth_file)
if 'Yang' not in flags.data_set:
plotMSELossDistrib(pred_file, truth_file, flags)
def evaluate_from_model(model_dir):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:return: None
"""
# Retrieve the flag object
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
print("Retrieving flag object for parameters")
flags = flag_reader.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
# Get the data
train_loader, test_loader = data_reader.read_data(flags)
print("Making network now")
# Make Network
ntwk = Network(Forward, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model)
# Evaluation process
print("Start eval now:")
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param multi_flag: The switch to turn on if you want to generate all different inference trial results
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
print(model_dir)
flags = load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.backprop_step = eval_flags.backprop_step
if flags.data_set == 'ballistics':
flags.test_ratio = 0.001
elif flags.data_set == 'sine_wave':
flags.test_ratio = 0.005
elif flags.data_set == 'robotic_arm':
flags.test_ratio = 0.2
elif flags.data_set == 'sine_test_1d':
flags.test_ratio = 0.05
flags.batch_size = 1 # For backprop eval mode, batchsize is always 1
flags.lr = 0.05
flags.eval_batch_size = eval_flags.eval_batch_size
flags.train_step = eval_flags.train_step
# Get the data
train_loader, test_loader = data_reader.read_data(
flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(Backprop,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
pred_file, truth_file = ntwk.evaluate(
save_dir='/work/sr365/multi_eval/Backprop/' + flags.data_set,
save_all=True)
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def predict_from_model(pre_trained_model,
Xpred_file,
no_plot=True,
load_state_dict=None):
"""
Predicting interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param Xpred_file: The Prediction file position
:param no_plot: If True, do not plot (For multi_eval)
:param load_state_dict: The new way to load the model for ensemble MM
:return: None
"""
# Retrieve the flag object
print("This is doing the prediction for file", Xpred_file)
print("Retrieving flag object for parameters")
if (pre_trained_model.startswith("models")):
eval_model = pre_trained_model[7:]
print("after removing prefix models/, now model_dir is:", eval_model)
flags = load_flags(pre_trained_model) # Get the pre-trained model
flags.eval_model = pre_trained_model # Reset the eval mode
flags.test_ratio = 0.1 #useless number
# Get the data, this part is useless in prediction but just for simplicity
#train_loader, test_loader = data_reader.read_data(flags)
print("Making network now")
# Make Network
ntwk = Network(NA,
flags,
train_loader=None,
test_loader=None,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if not no_plot:
# Plot the MSE distribution
pred_file, truth_file = ntwk.predict(Xpred_file,
no_save=False,
load_state_dict=load_state_dict)
flags.eval_model = pred_file.replace(
'.', '_') # To make the plot name different
plotMSELossDistrib(pred_file, truth_file, flags)
else:
pred_file, truth_file = ntwk.predict(Xpred_file,
no_save=True,
load_state_dict=load_state_dict)
print("Evaluation finished")
return pred_file, truth_file, flags
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
# Set up the test_ratio
if flags.data_set == 'ballistics':
flags.test_ratio = 0.1
elif flags.data_set == 'sine_wave':
flags.test_ratio = 0.1
elif flags.data_set == 'robotic_arm':
flags.test_ratio = 0.1
# Get the data
train_loader, test_loader = data_reader.read_data(flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(INN, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model)
print(ntwk.ckpt_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters() if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
ntwk.evaluate_multiple_time()
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'meta_material' and not multi_flag:
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
# If gaussian, plot the scatter plot
if flags.data_set == 'gaussian_mixture':
Xpred = helper_functions.get_Xpred(path='data/', name=flags.eval_model)
Ypred = helper_functions.get_Ypred(path='data/', name=flags.eval_model)
# Plot the points scatter
generate_Gaussian.plotData(Xpred, Ypred, save_dir='data/' + flags.eval_model.replace('/','_') + 'generation plot.png', eval_mode=True)
def ensemble_predict(model_list,
Xpred_file,
model_dir=None,
no_plot=True,
remove_extra_files=True,
state_dict=False):
"""
This predicts the output from an ensemble of models
:param model_list: The list of model names to aggregate
:param Xpred_file: The Xpred_file that you want to predict
:param model_dir: The directory to plot the plot
:param no_plot: If True, do not plot (For multi_eval)
:param remove_extra_files: Remove all the files generated except for the ensemble one
:param state_dict: New way to load model using state_dict instead of load module
:return: The prediction Ypred_file
"""
print("this is doing ensemble prediction for models :", model_list)
pred_list = []
# Get the predictions into a list of np array
for pre_trained_model in model_list:
if state_dict is False:
pred_file, truth_file, flags = predict_from_model(
pre_trained_model, Xpred_file)
# This line is to plot all histogram, make sure comment the pred_list.append line below as well for getting all the histograms
#pred_file, truth_file, flags = predict_from_model(pre_trained_model, Xpred_file, no_plot=False)
else:
model_folder = os.path.join('..', 'Data', 'Yang_sim',
'model_param')
pred_file, truth_file, flags = predict_from_model(
model_folder, Xpred_file, load_state_dict=pre_trained_model)
pred_list.append(np.copy(np.expand_dims(pred_file, axis=2)))
# Take the mean of the predictions
pred_all = np.concatenate(pred_list, axis=2)
pred_mean = np.mean(pred_all, axis=2)
save_name = Xpred_file.replace('Xpred', 'Ypred')
np.savetxt(save_name, pred_mean)
# If no_plot, then return
if no_plot:
return
# saving the plot down
flags.eval_model = ''
model_dir = None
if model_dir is None:
return plotMSELossDistrib(save_name, truth_file, flags)
else:
return plotMSELossDistrib(save_name,
truth_file,
flags,
save_dir=model_dir)
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
if model_dir.startswith('/'): # It is a absolute path
flags = helper_functions.load_flags(model_dir)
else:
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.test_ratio = get_test_ratio_helper(flags)
# 2020.10.10 only, delete afterward
flags.test_ratio *= 2
# Get the data
train_loader, test_loader = data_reader.read_data(
flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(MDN,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print(model_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
ntwk.evaluate_multiple_time()
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'meta_material' and not multi_flag:
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False, test_ratio=None):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.batch_size = 1
flags.backprop_step=300
flags.eval_batch_size=2048
if test_ratio is None:
flags.test_ratio = get_test_ratio_helper(flags)
else:
# To make the test ratio swipe with respect to inference time
# also making the batch size large enough
flags.test_ratio = test_ratio
# Get the data
train_loader, test_loader = data_reader.read_data(flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(make_cINN_and_NA, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model)
#print(model_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model_cINN.parameters() if p.requires_grad)
print(pytorch_total_params)
pytorch_total_params = sum(p.numel() for p in ntwk.model_NA.parameters() if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
pred_file, truth_file = ntwk.evaluate(save_dir='/work/sr365/NIPS_multi_eval_backup/multi_eval/hybrid_cINN_NA_0bp/'+flags.data_set, save_all=True)
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'meta_material' and not multi_flag:
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def infer(pre_trained_model, Xpred_file, no_plot=True):
# Retrieve the flag object
print("This is doing the prediction for file", Xpred_file)
print("Retrieving flag object for parameters")
if (pre_trained_model.startswith("models")):
eval_model = pre_trained_model[7:]
print("after removing prefix models/, now model_dir is:", eval_model)
flags = load_flags(pre_trained_model) # Get the pre-trained model
flags.eval_model = pre_trained_model # Reset the eval mode
flags.test_ratio = 0.1 # useless number
# Get the data, this part is useless in prediction but just for simplicity
train_loader, test_loader = data_reader.read_data(flags)
print("Making network now")
# Make Network
ntwk = Network(Backprop,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if not no_plot:
# Plot the MSE distribution
pred_file, truth_file = ntwk.predict(Xpred_file, no_save=False)
flags.eval_model = pred_file.replace(
'.', '_') # To make the plot name different
plotMSELossDistrib(pred_file, truth_file, flags)
else:
pred_file, truth_file = ntwk.predict(Xpred_file, no_save=True)
print("Evaluation finished")
return pred_file, truth_file, flags
def evaluate_from_model(model_dir,
multi_flag=False,
eval_data_all=False,
save_misc=False,
MSE_Simulator=False,
save_Simulator_Ypred=True,
init_lr=0.5,
BDY_strength=1):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param multi_flag: The switch to turn on if you want to generate all different inference trial results
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
print(model_dir)
flags = load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.test_ratio = get_test_ratio_helper(flags)
if flags.data_set == 'Peurifoy':
flags.eval_batch_size = 10000
elif flags.data_set == 'Chen':
flags.eval_batch_size = 10000
elif flags.data_set == 'Yang' or flags.data_set == 'Yang_sim':
flags.eval_batch_size = 2000
flags.batch_size = flags.eval_batch_size
flags.lr = init_lr
flags.BDY_strength = BDY_strength
flags.eval_batch_size = eval_flags.eval_batch_size
flags.train_step = eval_flags.train_step
# delete after usage: 02.07 for vilidating that ball and sine is convex problem
# Use a very small eval batch size and expected to see that meta and robo getting much worse performance
# and the ball and sine getting nearly identical one
# flags.eval_batch_size = 2
print(flags)
flags.batch_size = 500
# Get the data
eval_data_all = True
train_loader, test_loader = data_reader.read_data(
flags, eval_data_all=eval_data_all)
print("LENGTH: ", len(test_loader))
print("Making network now")
# Make Network
ntwk = Network(Forward,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
dest_dir = '/home/sr365/MM_bench_multi_eval/NA/'
#dest_dir = '/data/users/ben/multi_eval/NA_lr' + str(init_lr) + 'bdy_' + str(BDY_strength)+'/'
if not os.path.isdir(dest_dir):
os.mkdir(dest_dir)
dest_dir += flags.data_set
if not os.path.isdir(dest_dir):
os.mkdir(dest_dir)
#pred_file, truth_file = ntwk.evaluate(save_dir='/work/sr365/multi_eval/NA/' + flags.data_set, save_all=True,
pred_file, truth_file = ntwk.evaluate(
save_dir=dest_dir,
save_all=True,
save_misc=save_misc,
MSE_Simulator=MSE_Simulator,
save_Simulator_Ypred=save_Simulator_Ypred)
else:
pred_file, truth_file = ntwk.evaluate(
save_dir='data/' + flags.data_set,
save_misc=save_misc,
MSE_Simulator=MSE_Simulator,
save_Simulator_Ypred=save_Simulator_Ypred)
# Plot the MSE distribution
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def evaluate_from_model(model_dir,
multi_flag=False,
eval_data_all=False,
modulized_flag=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.test_ratio = get_test_ratio_helper(flags)
# Get the data
train_loader, test_loader = data_reader.read_data(
flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(INN,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print(ntwk.ckpt_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if modulized_flag:
ntwk.evaluate_modulized_multi_time()
elif multi_flag:
ntwk.evaluate_multiple_time()
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'Yang_sim' and not multi_flag and not modulized_flag: # meta-material does not have simulator, hence no Ypred given
MSE = plotMSELossDistrib(pred_file, truth_file, flags)
# Add this MSE back to the folder
flags.best_validation_loss = MSE
helper_functions.save_flags(flags, os.path.join("models", model_dir))
elif flags.data_set == 'Yang_sim' and not multi_flag and not modulized_flag:
# Save the current path for getting back in the future
cwd = os.getcwd()
abs_path_Xpred = os.path.abspath(pred_file.replace('Ypred', 'Xpred'))
# Change to NA dictory to do prediction
os.chdir('../NA/')
MSE = predict.ensemble_predict_master('../Data/Yang_sim/state_dicts/',
abs_path_Xpred,
no_plot=False)
# Add this MSE back to the folder
flags.best_validation_loss = MSE
os.chdir(cwd)
helper_functions.save_flags(flags, os.path.join("models", model_dir))
print("Evaluation finished")
from utils.evaluation_helper import plotMSELossDistrib
big_folder = '/home/sr365/MM_Bench/NA/'
#big_folder = '/home/sr365/MM_Bench/NA/'
for folder in os.listdir(big_folder):
# ignore if not a folder or no data in folder name
if 'data' not in folder or not os.path.isdir(
os.path.join(big_folder, folder)) or 'Chen' not in folder:
continue
print('going into folder ', folder)
# Loop over the data folder
for file in os.listdir(os.path.join(big_folder, folder)):
if 'test_Ytruth' not in file:
continue
print('going in file', file)
truth_file = os.path.join(big_folder, folder, file)
pred_file = os.path.join(big_folder, folder,
file.replace('Ytruth', 'Ypred'))
# Make sure Ypred is also present
if not os.path.isfile(pred_file):
print('no Ypred file, abort!')
continue
print('doing MSE plot for file ', file, 'in folder ',
os.path.join(big_folder, folder))
plotMSELossDistrib(pred_file=pred_file,
truth_file=truth_file,
save_dir=os.path.join(big_folder, folder))
def retrain_different_dataset(index):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ['mm1', 'mm2', 'mm3', 'mm4', 'mm5']
for eval_model in data_set_list:
flags = load_flags(os.path.join("prev_models", eval_model))
flags.model_name = "retrain_" + str(index) + flags.model_name
flags.data_dir = '/work/sr365/Christian_data_augmented'
flags.ckpt_dir = '/work/sr365/MM_ensemble'
flags.train_step = 500
flags.test_ratio = 0.2
training_from_flag(flags)
if __name__ == '__main__':
# Read the parameters to be set
from utils import evaluation_helper
flags = flag_reader_ensemble.read_flag()
evaluation_helper.plotMSELossDistrib('datapool/Ypred_ensemble.csv',
'datapool/Ytruth.csv', flags)
# Call the train from flag function
#for i in range(3):
# training_from_flag(flags)
print(type(flags))
# Do the retraining for all the data set to get the training
#for i in range(1):
# retrain_different_dataset(i)
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False, save_misc=False,
MSE_Simulator=False, save_Simulator_Ypred=True,
init_lr=0.01, lr_decay=0.9, BDY_strength=1, save_dir='data/',
noise_level=0,
md_coeff=0, md_start=None, md_end=None, md_radius=None,
eval_batch_size=None):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param multi_flag: The switch to turn on if you want to generate all different inference trial results
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
print(model_dir)
flags = load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.test_ratio = get_test_ratio_helper(flags)
flags.backprop_step = eval_flags.backprop_step
#flags.test_ratio = 0.02
if flags.data_set != None: #== 'Yang_sim':
save_Simulator_Ypred = False
print("this is Yang sim dataset, setting the save_Simulator_Ypred to False")
flags.batch_size = 1 # For backprop eval mode, batchsize is always 1
flags.BDY_strength = BDY_strength
flags.train_step = eval_flags.train_step
flags.backprop_step = 300
# MD Loss: new version
if md_coeff is not None:
flags.md_coeff = md_coeff
if md_start is not None:
flags.md_start = md_start
if md_end is not None:
flags.md_end = md_end
if md_radius is not None:
flags.md_radius = md_radius
############################# Thing that are changing #########################
flags.lr = init_lr
flags.lr_decay_rate = lr_decay
flags.eval_batch_size = 2048 if eval_batch_size is None else eval_batch_size
flags.optim = 'Adam'
###############################################################################
print(flags)
# if flags.data_set == 'Peurifoy':
# flags.eval_batch_size = 10000
# elif flags.data_set == 'Chen':
# flags.eval_batch_size = 10000
# elif flags.data_set == 'Yang' or flags.data_set == 'Yang_sim':
# flags.eval_batch_size = 2000
#
# flags.batch_size = flags.eval_batch_size
# Get the data
train_loader, test_loader = data_reader.read_data(flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(NA, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters() if p.requires_grad)
print(pytorch_total_params)
# pred_file, truth_file = ntwk.validate_model(save_dir='data/' + flags.data_set+'_best_model', save_misc=save_misc,
# MSE_Simulator=MSE_Simulator, save_Simulator_Ypred=save_Simulator_Ypred)
# Evaluation process
print("Start eval now:")
if multi_flag:
#dest_dir = '/home/sr365/mm_bench_multi_eval_Chen_sweep/NA_init_lr_{}_decay_{}_batch_{}_bp_{}_noise_lvl_{}/'.format(init_lr, lr_decay, flags.eval_batch_size, flags.backprop_step, noise_level)
#dest_dir = '/home/sr365/mm_bench_compare_MDNA_loss/NA_init_lr_{}_decay_{}_MD_loss_{}'.format(flags.lr, flags.lr_decay_rate, flags.md_coeff)
#dest_dir = '/home/sr365/MM_bench_multi_eval/NA_RMSprop/'
#dest_dir = '/data/users/ben/multi_eval/NA_lr' + str(init_lr) + 'bdy_' + str(BDY_strength)+'/'
dest_dir = os.path.join('/home/sr365/MDNA_temp/', save_dir)
dest_dir = os.path.join(dest_dir, flags.data_set)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
#pred_file, truth_file = ntwk.evaluate(save_dir='/work/sr365/multi_eval/NA/' + flags.data_set, save_all=True,
pred_file, truth_file = ntwk.evaluate(save_dir=dest_dir, save_all=True,
save_misc=save_misc, MSE_Simulator=MSE_Simulator,
save_Simulator_Ypred=save_Simulator_Ypred,
noise_level=noise_level)
else:
# Creat the directory is not exist
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
pred_file, truth_file = ntwk.evaluate(save_dir=save_dir, save_misc=save_misc,
MSE_Simulator=MSE_Simulator,
save_Simulator_Ypred=save_Simulator_Ypred,
noise_level=noise_level)
#pred_file, truth_file = ntwk.evaluate(save_dir='data/'+flags.data_set,save_misc=save_misc, MSE_Simulator=MSE_Simulator, save_Simulator_Ypred=save_Simulator_Ypred)
if 'Yang' in flags.data_set:
return
# Plot the MSE distribution
MSE = plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
return MSE
def evaluate_from_model(model_dir,
multi_flag=False,
eval_data_all=False,
save_misc=False,
MSE_Simulator=False,
save_Simulator_Ypred=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param multi_flag: The switch to turn on if you want to generate all different inference trial results
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
print(model_dir)
flags = load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.backprop_step = eval_flags.backprop_step
if flags.data_set == 'ballistics':
flags.test_ratio = 0.0078 # 12800 in total
elif flags.data_set == 'sine_wave':
flags.test_ratio = 0.001 # 8000 in total
elif flags.data_set == 'robotic_arm':
flags.test_ratio = 0.1 # 10000 in total
else:
flags.test_ratio = 0.0051062 / 2
#flags.test_ratio = 0
#flags.test_ratio = 0.00025 # 20000 in total for Meta material
flags.batch_size = 1 # For backprop eval mode, batchsize is always 1
flags.lr = 1e-2
if flags.data_set == 'ballistics':
flags.lr = 1
flags.train_step = eval_flags.train_step
for i in range(4000, 5000, 2000):
for j in range(3):
flags.eval_batch_size = i
# Get the data
train_loader, test_loader = data_reader.read_data(
flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(Backprop,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel()
for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
pred_file, truth_file = ntwk.evaluate(
save_dir='D:/Yang_MM_Absorber_ML/NA/' + flags.data_set,
save_all=True,
save_misc=save_misc,
MSE_Simulator=MSE_Simulator,
save_Simulator_Ypred=save_Simulator_Ypred)
else:
pred_file, truth_file = ntwk.evaluate(
save_dir='D:/Yang_MM_Absorber_ML/Backprop/data/' + str(i) +
'/' + str(j + 1),
save_misc=save_misc,
MSE_Simulator=MSE_Simulator,
save_Simulator_Ypred=save_Simulator_Ypred)
# Plot the MSE distribution
plotMSELossDistrib(
pred_file,
truth_file,
flags,
save_dir='D:/Yang_MM_Absorber_ML/Backprop/data/' + str(i) +
'/' + str(j + 1))
print("Evaluation finished")
