def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
# Set up the test_ratio
if flags.data_set == 'ballistics':
flags.test_ratio = 0.1
elif flags.data_set == 'sine_wave':
flags.test_ratio = 0.1
elif flags.data_set == 'robotic_arm':
flags.test_ratio = 0.1
# Get the data
train_loader, test_loader = data_reader.read_data(flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(INN, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model)
print(ntwk.ckpt_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters() if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
ntwk.evaluate_multiple_time()
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'meta_material' and not multi_flag:
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
# If gaussian, plot the scatter plot
if flags.data_set == 'gaussian_mixture':
Xpred = helper_functions.get_Xpred(path='data/', name=flags.eval_model)
Ypred = helper_functions.get_Ypred(path='data/', name=flags.eval_model)
# Plot the points scatter
generate_Gaussian.plotData(Xpred, Ypred, save_dir='data/' + flags.eval_model.replace('/','_') + 'generation plot.png', eval_mode=True)
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
if model_dir.startswith('/'): # It is a absolute path
flags = helper_functions.load_flags(model_dir)
else:
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.test_ratio = get_test_ratio_helper(flags)
# 2020.10.10 only, delete afterward
flags.test_ratio *= 2
# Get the data
train_loader, test_loader = data_reader.read_data(
flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(MDN,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print(model_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
ntwk.evaluate_multiple_time()
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'meta_material' and not multi_flag:
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False, test_ratio=None):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
flags = load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
if test_ratio is None:
flags.test_ratio = get_test_ratio_helper(flags)
else:
# To make the test ratio swipe with respect to inference time
# also making the batch size large enough
flags.test_ratio = test_ratio
flags.batch_size = 2000
# Get the data
train_loader, test_loader = data_reader.read_data(flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(Forward, Backward, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model_f.parameters() if p.requires_grad) +\
sum(p.numel() for p in ntwk.model_b.parameters() if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
ntwk.evaluate_multiple_time()
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'meta_material' and not multi_flag:
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def evaluate_from_model(model_dir,
multi_flag=False,
eval_data_all=False,
modulized_flag=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.test_ratio = get_test_ratio_helper(flags)
# Get the data
train_loader, test_loader = data_reader.read_data(
flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(INN,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print(ntwk.ckpt_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if modulized_flag:
ntwk.evaluate_modulized_multi_time()
elif multi_flag:
ntwk.evaluate_multiple_time()
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'Yang_sim' and not multi_flag and not modulized_flag: # meta-material does not have simulator, hence no Ypred given
MSE = plotMSELossDistrib(pred_file, truth_file, flags)
# Add this MSE back to the folder
flags.best_validation_loss = MSE
helper_functions.save_flags(flags, os.path.join("models", model_dir))
elif flags.data_set == 'Yang_sim' and not multi_flag and not modulized_flag:
# Save the current path for getting back in the future
cwd = os.getcwd()
abs_path_Xpred = os.path.abspath(pred_file.replace('Ypred', 'Xpred'))
# Change to NA dictory to do prediction
os.chdir('../NA/')
MSE = predict.ensemble_predict_master('../Data/Yang_sim/state_dicts/',
abs_path_Xpred,
no_plot=False)
# Add this MSE back to the folder
flags.best_validation_loss = MSE
os.chdir(cwd)
helper_functions.save_flags(flags, os.path.join("models", model_dir))
print("Evaluation finished")
