def predict(model_dir, Ytruth_file, multi_flag=False):
"""
Predict the output from given spectra
"""
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
if model_dir.startswith('/'): # It is a absolute path
flags = helper_functions.load_flags(model_dir)
else:
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
ntwk = Network(INN,
flags,
train_loader=None,
test_loader=None,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
pred_file, truth_file = ntwk.predict(Ytruth_file)
if 'Yang' not in flags.data_set:
plotMSELossDistrib(pred_file, truth_file, flags)
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
if model_dir.startswith('/'): # It is a absolute path
flags = helper_functions.load_flags(model_dir)
else:
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.test_ratio = get_test_ratio_helper(flags)
# 2020.10.10 only, delete afterward
flags.test_ratio *= 2
# Get the data
train_loader, test_loader = data_reader.read_data(
flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(MDN,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print(model_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
ntwk.evaluate_multiple_time()
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'meta_material' and not multi_flag:
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def test_categorical_variables():
data_set_list = ["Peurifoy","Chen","Yang_sim"]
SOps = ["roulette","decimation","tournament"]
XOps = ["uniform","single-point"]
for i in [1,2]:
for d in data_set_list:
for x in XOps:
for s in SOps:
dir = d + '_best_model'
flags = load_flags(os.path.join("models", dir))
flags.cross_operator = x
flags.selection_operator = s
flags.data_set = d
flags.eval_model = dir
flags.crossover = 0.8
flags.elitism = 500
flags.k = 500
flags.mutation = 0.05
flags.population = flags.eval_batch_size
flags.ga_eval = False
flags.generations = 10
flags.xtra = i
evaluate_from_model(flags.eval_model,preset_flag=flags,save_Simulator_Ypred=True)
def evaluate_from_model(model_dir):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:return: None
"""
# Retrieve the flag object
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
print("Retrieving flag object for parameters")
print(model_dir)
flags = load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
# Get the data
train_loader, test_loader = data_reader.read_data(flags)
print("Making network now")
# Make Network
ntwk = Network(Backprop,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
# Evaluation process
print("Start eval now:")
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def test_gen_pop():
data_set_list =["Peurifoy","Chen","Yang_sim"]
c = 0
l = os.listdir('data/sweep03')
for d in data_set_list:
for p in [25,50,75,100,150]:
for g in [10,25,50,100,150]:
if d+'_'+str(g)+'_'+str(p) in l:
continue
dir = d + '_best_model'
flags = load_flags(os.path.join("models", dir))
flags.cross_operator = 'single-point'
flags.selection_operator = 'roulette'
flags.data_set = d
flags.eval_model = dir
flags.crossover = 0.8
flags.elitism = int(p/5)
flags.k = int(p/5)
flags.mutation = 0.05
flags.ga_eval = False
flags.generations = g
flags.population = p
flags.xtra = 50
evaluate_from_model(flags.eval_model, preset_flag=flags, save_Simulator_Ypred=True)
def predict_from_model(pre_trained_model, Xpred_file, shrink_factor=1, save_name=''):
"""
Predicting interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:return: None
"""
# Retrieve the flag object
print("This is doing the prediction for file", Xpred_file)
print("Retrieving flag object for parameters")
if (pre_trained_model.startswith("models")):
eval_model = pre_trained_model[7:]
print("after removing prefix models/, now model_dir is:", eval_model)
flags = load_flags(pre_trained_model) # Get the pre-trained model
flags.eval_model = eval_model # Reset the eval mode
# Get the data, this part is useless in prediction but just for simplicity
train_loader, test_loader = data_reader.read_data(flags)
print("Making network now")
# Make Network
ntwk = Network(Backprop, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters() if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
pred_file, truth_file = ntwk.predict(Xpred_file, save_prefix=save_name + 'shrink_factor' + str(shrink_factor), shrink_factor=shrink_factor)
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param multi_flag: The switch to turn on if you want to generate all different inference trial results
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
print(model_dir)
flags = load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.backprop_step = eval_flags.backprop_step
if flags.data_set == 'ballistics':
flags.test_ratio = 0.001
elif flags.data_set == 'sine_wave':
flags.test_ratio = 0.005
elif flags.data_set == 'robotic_arm':
flags.test_ratio = 0.2
elif flags.data_set == 'sine_test_1d':
flags.test_ratio = 0.05
flags.batch_size = 1 # For backprop eval mode, batchsize is always 1
flags.lr = 0.05
flags.eval_batch_size = eval_flags.eval_batch_size
flags.train_step = eval_flags.train_step
# Get the data
train_loader, test_loader = data_reader.read_data(
flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(Backprop,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
pred_file, truth_file = ntwk.evaluate(
save_dir='/work/sr365/multi_eval/Backprop/' + flags.data_set,
save_all=True)
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def predict_from_model(pre_trained_model,
Xpred_file,
no_plot=True,
load_state_dict=None):
"""
Predicting interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param Xpred_file: The Prediction file position
:param no_plot: If True, do not plot (For multi_eval)
:param load_state_dict: The new way to load the model for ensemble MM
:return: None
"""
# Retrieve the flag object
print("This is doing the prediction for file", Xpred_file)
print("Retrieving flag object for parameters")
if (pre_trained_model.startswith("models")):
eval_model = pre_trained_model[7:]
print("after removing prefix models/, now model_dir is:", eval_model)
flags = load_flags(pre_trained_model) # Get the pre-trained model
flags.eval_model = pre_trained_model # Reset the eval mode
flags.test_ratio = 0.1 #useless number
# Get the data, this part is useless in prediction but just for simplicity
#train_loader, test_loader = data_reader.read_data(flags)
print("Making network now")
# Make Network
ntwk = Network(NA,
flags,
train_loader=None,
test_loader=None,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if not no_plot:
# Plot the MSE distribution
pred_file, truth_file = ntwk.predict(Xpred_file,
no_save=False,
load_state_dict=load_state_dict)
flags.eval_model = pred_file.replace(
'.', '_') # To make the plot name different
plotMSELossDistrib(pred_file, truth_file, flags)
else:
pred_file, truth_file = ntwk.predict(Xpred_file,
no_save=True,
load_state_dict=load_state_dict)
print("Evaluation finished")
return pred_file, truth_file, flags
def retrain_different_dataset():
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ["meta_material"]
#data_set_list = ["meta_material","gaussian_model_0310","robotic_armreg0.0005trail_0_complexity_swipe_layer300_num6","sine_wave"]
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model))
flags.model_name = "retrain_time_eval" + flags.model_name
training_from_flag(flags)
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
# Set up the test_ratio
if flags.data_set == 'ballistics':
flags.test_ratio = 0.1
elif flags.data_set == 'sine_wave':
flags.test_ratio = 0.1
elif flags.data_set == 'robotic_arm':
flags.test_ratio = 0.1
# Get the data
train_loader, test_loader = data_reader.read_data(flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(INN, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model)
print(ntwk.ckpt_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters() if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
ntwk.evaluate_multiple_time()
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'meta_material' and not multi_flag:
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
# If gaussian, plot the scatter plot
if flags.data_set == 'gaussian_mixture':
Xpred = helper_functions.get_Xpred(path='data/', name=flags.eval_model)
Ypred = helper_functions.get_Ypred(path='data/', name=flags.eval_model)
# Plot the points scatter
generate_Gaussian.plotData(Xpred, Ypred, save_dir='data/' + flags.eval_model.replace('/','_') + 'generation plot.png', eval_mode=True)
def modulized_evaluate_from_model(model_dir, operate_dir, FF=False, BP=False):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param operate_dir: The directory to operate in (with all the Xpred,Ypred files)
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
print(model_dir)
flags = load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
if BP:
flags.backprop_step = 300
else:
flags.backprop_step = 1
flags.test_ratio = get_test_ratio_helper(flags)
if flags.data_set == 'meta_material':
save_Simulator_Ypred = False
print(
"this is MM dataset, there is no simple numerical simulator therefore setting the save_Simulator_Ypred to False"
)
flags.batch_size = 1 # For backprop eval mode, batchsize is always 1
flags.lr = 0.5
flags.eval_batch_size = 2048
flags.train_step = 500
print(flags)
# Make Network
ntwk = Network(NA,
flags,
train_loader=None,
test_loader=None,
inference_mode=True,
saved_model=flags.eval_model)
# Set up the files
Xpred_list, Xt, Yt = get_xpred_ytruth_xtruth_from_folder(operate_dir)
X_init_mat = reshape_xpred_list_to_mat(Xpred_list)
# Evaluation process
print("Start eval now:")
ntwk.modulized_bp_ff(X_init_mat=X_init_mat,
Ytruth=Yt,
save_dir=operate_dir,
save_all=True,
FF=FF)
def retrain_different_dataset(index):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
#data_set_list = ["peurifoy"]
data_set_list = ["meta_material","robotic_arm","sine_wave","ballistics","peurifoy","chen"]
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model))
flags.model_name = "retrain" + str(index) + eval_model
flags.geoboundary = [-1, 1, -1, 1] # the geometry boundary of meta-material dataset is already normalized in current version
flags.train_step = 500
flags.test_ratio = 0.2
training_from_flag(flags)
def retrain_different_dataset(index):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ["robotic_arm", "sine_wave", "ballistics", "meta_material"]
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model))
flags.model_name = "retrain_" + str(index) + eval_model
flags.geoboundary = [-1, 1, -1, 1]
flags.batch_size = 1024
flags.train_step = 500
flags.test_ratio = 0.2
training_from_flag(flags)
def retrain_different_dataset():
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = [
"robotic_armcouple_layer_num5dim_total4",
"sine_wavecouple_layer_num6dim_total5"
]
#data_set_list = ["robotic_armcouple_layer_num5dim_total4", "sine_wavecouple_layer_num6dim_total5"]
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model))
flags.model_name = "retrain_time_eval" + flags.model_name
training_from_flag(flags)
def retrain_different_dataset(index):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ['mm1', 'mm2', 'mm3', 'mm4', 'mm5']
for eval_model in data_set_list:
flags = load_flags(os.path.join("prev_models", eval_model))
flags.model_name = "retrain_" + str(index) + flags.model_name
flags.data_dir = '/work/sr365/Christian_data_augmented'
flags.ckpt_dir = '/work/sr365/MM_ensemble'
flags.train_step = 500
flags.test_ratio = 0.2
training_from_flag(flags)
def evaluate_forward_model(dirx, n_samples, invs=False):
print("DIRECTORY: ", dirx)
flags = load_flags(dirx)
flags.batch_size = 1
train_loader, test_loader = data_reader.read_data(flags)
GEN = GA(flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=dirx)
GEN.model.eval()
avg_mse, avg_mre, avg_rse = 0, 0, 0
for i, (g, s) in enumerate(test_loader):
if invs:
z = s
s = g
g = z
g = g.cuda()
s = s.cuda()
ps = GEN.model(g)
if invs:
pg = ps
z = g
g = s
s = z
ps = simulator(flags.data_set, pg.cpu().data.numpy())
ps = torch.from_numpy(ps).cuda()
mse = torch.nn.functional.mse_loss(s, ps)
rse = torch.sqrt(torch.sum(torch.pow(s - ps, 2))) / torch.sqrt(
torch.sum(torch.pow(s, 2)))
mre = torch.mean(torch.abs(torch.div(s - ps, s)))
avg_mse += mse.item()
avg_rse += rse.item()
avg_mre += mre.item()
if i == (n_samples - 1):
print('BROKE at sample {}'.format(i))
break
avg_mse /= n_samples
avg_mre /= n_samples
avg_rse /= n_samples
print("\nMSE:\t{}\nMRE:\t{}\nRSE:\t{}".format(avg_mse, avg_mre, avg_rse))
def creat_mm_dataset():
"""
Function to create the meta-material dataset from the saved checkpoint files
:return:
"""
# Define model folder
model_folder = os.path.join('..', 'Simulated_DataSets',
'Meta_material_Neural_Simulator',
'meta_material')
# Load the flags to construct the model
flags = load_flags(model_folder)
flags.eval_model = model_folder
ntwk = Network(NA,
flags,
train_loader=None,
test_loader=None,
inference_mode=True,
saved_model=flags.eval_model)
# This is the full file version, which would take a while. Testing pls use the next line one
geometry_points = os.path.join('..', 'Simulated_DataSets',
'Meta_material_Neural_Simulator', 'dataIn',
'data_x.csv')
# Small version is for testing, the large file taks a while to be generated...
#geometry_points = os.path.join('..', 'Simulated_DataSets', 'Meta_material_Neural_Simulator', 'dataIn', 'data_x_small.csv')
Y_filename = geometry_points.replace('data_x', 'data_y_1')
# Set up the list of prediction files
pred_list = []
# for each model saved, load the dictionary and do the inference
for i in range(5):
print('predicting for {}th model saved'.format(i + 1))
state_dict_file = os.path.join('..', 'Simulated_DataSets',
'Meta_material_Neural_Simulator',
'state_dicts', 'mm{}.pth'.format(i + 1))
pred_file, truth_file = ntwk.predict(Xpred_file=geometry_points,
load_state_dict=state_dict_file,
no_save=True)
pred_list.append(pred_file)
Y_ensemble = np.zeros(shape=(*np.shape(pred_file), 5))
# Combine the predictions by doing the average
for i in range(5):
Y_ensemble[:, :, i] = pred_list[i]
Y_ensemble = np.mean(Y_ensemble, axis=2)
#X = pd.read_csv(geometry_points, header=None, sep=' ').values
#MM_data = np.concatenate((X, Y_ensemble), axis=1)
#MM_data_file = geometry_points.replace('data_x', 'dataIn/MM_data')
np.savetxt(Y_filename, Y_ensemble)
def retrain_different_dataset(index):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ['Peurifoy', 'Yang_sim', 'Chen']
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model + '_best_model'))
flags.model_name = "retrain" + str(index) + eval_model
flags.geoboundary = [
-1, 1, -1, 1
] # the geometry boundary of meta-material dataset is already normalized in current version
flags.train_step = 500
flags.test_ratio = 0.2
training_from_flag(flags)
def retrain_different_dataset():
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = [
"gaussian_mixturekl_coeff0.04lr0.01reg0.005",
"robotic_armlayer_num6unit_500reg0.005trail2",
"meta_materialkl_coeff0.06lr0.001reg0.005",
"sine_wavekl_coeff0.04lr0.001reg0.005"
]
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model))
flags.model_name = "retrain_time_eval" + flags.model_name
training_from_flag(flags)
def retrain_different_dataset():
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = [
"robotic_armcouple_layer_num6trail_0",
"sine_wavecouple_layer_num8trail_0",
"meta_materialcouple_layer_num5trail_1",
"gaussian_mixturecouple_layer_num6trail_1"
]
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model))
flags.model_name = "retrain_time_eval" + flags.model_name
training_from_flag(flags)
def retrain_different_dataset():
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ['ballistics', 'robotic_arm']
#data_set_list = ['meta_material']
for eval_model in data_set_list:
flags = load_flags(os.path.join("prev_models", eval_model))
flags.model_name = "retrain_" + flags.model_name
#flags.geoboundary = [-1,1,-1,1]
#flags.data_dir = '/work/sr365/MM_ensemble/'
flags.train_step = 500
flags.test_ratio = 0.2
training_from_flag(flags)
def retrain_different_dataset(index):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ["meta_material", "robotic_arm", "sine_wave", "ballistics"]
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model))
flags.model_name = "retrain_" + flags.model_name
flags.ckpt_dir = 'models/'
flags.batch_size = 1024
flags.train_step = 500
flags.test_ratio = 0.2
flags.stop_threshold = -float('inf')
training_from_flag(flags)
def retrain_different_dataset(index):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
#data_set_list = ['ballistics', 'robotic_arm', 'sine_wave']
data_set_list = ['meta_material']
for eval_model in data_set_list:
flags = load_flags(os.path.join("prev_models", eval_model))
flags.model_name = "retrain" + str(index) + eval_model
flags.batch_size = 1024
flags.geoboundary = [-1, 1, -1, 1]
flags.train_step = 500
flags.test_ratio = 0.2
flags.stop_threshold = -float('inf')
training_from_flag(flags)
def evaluate_from_model(model_dir, multi_flag=False, eval_data_all=False, test_ratio=None):
"""
Evaluating interface. 1. Retreive the flags 2. get data 3. initialize network 4. eval
:param model_dir: The folder to retrieve the model
:param eval_data_all: The switch to turn on if you want to put all data in evaluation data
:return: None
"""
# Retrieve the flag object
print("Retrieving flag object for parameters")
if (model_dir.startswith("models")):
model_dir = model_dir[7:]
print("after removing prefix models/, now model_dir is:", model_dir)
flags = helper_functions.load_flags(os.path.join("models", model_dir))
flags.eval_model = model_dir # Reset the eval mode
flags.batch_size = 1
flags.backprop_step=300
flags.eval_batch_size=2048
if test_ratio is None:
flags.test_ratio = get_test_ratio_helper(flags)
else:
# To make the test ratio swipe with respect to inference time
# also making the batch size large enough
flags.test_ratio = test_ratio
# Get the data
train_loader, test_loader = data_reader.read_data(flags, eval_data_all=eval_data_all)
print("Making network now")
# Make Network
ntwk = Network(make_cINN_and_NA, flags, train_loader, test_loader, inference_mode=True, saved_model=flags.eval_model)
#print(model_dir)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model_cINN.parameters() if p.requires_grad)
print(pytorch_total_params)
pytorch_total_params = sum(p.numel() for p in ntwk.model_NA.parameters() if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if multi_flag:
pred_file, truth_file = ntwk.evaluate(save_dir='/work/sr365/NIPS_multi_eval_backup/multi_eval/hybrid_cINN_NA_0bp/'+flags.data_set, save_all=True)
else:
pred_file, truth_file = ntwk.evaluate()
# Plot the MSE distribution
if flags.data_set != 'meta_material' and not multi_flag:
plotMSELossDistrib(pred_file, truth_file, flags)
print("Evaluation finished")
def retrain_different_dataset(index):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ["meta_material", "robotic_arm", "sine_wave", "ballistics"]
for train_model in data_set_list:
flags = load_flags(os.path.join("models", train_model))
#if train_model is 'meta_material':
# flags.data_dir = os.path.join('../', 'Simulated_DataSets', 'Meta_material_Neural_Simulator')
flags.model_name = "retrain" + str(index) + train_model
flags.ckpt_dir = 'models/'
flags.batch_size = 1024
flags.train_step = 500
flags.test_ratio = 0.2
flags.stop_threshold = -float('inf')
training_from_flag(flags)
def retrain_different_dataset(index):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ['robotic_arm']
for eval_model in data_set_list:
for j in range(4, 10):
flags = load_flags(os.path.join("prev_models", eval_model))
flags.model_name = "retrain_" + str(index) + '_layer_' + str(
j) + flags.model_name
flags.couple_layer_num = j
flags.batch_size = 1024
flags.train_step = 500
flags.test_ratio = 0.2
flags.stop_threshold = -float('inf')
training_from_flag(flags)
def retrain_different_dataset(ind=None):
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = ['sine_wave']
#data_set_list = ['meta_material']
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model))
flags.model_name = "retrain_" + flags.model_name
if ind is not None:
flags.model_name += 'trail_{}'.format(ind)
#flags.geoboundary = [-1,1,-1,1]
#flags.data_dir = '/work/sr365/MM_ensemble/'
flags.train_step = 500
flags.test_ratio = 0.2
training_from_flag(flags)
def retrain_different_dataset():
"""
This function is to evaluate all different datasets in the model with one function call
"""
from utils.helper_functions import load_flags
data_set_list = [
"robotic_armreg0.0005trail_0_backward_complexity_swipe_layer500_num6",
"sine_wavereg0.005trail_1_complexity_swipe_layer1000_num8",
"ballisticsreg0.0005trail_0_complexity_swipe_layer500_num5",
"meta_materialreg0.0005trail_2_complexity_swipe_layer1000_num6"
]
for eval_model in data_set_list:
flags = load_flags(os.path.join("models", eval_model))
flags.model_name = "retrain_time_eval" + flags.model_name
flags.train_step = 500
flags.test_ratio = 0.2
training_from_flag(flags)
def infer(pre_trained_model, Xpred_file, no_plot=True):
# Retrieve the flag object
print("This is doing the prediction for file", Xpred_file)
print("Retrieving flag object for parameters")
if (pre_trained_model.startswith("models")):
eval_model = pre_trained_model[7:]
print("after removing prefix models/, now model_dir is:", eval_model)
flags = load_flags(pre_trained_model) # Get the pre-trained model
flags.eval_model = pre_trained_model # Reset the eval mode
flags.test_ratio = 0.1 # useless number
# Get the data, this part is useless in prediction but just for simplicity
train_loader, test_loader = data_reader.read_data(flags)
print("Making network now")
# Make Network
ntwk = Network(Backprop,
flags,
train_loader,
test_loader,
inference_mode=True,
saved_model=flags.eval_model)
print("number of trainable parameters is :")
pytorch_total_params = sum(p.numel() for p in ntwk.model.parameters()
if p.requires_grad)
print(pytorch_total_params)
# Evaluation process
print("Start eval now:")
if not no_plot:
# Plot the MSE distribution
pred_file, truth_file = ntwk.predict(Xpred_file, no_save=False)
flags.eval_model = pred_file.replace(
'.', '_') # To make the plot name different
plotMSELossDistrib(pred_file, truth_file, flags)
else:
pred_file, truth_file = ntwk.predict(Xpred_file, no_save=True)
print("Evaluation finished")
return pred_file, truth_file, flags
def test_num_samples():
# NA: Yang - 16.5, Peurifoy - 21.67, Chen -3.53
#gen = [164,165]#[30,28] #28 Chen, #120 Peurifoy, #Yang 120
#pop = [312,320] #[800, 180, 148] #? Peurifoy, #800 Yang, 148 Chen
#elite = [78,80] #[400, 45, 37]
#saveto = '_gen_50' #'_gen_lim_1'
# GPU 1
# gen = [300,120,300]
# ds = ['Peurifoy','Peurifoy','Chen']
# saveto = ['unres','res','unres']
# GPU 0
#gen = [300,28,120]
#ds = ['Yang_sim','Chen','Yang_sim']
#saveto = ['res','res','unres']
ds = ['Yang_sim','Yang_sim']
gen = [120,300,120,300]
saveto = ['GA2','GA1','GA2','GA1','GA2','GA1']
for i,dset in enumerate(ds):
dxy = dset + '_best_model'
flags = load_flags(os.path.join("models", dxy))
flags.data_set = dset
flags.cross_operator = 'single-point'
flags.selection_operator = 'roulette'
flags.eval_model = dxy
flags.crossover = 0.8
flags.elitism = 500
flags.k = 500
flags.mutation = 0.05
flags.population = 2048
flags.ga_eval = False
flags.generations = gen[i]
flags.xtra = 20
flags.save_to = saveto[i]
evaluate_from_model(flags.eval_model, preset_flag=flags, save_Simulator_Ypred=False, multi_flag=True)
