def get_models_paths(model_id, epoch_i=-1, source_path="../results/bidaf/"):
"""
Function that returns the path of the files where the data of the model are loaded:
- pickle_results_path: Where the pickle results with the config_file and training_logger are
- model_file_path: Where the weights for the model are.
model_id: Identifies the experiment to be loaded
epoch_id: Identifies the epoch
"""
pickle_results_folder = source_path + model_id + "/training_logger/"
model_file_folder = source_path + model_id + "/models/"
if (epoch_i == -1):
# Get the latest element form the folders
files_path = ul.get_allPaths(pickle_results_folder)
if (len(files_path) > 0):
# Load the latest params !
files_path = sorted(files_path,
key=ul.cmp_to_key(ul.filenames_comp))
epoch_i = int(files_path[-1].split("/")[-1].split(".")[0])
print("Last epoch: ", epoch_i)
pickle_results_path = pickle_results_folder + str(epoch_i) + ".pkl"
model_file_path = model_file_folder + str(epoch_i) + ".prm"
return pickle_results_path, model_file_path
def create_video_from_images(video_fotograms_folder,
output_file="out.avi",
fps=2):
images_path = ul.get_allPaths(video_fotograms_folder)
images_path = sorted(images_path, key=ul.cmp_to_key(ul.filenames_comp))
# print(images_path)
vul.create_video(images_path, output_file=output_file, fps=fps)
def get_all_checkpoints_paths(path):
"""
This function gets all the checkpints in the folder, since each of them has a file like:
General_model1-61.index
"""
all_paths = ul.get_allPaths(path)
model_checkpoints = []
for path in all_paths:
name = path.split("/")[-1]
if (name.find(".index") != -1):
name = name.split(".index")[0]
model_checkpoints.append(name)
def get_number(element):
return int(element.split("-")[-1])
model_checkpoints.sort(key = get_number)
return model_checkpoints
def get_all_checkpoints_paths(path):
"""
This function gets all the checkpints in the folder, since each of them has a file like:
General_model1-61.index
"""
all_paths = ul.get_allPaths(path)
model_checkpoints = []
for path in all_paths:
name = path.split("/")[-1]
if (name.find(".index") != -1):
name = name.split(".index")[0]
model_checkpoints.append(name)
def get_number(element):
return int(element.split("-")[-1])
model_checkpoints.sort(key=get_number)
return model_checkpoints
def get_all_pickles_training(source_path = "../results/bidaf/", include_models = False):
"""
Function that returns the pickle path of all models in the folder
"""
pickle_results_path = []
model_results_path = []
files_path = ul.get_allPaths(source_path)
if (len(files_path) >0):
# Load the latest params !
for i in range(len(files_path)):
if(files_path[i].split(".")[-1] == "pkl"):
pickle_results_path.append(files_path[i])
if (include_models):
epoch_i = files_path[i].split("/")[-1].split(".")[0]
model_path = "/".join([files_path[i].split("/")[j] \
for j in range(len(files_path[i].split("/"))-2)])
model_path += "/models/" + str(epoch_i) + ".prm"
model_results_path.append(model_path)
if(include_models):
return pickle_results_path,model_results_path
return pickle_results_path
def get_all_pickles_training(source_path="../results/bidaf/",
include_models=False):
"""
Function that returns the pickle path of all models in the folder
"""
pickle_results_path = []
model_results_path = []
files_path = ul.get_allPaths(source_path)
if (len(files_path) > 0):
# Load the latest params !
for i in range(len(files_path)):
if (files_path[i].split(".")[-1] == "pkl"):
pickle_results_path.append(files_path[i])
if (include_models):
epoch_i = files_path[i].split("/")[-1].split(".")[0]
model_path = "/".join([files_path[i].split("/")[j] \
for j in range(len(files_path[i].split("/"))-2)])
model_path += "/models/" + str(epoch_i) + ".prm"
model_results_path.append(model_path)
if (include_models):
return pickle_results_path, model_results_path
return pickle_results_path
def get_models_paths(model_id, epoch_i = -1, source_path = "../results/bidaf/"):
"""
Function that returns the path of the files where the data of the model are loaded:
- pickle_results_path: Where the pickle results with the config_file and training_logger are
- model_file_path: Where the weights for the model are.
model_id: Identifies the experiment to be loaded
epoch_id: Identifies the epoch
"""
pickle_results_folder = source_path +model_id + "/training_logger/"
model_file_folder = source_path + model_id + "/models/"
if (epoch_i == -1):
# Get the latest element form the folders
files_path = ul.get_allPaths(pickle_results_folder)
if (len(files_path) >0):
# Load the latest params !
files_path = sorted(files_path, key = ul.cmp_to_key(ul.filenames_comp))
epoch_i = int(files_path[-1].split("/")[-1].split(".")[0])
print ("Last epoch: ",epoch_i)
pickle_results_path = pickle_results_folder+ str(epoch_i) + ".pkl"
model_file_path =model_file_folder + str(epoch_i) + ".prm"
return pickle_results_path,model_file_path
shuffle=True)
"""
######################## Instantiate Architecture ########################
"""
#myGeneralVBModel = RNN_names_classifier(cf_a, prior)
myGeneralVBModel = RNN_names_classifier_fullVB(cf_a, prior)
myGeneralVBModel.set_languages(all_categories)
#print (myGeneralVBModel.predict_language(Xtrain))
# Set the model in training mode for the forward pass. self.training = True
# This is for Dropout and VI
myGeneralVBModel.train()
if (load_previous_state):
files_path = ul.get_allPaths(folder_model)
if (len(files_path) > 0):
# Load the latest params !
files_path = sorted(files_path,
key=ul.cmp_to_key(ul.filenames_comp_model_param))
myGeneralVBModel.load(files_path[-1])
"""
######################## Visualize variables ########################
Code that visualizes the architecture, data propagation and gradients.
"""
if (see_variables):
print(myGeneralVBModel)
myGeneralVBModel.print_parameters()
myGeneralVBModel.print_parameters_names()
myGeneralVBModel.print_named_parameters()
import import_folders
import utilities_lib as ul
import convert_lib as convl
#updates_folder = "../Hanseatic/MQL4/Files/"
#storage_folder = "./storage/Hanseatic/"
#updates_folder = "../FxPro/MQL4/Files/"
#storage_folder = "./storage/FxPro/"
updates_folder = "../GCI/MQL4/Files/"
storage_folder = "./storage/GCI/"
hst_folder ="../GCI/history/GCI-Demo/"
#hst_folder ="../FxPro/history/FxPro.com-Demo04/"
#hst_folder ="../Hanseatic/history/HBS-CFD-Server/"
#convl.process_hst(hst_file)
#hst_file ="../GCI/history/GCI-Demo/EURCAD.s1.hst"
all_paths = ul.get_allPaths(hst_folder)
for path in all_paths:
convl.process_hst(path)
import import_folders
import utilities_lib as ul
import convert_lib as convl
#updates_folder = "../Hanseatic/MQL4/Files/"
#storage_folder = "./storage/Hanseatic/"
#updates_folder = "../FxPro/MQL4/Files/"
#storage_folder = "./storage/FxPro/"
updates_folder = "../GCI/MQL4/Files/"
storage_folder = "./storage/GCI/"
hst_folder = "../GCI/history/GCI-Demo/"
#hst_folder ="../FxPro/history/FxPro.com-Demo04/"
#hst_folder ="../Hanseatic/history/HBS-CFD-Server/"
#convl.process_hst(hst_file)
#hst_file ="../GCI/history/GCI-Demo/EURCAD.s1.hst"
all_paths = ul.get_allPaths(hst_folder)
for path in all_paths:
convl.process_hst(path)
#training_set = Dataset(Xtrain, Ytrain)
train = data_utils.TensorDataset(Xtrain, Ytrain)
train_loader = data_utils.DataLoader(train, batch_size=cf_a.batch_size_train, shuffle=True)
"""
######################## Instantiate Architecture ########################
"""
cf_a.Nsamples_train = Ntr
myGeneralVBModel = GeneralVBModel(cf_a)
# Set the model in training mode for the forward pass. self.training = True
# This is for Dropout and VI
myGeneralVBModel.train()
if (load_previous_state):
files_path = ul.get_allPaths(folder_model)
if (len(files_path) >0):
# Load the latest params !
files_path = sorted(files_path, key = ul.cmp_to_key(ul.filenames_comp))
myGeneralVBModel.load(files_path[-1])
myGeneralVBModel.to(device = device)
"""
######################## Visualize variables ########################
Code that visualizes the architecture, data propagation and gradients.
"""
if (see_variables):
print(myGeneralVBModel)
myGeneralVBModel.print_parameters()
myGeneralVBModel.print_parameters_names()
#######################################################################################################################
#### Obtain the evolution of the centroids to plot them properly #####################################################
#######################################################################################################################
if(plot_evolution):
spf.plot_final_distribution([X1,X2,X3],[mu1,mu2,mu3],[cov1,cov2,cov3], [K_G, K_W, K_vMF],myDManager, logl,theta_list,mode_theta_list,folder_images)
if (plot_evolution_2):
spf.plot_multiple_iterations([X1,X2,X3],[mu1,mu2,mu3],[cov1,cov2,cov3], [K_G, K_W, K_vMF],myDManager, logl,theta_list,mode_theta_list,folder_images)
if (plot_evolution_video):
folder_images_gif = "../pics/Trapying/EM_HMM/gif/"
spf.generate_images_iterations([X1,X2,X3],[mu1,mu2,mu3],[cov1,cov2,cov3], [K_G, K_W, K_vMF],myDManager, logl,theta_list,mode_theta_list,folder_images_gif)
#### Load the images
images_path = ul.get_allPaths(folder_images_gif, fullpath = "no")
images_path.sort(cmp = ul.comparador_images_names)
### Create Gif ###
output_file_gif = 'Evolution_gif. K_G:'+str(K_G)+ ', K_W:' + str(K_W) + ', K_vMF:' + str(K_vMF)+".gif"
ul.create_gif(images_path,folder_images + output_file_gif, duration = 0.1)
## Create video ##
output_file = folder_images + 'Evolution_video. K_G:'+str(K_G)+ ', K_W:' + str(K_W) + ', K_vMF:' + str(K_vMF) +'.avi'
ul.create_video(images_path, output_file = output_file, fps = 5)
if (plot_evolution_ll_video):
folder_images_gif = "../pics/Trapying/EM_HMM/gif/"
spf.generate_images_iterations_ll([X1,X2,X3],[mu1,mu2,mu3],[cov1,cov2,cov3], [K_G, K_W, K_vMF],myDManager, logl,theta_list,mode_theta_list,folder_images_gif)
#### Load the images
images_path = ul.get_allPaths(folder_images_gif, fullpath = "no")
images_path.sort(cmp = ul.comparador_images_names)
## Create video ##
def create_video_from_images(video_fotograms_folder, output_file = "out.avi", fps = 2):
images_path = ul.get_allPaths(video_fotograms_folder)
images_path = sorted(images_path, key = ul.cmp_to_key(ul.filenames_comp))
# print(images_path)
vul.create_video(images_path, output_file = output_file, fps = fps)
