def savefig(
self,
file_dir="./image.png",
bbox_inches='tight',
sizeInches=[], # The size in inches as a list
close=False, # If we close the figure once saved
dpi=100): # Density of pixels !! Same image but more cuality ! Pixels
try:
folders = file_dir.split("/")
folders.pop(-1)
path = "/".join(folders)
ul.create_folder_if_needed(path)
except:
pass
## Function to save the current figure in the desired format
## Important !! Both dpi and sizeInches affect the number of pixels of the image
# dpi: It is just for quality, same graph but more quality.
# sizeInches: You change the proportions of the window. The fontsize and
# thickness of lines is the same. So as the graph grows, they look smaller.
F = self.fig # ?? NO work ?
# F = pylab.gcf()
Winches, Hinches = F.get_size_inches() # 8,6
# print Winches, Hinches
if (len(sizeInches) > 0):
F.set_size_inches((sizeInches[0], sizeInches[1]))
self.fig.savefig(file_dir, bbox_inches=bbox_inches, dpi=dpi)
# Transform back
F.set_size_inches((Winches, Hinches))
if (close == True):
plt.close()
def save_to_csv(self, file_dir="./storage/"):
ul.create_folder_if_needed(file_dir)
whole_path = file_dir + ul.period_dic[
self.period] + "/" + self.symbol + "_" + ul.period_dic[
self.period] + ".csv"
ul.create_folder_if_needed(file_dir + ul.period_dic[self.period] + "/")
self.TD.to_csv(whole_path, sep=',')
def save_to_csv(self, file_dir = "./storage/", force = False):
# This function saves the TD to a csv file
if (self.trimmed == True and force == False):
print ("You cannot save the file since you trimmed it, Use force = True")
else:
ul.create_folder_if_needed(file_dir)
whole_path = file_dir + ul.period_dic[self.period] + "/" + \
self.symbolID + "_" + ul.period_dic[self.period] + ".csv"
ul.create_folder_if_needed(file_dir + ul.period_dic[self.period] + "/")
self.TD.to_csv(whole_path, sep=',')
def save_to_csv(self, file_dir = "./storage/", force = False):
# This function saves the TD to a csv file
if (self.trimmed == True and force == False):
print ("You cannot save the file since you trimmed it, Use force = True")
else:
ul.create_folder_if_needed(file_dir)
whole_path = file_dir + ul.period_dic[self.period] + "/" + \
self.symbolID + "_" + ul.period_dic[self.period] + ".csv"
ul.create_folder_if_needed(file_dir + ul.period_dic[self.period] + "/")
self.TD.to_csv(whole_path, sep=',')
def get_dir(self, remotedir, localdir = "./", recursive = False, preserve_mtime=False):
"""
Function that downloads the specified file from the server into the local file name provided.
If Recursive it will download all subfolders.
"""
self.connect()
ul.create_folder_if_needed(localdir)
if(recursive):
self.sftp.get_r(remotedir, localdir, preserve_mtime=preserve_mtime)
else:
self.sftp.get_d(remotedir, localdir, preserve_mtime=preserve_mtime)
self.close()
def get_dir(self,
remotedir,
localdir="./",
recursive=False,
preserve_mtime=False):
"""
Function that downloads the specified file from the server into the local file name provided.
If Recursive it will download all subfolders.
"""
self.connect()
ul.create_folder_if_needed(localdir)
if (recursive):
self.sftp.get_r(remotedir, localdir, preserve_mtime=preserve_mtime)
else:
self.sftp.get_d(remotedir, localdir, preserve_mtime=preserve_mtime)
self.close()
## Import standard libraries
import tensorflow as tf
# Import libraries
from graph_lib import gl
import numpy as np
import matplotlib.pyplot as plt
import utilities_lib as ul
import tf_general_model1 as tfm1
plt.close("all")
tb_folder = "../TensorBoard/Examples/3_LR_copmlete_Model/"
variables_folder = "../TensorBoard/Examples/3_LR_copmlete_Model/Variables/"
folder_images = "../pics/Trapying/TensorBoard_Examples/"
ul.create_folder_if_needed(tb_folder)
ul.create_folder_if_needed(variables_folder)
ul.create_folder_if_needed(folder_images)
##########################################################################
################ Load the Data ########################################
##########################################################################
X_data_tr = np.array([1.2,1.8,3.2,3.9,4.7,6.4, 4.4])
Y_data_tr = np.array([0, -1,-2,-3,-4, -5, -9])
X_data_val = np.array([8,9,10,11])
Y_data_val = np.array([-7.1, -7.9,-9.2,-10.1])
#### Hyperparameters ####
## Import TF creared import BBB_LSTM_configs as Bconf import BBB_LSTM_Model as BM import Variational_inferences_lib as VI import util_autoparallel as util import pickle_lib as pkl from graph_lib import gl import numpy as np import matplotlib.pyplot as plt import utilities_lib as ul tb_folder = "../TensorBoard/Examples/1_basic/" ul.create_folder_if_needed(tb_folder) ## Destroy the graph tf.reset_default_graph() ## Build the graph ## We need a global initializer a = tf.constant(5.0) b = tf.constant(6.9) c = tf.multiply(a,b) # We can use normal operators over tensors d = a * c ## Create a Session object to lauch the graph
eta_KL) + "/"
video_fotograms_folder_weights = folder_images + "Weights/"
video_fotograms_folder_training = folder_images + "Training/"
folder_model = "../models/pyTorch/BasicExamples_Bayesian/MLP/"
"""
###################### Setting up func ###########################
"""
## Set up seeds for more reproducible results
random_seed = 0 #time_now.microsecond
np.random.seed(random_seed)
torch.manual_seed(random_seed)
torch.cuda.manual_seed(random_seed)
## Windows and folder management
plt.close("all") # Close all previous Windows
ul.create_folder_if_needed(folder_images)
ul.create_folder_if_needed(video_fotograms_folder_weights)
ul.create_folder_if_needed(video_fotograms_folder_training)
ul.create_folder_if_needed(folder_model)
if (create_video_training):
ul.remove_files(video_fotograms_folder_weights)
ul.remove_files(video_fotograms_folder_training)
"""
######################## LOAD AND PROCESS THE DATA ########################
"""
## Original Numpy data. The last data point is an outlier !!
if (linear_data):
[X_data_tr, Y_data_tr, X_data_val,
Y_data_val] = dl.get_linear_dataset(outlier=False)
folder_images = "../pics/Pytorch/BasicExamples_Bayesian/MLP_classification/"+ str(eta_KL) + "/"
video_fotograms_folder_weights = folder_images +"Weights/"
video_fotograms_folder_training = folder_images +"Training/"
folder_model = "../models/pyTorch/BasicExamples_Bayesian/MLP/"
"""
###################### Setting up func ###########################
"""
## Set up seeds for more reproducible results
random_seed = 0 #time_now.microsecond
np.random.seed(random_seed)
torch.manual_seed(random_seed)
torch.cuda.manual_seed(random_seed)
## Windows and folder management
plt.close("all") # Close all previous Windows
ul.create_folder_if_needed(folder_images)
ul.create_folder_if_needed(video_fotograms_folder_weights)
ul.create_folder_if_needed(video_fotograms_folder_training)
ul.create_folder_if_needed(folder_model)
if(create_video_training):
ul.remove_files(video_fotograms_folder_weights)
ul.remove_files(video_fotograms_folder_training)
"""
######################## LOAD AND PROCESS THE DATA ########################
"""
## Original Numpy data. The last data point is an outlier !!
[X_data_tr, Y_data_tr, X_data_val,Y_data_val, enc] = dl.get_toy_classification_data(n_samples=100, centers=3, n_features=2)
xgrid_real_func = None; ygrid_real_func = None;
def generate_images_iterations_ll(Xs,mus,covs, Ks ,myDManager, logl,theta_list,model_theta_list,folder_images_gif):
# os.remove(folder_images_gif) # Remove previous images if existing
"""
WARNING: MEANT FOR ONLY 3 Distributions due to the color RGB
"""
import shutil
ul.create_folder_if_needed(folder_images_gif)
shutil.rmtree(folder_images_gif)
ul.create_folder_if_needed(folder_images_gif)
######## Plot the original data #####
Xdata = np.concatenate(Xs,axis = 1).T
colors = ["r","b","g"]
K_G,K_W,K_vMF = Ks
### FOR EACH ITERATION
for i in range(len(theta_list)): # theta_list
indx = i
gl.init_figure()
ax1 = gl.subplot2grid((1,2), (0,0), rowspan=1, colspan=1)
## Get the relative ll of the Gaussian denoising cluster.
ll = myDManager.pdf_log_K(Xdata,theta_list[indx])
N,K = ll.shape
# print ll.shape
for j in range(N): # For every sample
#TODO: Can this not be done without a for ?
# Normalize the probability of the sample being generated by the clusters
Marginal_xi_probability = gf.sum_logs(ll[j,:])
ll[j,:] = ll[j,:]- Marginal_xi_probability
ax1 = gl.scatter(Xdata[j,0],Xdata[j,1], labels = ['EM Evolution. Kg:'+str(K_G)+ ', Kw:' + str(K_W) + ', K_vMF:' + str(K_vMF), "X1","X2"],
color = (np.exp(ll[j,1]), np.exp(ll[j,0]), np.exp(ll[j,2])) , ### np.exp(ll[j,2])
alpha = 1, nf = 0)
# Only doable if the clusters dont die
for k_c in myDManager.clusterk_to_Dname.keys():
k = myDManager.clusterk_to_thetak[k_c]
distribution_name = myDManager.clusterk_to_Dname[k_c] # G W
if (distribution_name == "Gaussian"):
## Plot the ecolution of the mu
#### Plot the Covariance of the clusters !
mean,w,h,theta = bMA.get_gaussian_ellipse_params( mu = theta_list[indx][k][0], Sigma = theta_list[indx][k][1], Chi2val = 2.4477)
r_ellipse = bMA.get_ellipse_points(mean,w,h,theta)
gl.plot(r_ellipse[:,0], r_ellipse[:,1], ax = ax1, ls = "-.", lw = 3,
AxesStyle = "Normal2",
legend = ["Kg(%i). pi:%0.2f"%(k, float(model_theta_list[indx][0][0,k]))])
elif(distribution_name == "Watson"):
#### Plot the pdf of the distributino !
## Distribution parameters for Watson
kappa = float(theta_list[indx][k][1]); mu = theta_list[-1][k][0]
Nsa = 1000
# Draw 2D samples as transformation of the angle
Xalpha = np.linspace(0, 2*np.pi, Nsa)
Xgrid= np.array([np.cos(Xalpha), np.sin(Xalpha)])
probs = [] # Vector with probabilities
for i in range(Nsa):
probs.append(np.exp(Wad.Watson_pdf_log(Xgrid[:,i],[mu,kappa]) ))
probs = np.array(probs)
# Plot it in polar coordinates
X1_w = (1 + probs) * np.cos(Xalpha)
X2_w = (1 + probs) * np.sin(Xalpha)
gl.plot(X1_w,X2_w,
alpha = 1, lw = 3, ls = "-.", legend = ["Kw(%i). pi:%0.2f"%(k, float(model_theta_list[indx][0][0,k]))])
elif(distribution_name == "vonMisesFisher"):
#### Plot the pdf of the distributino !
## Distribution parameters for Watson
kappa = float(theta_list[indx][k][1]); mu = theta_list[indx][k][0]
Nsa = 1000
# Draw 2D samples as transformation of the angle
Xalpha = np.linspace(0, 2*np.pi, Nsa)
Xgrid= np.array([np.cos(Xalpha), np.sin(Xalpha)])
probs = [] # Vector with probabilities
for i in range(Nsa):
probs.append(np.exp(vMFd.vonMisesFisher_pdf_log(Xgrid[:,i],[mu,kappa]) ))
probs = np.array(probs)
probs = probs.reshape((probs.size,1)).T
# Plot it in polar coordinates
X1_w = (1 + probs) * np.cos(Xalpha)
X2_w = (1 + probs) * np.sin(Xalpha)
# print X1_w.shape, X2_w.shape
gl.plot(X1_w,X2_w,
alpha = 1, lw = 3, ls = "-.", legend = ["Kvmf(%i). pi:%0.2f"%(k, float(model_theta_list[indx][0][0,k]))])
gl.set_zoom(xlim = [-6,6], ylim = [-6,6], ax = ax1)
ax2 = gl.subplot2grid((1,2), (0,1), rowspan=1, colspan=1)
if (indx == 0):
gl.add_text(positionXY = [0.1,.5], text = r' Initilization Incomplete LogLike: %.2f'%(logl[0]),fontsize = 15)
pass
elif (indx >= 1):
gl.plot(range(1,np.array(logl).flatten()[1:].size +1),np.array(logl).flatten()[1:(indx+1)], ax = ax2,
legend = ["Iteration %i, Incom LL: %.2f"%(indx, logl[indx])], labels = ["Convergence of LL with generated data","Iterations","LL"], lw = 2)
gl.scatter(1, logl[1], lw = 2)
pt = 0.05
gl.set_zoom(xlim = [0,len(logl)], ylim = [logl[1] - (logl[-1]-logl[1])*pt,logl[-1] + (logl[-1]-logl[1])*pt], ax = ax2)
gl.subplots_adjust(left=.09, bottom=.10, right=.90, top=.95, wspace=.2, hspace=0.01)
gl.savefig(folder_images_gif +'gif_'+ str(indx) + '.png',
dpi = 100, sizeInches = [16, 8], close = "yes",bbox_inches = None)
gl.close("all")
train_config.embedding = False
eval_config.embedding = False
train_config.Y_cardinality = 2
eval_config.Y_cardinality= 2
eval_config.batch_size = 20
############# PATH WHERE TO SAVE AND LATER LOAD THE OPERATIONS AND PARAMETERS AND RESULTS #############
## TensorBoard for seeing the intermediate results !
#subprocess.Popen(["tensorboard","--logdir="+ save_path])
save_path += "nl:"+str(train_config.num_layers) +"_nh:" + str(train_config.hidden_size)
save_path += "_nt:" + str(train_config.num_steps) + "_Dx:" + str(train_config.X_dim) + "/"
utils.create_folder_if_needed(save_path)
#####################################################################################
#### TF graph for just creating the models. We create 3 models, one for train, validation
tf.reset_default_graph()
print ("BUILDING FIRST GRAPH")
#def change_random_seed(seed):
# global prng
# prng = np.random.RandomState(seed)
# tf.set_random_seed(seed)
# change_random_seed(global_random_seed)
build_Graph = 0
train_models = 0
test_models = 0
video_fotograms_folder = folder_images +"video_Bayesian1/"
video_fotograms_folder2 = folder_images +"video_Bayesian2/"
video_fotograms_folder3 = folder_images +"video_Bayesian3/"
video_fotograms_folder4 = folder_images +"video_Bayesian4/"
folder_model = "../models/pyTorch/Basic0_Bayesian_RNN/"
################ PLOTTING OPTIONS ################
create_video_training = 1
Step_video = 1
"""
###################### Setting up func ###########################
"""
## Windows and folder management
plt.close("all") # Close all previous Windows
ul.create_folder_if_needed(folder_images)
ul.create_folder_if_needed(video_fotograms_folder)
ul.create_folder_if_needed(video_fotograms_folder2)
ul.create_folder_if_needed(video_fotograms_folder3)
ul.create_folder_if_needed(video_fotograms_folder4)
ul.create_folder_if_needed(folder_model)
if(1):
ul.remove_files(video_fotograms_folder)
ul.remove_files(video_fotograms_folder2)
ul.remove_files(video_fotograms_folder3)
ul.remove_files(video_fotograms_folder4)
## Set up seeds for more reproducible results
np.random.seed(0)
torch.manual_seed(0)
def __init__(self, name = "IoTubes chart", source = None):
self.name = name;
##################### ###########################
self.source = source; # Sata structure from which we can obtain data
self.rt_sampling = None; # Periodic Call temporal to sample data from sources
self.rt_plotting = None; # Periodic Call temporal to plot the data, update chart
self.period_sampling = 0.150 #*11;
self.period_plotting = 0.500 #* 3;
############### Chart related elements ##################
self.fig = None; # Figure where we are plotting
self.data_axes = None; # Axes where we have the data
self.show_window = 300; # Number of samples to show in the window
self.size_buffer = 1000; # Maximum number of samples in the bufer ?
## Data Related
self.data_buffer = [] # Here we store all the aquited values
self.time_buffer = []
self.output_folder = "./IoTubes/"
self.images_folder = self.output_folder + "images/"
self.disk_folder = self.output_folder + "sensors/"
self.reports_folder = self.output_folder + "reports/"
self.output_file_index = 0; # Index to create several files
ul.create_folder_if_needed(self.output_folder);
##### Specific to the task ######
self.Monitor = Monitor( name = "PH", units = "pH", desired_value = 5.3, range_warning = 2.5, range_stop = 3.2);
self.Monitor_time = Monitor( name = "Time", units = "minutes", desired_value = 150, range_warning = 10, range_stop = 30);
self.mySFTP = FTP_lib.SFTP_class()
self.email_config = email_config;
######## DDBB data #######
self.SQL_DDBB_config = SQL_DDBB_config
self.time_now= dt.datetime.now();
## Assure uniqueness of id, the starting second when it was creted.
self.machine_ID = "ID12342"
self.piping_ID = "Piping_C7"
self.Responsibles = ["Carsten", "Ole P. Ness"];
# FDA Cleaning requirements
self.FDA_cleaning_procedure = "FDA53531984EX"
self.cleaning_ID = "CKJ" + self.machine_ID +"%i%i%i%i%i%i"%(self.time_now.year,self.time_now.month,self.time_now.day,self.time_now.hour,self.time_now.minute,self.time_now.second)
self.first_time_cleaningID = True;
self.plots_data = None
## Cleaning Report data
self.report = None
# Flag for the first time you plot.
self.first_plot_flag = True
## Data lock so that the sampling and the plotting do not access the data at the same time
self.data_lock = Lock()
# If the timer function creates different threads for the task, it might be we f**k up charts.
self.plot_lock = Lock() # Not really needed in the end because a Timer() only has a Thread.
def save_to_csv(symbol, dataCSV, file_dir="./storage/"):
ul.create_folder_if_needed(file_dir)
whole_path = file_dir + symbol + ".csv"
dataCSV.to_csv(whole_path, sep=',')
plot_predictions = 0
plot_evolution_loss = 0
create_video_training = 1
plot_weights = 0
Step_video = 1
"""
###################### Setting up func ###########################
"""
## Set up seeds for more reproducible results
np.random.seed(0)
torch.manual_seed(0)
torch.cuda.manual_seed(0)
## Windows and folder management
plt.close("all") # Close all previous Windows
ul.create_folder_if_needed(folder_images)
ul.create_folder_if_needed(video_fotograms_folder)
ul.create_folder_if_needed(video_fotograms_folder2)
ul.create_folder_if_needed(video_fotograms_folder3)
ul.create_folder_if_needed(video_fotograms_folder4)
ul.create_folder_if_needed(folder_model)
if (0):
ul.remove_files(video_fotograms_folder)
ul.remove_files(video_fotograms_folder2)
ul.remove_files(video_fotograms_folder3)
ul.remove_files(video_fotograms_folder4)
"""
######################## LOAD AND PROCESS THE DATA ########################
"""
# load data and make training set
perform_EM = 1 perform_HMM_after_EM = 0 final_clusters_graph = 1 responsibility_graph = 1 ########################################################################## ################# DATA OBTAINING ###################################### ########################################################################## ######## SELECT SOURCE ######## dataSource = "Google" # Hanseatic FxPro GCI Yahoo [storage_folder, info_folder, updates_folder] = ul.get_foldersData(source = dataSource) folder_images = "../pics/Trapying/EM_trading/" ul.create_folder_if_needed(folder_images) ######## SELECT SYMBOLS AND PERIODS ######## symbols = ["XAUUSD","Mad.ITX", "EURUSD"] symbols = ["Alcoa_Inc"] symbols = ["GooG", "Alcoa_Inc"] periods = [5] # Create porfolio and load data cmplist = DBl.read_NASDAQ_companies(whole_path = "../storage/Google/companylist.csv") cmplist.sort_values(by = ['MarketCap'], ascending=[0],inplace = True) symbolIDs = cmplist.iloc[0:30]["Symbol"].tolist() symbol_ID_indx1 = 0 symbol_ID_indx2 = 1
def process_hst(filedir, filetype="new"):
## This funciton reads a hst file and createas a .csv !!
if filedir == None:
print "Enter a valid filedir (-f)"
quit()
if filetype != "new" and filetype != "old":
print "Enter a valid filetype (valid options are old and new)"
quit()
########################################################################
################ FORMATTING OF THE NAME ############
#### WHERE TO STORE THE CSV !!
########################################################################
filename = filedir.split("/")[-1]
original_filename = filename
filename = filename.split(".hst")[0]
# print filename
## This is of the type : Alcatel-Luc1440.hst
## We detect the period
periods = ul.periods
flag_detected = 0
# Check in decreasing order of length
periods = periods[::-1]
# periods = periods[:]
for period in periods[::-1]:
period_str = str(period)
per_len = len(period_str)
if (filename[-per_len:] == period_str):
flag_detected = 1
print period
break
# If we did not detect it
if (flag_detected != 1):
return -1
filename = filename[:-per_len] + "_" + ul.period_dic[period] + '.csv'
read = 0
openTime = []
openPrice = []
lowPrice = []
highPrice = []
closePrice = []
volume = []
with open(filedir, 'rb') as f:
while True:
if read >= HEADER_SIZE:
if filetype == "old":
buf = f.read(OLD_FILE_STRUCTURE_SIZE)
read += OLD_FILE_STRUCTURE_SIZE
if filetype == "new":
buf = f.read(NEW_FILE_STRUCTURE_SIZE)
read += NEW_FILE_STRUCTURE_SIZE
if not buf:
break
if filetype == "old":
bar = struct.unpack("<iddddd", buf)
openTime.append(
time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(bar[0])))
openPrice.append(bar[1])
highPrice.append(bar[3])
lowPrice.append(bar[2])
closePrice.append(bar[4])
volume.append(bar[5])
if filetype == "new":
# print filedir
bar = struct.unpack("<Qddddqiq", buf)
try:
openTime.append(
time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(bar[0])))
except ValueError:
print "Error: " + filedir
return -1
openPrice.append(bar[1])
highPrice.append(bar[2])
lowPrice.append(bar[3])
closePrice.append(bar[4])
volume.append(bar[5])
else:
buf = f.read(HEADER_SIZE)
read += HEADER_SIZE
data = {
'Time': openTime,
'Open': openPrice,
'High': highPrice,
'Low': lowPrice,
'Close': closePrice,
'Volume': volume
}
result = pd.DataFrame.from_dict(data)
result = result.set_index('Time')
# print result
# print filename
## Filepath
filepath = filedir.split(original_filename)[0]
filepath = "/".join(filepath.split("/")[:-2]) + "/CSVS/"
filepath = filepath + ul.period_dic[period] + "/"
ul.create_folder_if_needed(filepath)
# print filepath
result.to_csv(filepath + filename) # , header = False
def generate_images_iterations_ll(Xs, mus, covs, Ks, myDManager, logl,
theta_list, model_theta_list,
folder_images_gif):
# os.remove(folder_images_gif) # Remove previous images if existing
"""
WARNING: MEANT FOR ONLY 3 Distributions due to the color RGB
"""
import shutil
ul.create_folder_if_needed(folder_images_gif)
shutil.rmtree(folder_images_gif)
ul.create_folder_if_needed(folder_images_gif)
######## Plot the original data #####
Xdata = np.concatenate(Xs, axis=1).T
colors = ["r", "b", "g"]
K_G, K_W, K_vMF = Ks
### FOR EACH ITERATION
for i in range(len(theta_list)): # theta_list
indx = i
gl.init_figure()
ax1 = gl.subplot2grid((1, 2), (0, 0), rowspan=1, colspan=1)
## Get the relative ll of the Gaussian denoising cluster.
ll = myDManager.pdf_log_K(Xdata, theta_list[indx])
N, K = ll.shape
# print ll.shape
for j in range(N): # For every sample
#TODO: Can this not be done without a for ?
# Normalize the probability of the sample being generated by the clusters
Marginal_xi_probability = gf.sum_logs(ll[j, :])
ll[j, :] = ll[j, :] - Marginal_xi_probability
ax1 = gl.scatter(
Xdata[j, 0],
Xdata[j, 1],
labels=[
'EM Evolution. Kg:' + str(K_G) + ', Kw:' + str(K_W) +
', K_vMF:' + str(K_vMF), "X1", "X2"
],
color=(np.exp(ll[j, 1]), np.exp(ll[j, 0]),
np.exp(ll[j, 2])), ### np.exp(ll[j,2])
alpha=1,
nf=0)
# Only doable if the clusters dont die
for k_c in myDManager.clusterk_to_Dname.keys():
k = myDManager.clusterk_to_thetak[k_c]
distribution_name = myDManager.clusterk_to_Dname[k_c] # G W
if (distribution_name == "Gaussian"):
## Plot the ecolution of the mu
#### Plot the Covariance of the clusters !
mean, w, h, theta = bMA.get_gaussian_ellipse_params(
mu=theta_list[indx][k][0],
Sigma=theta_list[indx][k][1],
Chi2val=2.4477)
r_ellipse = bMA.get_ellipse_points(mean, w, h, theta)
gl.plot(r_ellipse[:, 0],
r_ellipse[:, 1],
ax=ax1,
ls="-.",
lw=3,
AxesStyle="Normal2",
legend=[
"Kg(%i). pi:%0.2f" %
(k, float(model_theta_list[indx][0][0, k]))
])
elif (distribution_name == "Watson"):
#### Plot the pdf of the distributino !
## Distribution parameters for Watson
kappa = float(theta_list[indx][k][1])
mu = theta_list[-1][k][0]
Nsa = 1000
# Draw 2D samples as transformation of the angle
Xalpha = np.linspace(0, 2 * np.pi, Nsa)
Xgrid = np.array([np.cos(Xalpha), np.sin(Xalpha)])
probs = [] # Vector with probabilities
for i in range(Nsa):
probs.append(
np.exp(Wad.Watson_pdf_log(Xgrid[:, i], [mu, kappa])))
probs = np.array(probs)
# Plot it in polar coordinates
X1_w = (1 + probs) * np.cos(Xalpha)
X2_w = (1 + probs) * np.sin(Xalpha)
gl.plot(X1_w,
X2_w,
alpha=1,
lw=3,
ls="-.",
legend=[
"Kw(%i). pi:%0.2f" %
(k, float(model_theta_list[indx][0][0, k]))
])
elif (distribution_name == "vonMisesFisher"):
#### Plot the pdf of the distributino !
## Distribution parameters for Watson
kappa = float(theta_list[indx][k][1])
mu = theta_list[indx][k][0]
Nsa = 1000
# Draw 2D samples as transformation of the angle
Xalpha = np.linspace(0, 2 * np.pi, Nsa)
Xgrid = np.array([np.cos(Xalpha), np.sin(Xalpha)])
probs = [] # Vector with probabilities
for i in range(Nsa):
probs.append(
np.exp(
vMFd.vonMisesFisher_pdf_log(
Xgrid[:, i], [mu, kappa])))
probs = np.array(probs)
probs = probs.reshape((probs.size, 1)).T
# Plot it in polar coordinates
X1_w = (1 + probs) * np.cos(Xalpha)
X2_w = (1 + probs) * np.sin(Xalpha)
# print X1_w.shape, X2_w.shape
gl.plot(X1_w,
X2_w,
alpha=1,
lw=3,
ls="-.",
legend=[
"Kvmf(%i). pi:%0.2f" %
(k, float(model_theta_list[indx][0][0, k]))
])
gl.set_zoom(xlim=[-6, 6], ylim=[-6, 6], ax=ax1)
ax2 = gl.subplot2grid((1, 2), (0, 1), rowspan=1, colspan=1)
if (indx == 0):
gl.add_text(positionXY=[0.1, .5],
text=r' Initilization Incomplete LogLike: %.2f' %
(logl[0]),
fontsize=15)
pass
elif (indx >= 1):
gl.plot(
range(1,
np.array(logl).flatten()[1:].size + 1),
np.array(logl).flatten()[1:(indx + 1)],
ax=ax2,
legend=["Iteration %i, Incom LL: %.2f" % (indx, logl[indx])],
labels=[
"Convergence of LL with generated data", "Iterations", "LL"
],
lw=2)
gl.scatter(1, logl[1], lw=2)
pt = 0.05
gl.set_zoom(xlim=[0, len(logl)],
ylim=[
logl[1] - (logl[-1] - logl[1]) * pt,
logl[-1] + (logl[-1] - logl[1]) * pt
],
ax=ax2)
gl.subplots_adjust(left=.09,
bottom=.10,
right=.90,
top=.95,
wspace=.2,
hspace=0.01)
gl.savefig(folder_images_gif + 'gif_' + str(indx) + '.png',
dpi=100,
sizeInches=[16, 8],
close="yes",
bbox_inches=None)
gl.close("all")
X = np.array(data_df.loc[:,input_features_names])
Y = np.array(data_df.loc[:,'Target_clas']).reshape(-1,1)
## Normalize variables !!!
from sklearn import preprocessing
"""
Load the model and the scalers
"""
folder_model = "../models/"
folder_predictions = "../predictions/"
key_classifier = "LSVM" # QDA # GNB RF
ul.create_folder_if_needed(folder_predictions)
scaler_X = pkl.load_pickle(folder_model + "scaler_X" +".pkl")[0]
scaler_Y = pkl.load_pickle(folder_model + "scaler_Y" +".pkl")[0]
classifier = pkl.load_pickle(folder_model + key_classifier +".pkl")[0]
X = scaler_X.transform(X)
Y = scaler_Y.transform(Y)
Ypred = classifier.predict_proba(X)[:,1]
output_df = tut.get_output_df(dates_predictions,Ypred)
output_df = output_df[output_df.index > dt.datetime(2015,4,1)]
output_df.to_csv(folder_predictions + key_classifier + ".csv")
## Reload predictions for checking
loaded_predictions = pd.read_csv(folder_predictions + key_classifier + ".csv",
eval_config.embedding = False
train_config.Y_cardinality = 2
eval_config.Y_cardinality = 2
eval_config.batch_size = 20
############# PATH WHERE TO SAVE AND LATER LOAD THE OPERATIONS AND PARAMETERS AND RESULTS #############
## TensorBoard for seeing the intermediate results !
#subprocess.Popen(["tensorboard","--logdir="+ save_path])
save_path += "nl:" + str(train_config.num_layers) + "_nh:" + str(
train_config.hidden_size)
save_path += "_nt:" + str(train_config.num_steps) + "_Dx:" + str(
train_config.X_dim) + "/"
utils.create_folder_if_needed(save_path)
#####################################################################################
#### TF graph for just creating the models. We create 3 models, one for train, validation
tf.reset_default_graph()
print("BUILDING FIRST GRAPH")
#def change_random_seed(seed):
# global prng
# prng = np.random.RandomState(seed)
# tf.set_random_seed(seed)
# change_random_seed(global_random_seed)
build_Graph = 0
train_models = 0
test_models = 0
## Create dict of classifiers
cl_d = dict()
cl_d["LR"] = lr; cl_d["LDA"] = lda;cl_d["QDA"] = qda; cl_d["GNB"] = gnb;
cl_d["KNN"] = gknn; cl_d["Tree"] = gtree; cl_d["RF"] = rf; cl_d["ERF"] = ert;
cl_d["LSVM"] = gsvmr; cl_d["RFSVM"] = sbmrf;
# cl_d["PolySVM"] = gsvmp;
if (save_model_to_disk):
# We save the last model to disk using pickle !
# Lazy way but since non Deep Learning models do not require much
# memory in general, it is justified.
folder_model = "../models/"
key_classifier = "LSVM" # QDA # GNB RF
ul.create_folder_if_needed(folder_model)
classifier = cl_d[key_classifier]
pkl.store_pickle(folder_model + key_classifier +".pkl", [classifier])
pkl.store_pickle(folder_model + "scaler_X" +".pkl", [scaler_X])
pkl.store_pickle(folder_model + "scaler_Y" +".pkl", [scaler_Y])
"""
PLOT THE RESULTS
"""
if (plot_performance_all):
def get_class_rate(Y, Ypred):
return np.mean(np.equal(Y,Ypred))
Evaluate_Model_Results = 1 # Run the model through the dataset # Option train_validation_dataset = False # train = True, Validation = False bayesian_ensemble = False # Either Bayesian ensemble ## Plotting options database_lengths_plot = 0 database_question_type_plot = 0 model_performances_plot = 0 analyze_example_query = 0 trim_VB_model = True ## Folder folder_images = "../pics/Thesis/Eval_model_dataset/" images_prefix = "Baseline_" # We add this prefix to the image names to avoid collision ul.create_folder_if_needed(folder_images) """ #################### MODEL TO LOAD #################### Information to load the experiments """ #source_path = "../Baseline_model/results/bidaf/" #model_id = "2018-11-28 01:27:02.281959" #source_path = "../data_VB2/CV_bayesian_2/results/bidaf/" #model_id = "2018-12-06 19:58:11.727693" source_path = "../data_VB_1H/results/bidaf/" model_id = "2018-12-07 22:12:05.592914" # 2018-12-08 08:54:34.545417 # "2018-10-10 21:16:33.652684" # "2018-09-30 00:29:50.035864" # "2018-09-26 20:41:06.955179" # "2018-09-27 08:36:18.902846"# "2018-09-26 20:41:06.955179" epoch_i = -1 # Epoch to load
cf_a.VB_similarity_function = True
cf_a.VB_highway_layers = True
cf_a.VB_span_start_predictor_linear = True
cf_a.VB_span_end_predictor_linear = True
"""
############## Create folder directories ################
Informatrion about where to save the partial results
"""
time_now = dt.datetime.now()
root_folder = cf_a.results_root_folder
pickle_results_path = root_folder + "results/bidaf/" + str(
time_now) + "/training_logger/"
mode_file_path = root_folder + "results/bidaf/" + str(time_now) + "/models/"
ul.create_folder_if_needed(pickle_results_path)
ul.create_folder_if_needed(mode_file_path)
"""
#################### Initialize random generators ####################
This will help make results more reproducible but there are at least these sources of uncertainty:
- Different GPU's might have different random generators.
- Also the generators of GPU and CPU are different
- ELMO estates are not reinitialized per batch, so we wil always get somewhat different
results every time we execute them. It is recommend it to warm it up after loading
by propagating a few samples first.
- The CUDA operations are not deterministic
"""
random_seed = time_now.microsecond
np.random.seed(random_seed)
torch.manual_seed(random_seed)
torch.cuda.manual_seed(random_seed)
cl_d["KNN"] = gknn
cl_d["Tree"] = gtree
cl_d["RF"] = rf
cl_d["ERF"] = ert
cl_d["LSVM"] = gsvmr
cl_d["RFSVM"] = sbmrf
# cl_d["PolySVM"] = gsvmp;
if (save_model_to_disk):
# We save the last model to disk using pickle !
# Lazy way but since non Deep Learning models do not require much
# memory in general, it is justified.
folder_model = "../models/"
key_classifier = "LSVM" # QDA # GNB RF
ul.create_folder_if_needed(folder_model)
classifier = cl_d[key_classifier]
pkl.store_pickle(folder_model + key_classifier + ".pkl", [classifier])
pkl.store_pickle(folder_model + "scaler_X" + ".pkl", [scaler_X])
pkl.store_pickle(folder_model + "scaler_Y" + ".pkl", [scaler_Y])
"""
PLOT THE RESULTS
"""
if (plot_performance_all):
def get_class_rate(Y, Ypred):
return np.mean(np.equal(Y, Ypred))
def get_CE(Y, Ypred):
from graph_lib import gl
import mpmath as mpm
import utilities_lib as ul
# Import specific model libraries
from scipy import spatial
from matplotlib import pyplot as plt
from matplotlib import cm as cm
import pickle_lib as pkl
plt.close("all") # Close all previous Windows
################## Set the output folders ##################
folder_data = "./data/artificial/"
ul.create_folder_if_needed(folder_data)
plot_flag = 0
### Generation of the easy signal
def mean_function(tgrid, f1 = 1, f2 = 5, a1 = 0.4, a2 = 0.2,
phi2 = 2*np.pi/7, m = 0.1 ):
# This function outputs the function that we want to estimate.
"""
m = 0.1 # Slope of the linear trend
f1 = 1 # Frquency of first sinusoid
f2 = 5 # Frequency of second sinusoid
a1 = 0.4; # Amplitud of the first sinuoid
a2 = 0.2 # Amplitud of the second sinusoid
phi2 = 2*np.pi/7 # Phase shifting of the second sinuoid
