def savefig( self, file_dir="./image.png", bbox_inches='tight', sizeInches=[], # The size in inches as a list close=False, # If we close the figure once saved dpi=100): # Density of pixels !! Same image but more cuality ! Pixels try: folders = file_dir.split("/") folders.pop(-1) path = "/".join(folders) ul.create_folder_if_needed(path) except: pass ## Function to save the current figure in the desired format ## Important !! Both dpi and sizeInches affect the number of pixels of the image # dpi: It is just for quality, same graph but more quality. # sizeInches: You change the proportions of the window. The fontsize and # thickness of lines is the same. So as the graph grows, they look smaller. F = self.fig # ?? NO work ? # F = pylab.gcf() Winches, Hinches = F.get_size_inches() # 8,6 # print Winches, Hinches if (len(sizeInches) > 0): F.set_size_inches((sizeInches[0], sizeInches[1])) self.fig.savefig(file_dir, bbox_inches=bbox_inches, dpi=dpi) # Transform back F.set_size_inches((Winches, Hinches)) if (close == True): plt.close()
def save_to_csv(self, file_dir="./storage/"): ul.create_folder_if_needed(file_dir) whole_path = file_dir + ul.period_dic[ self.period] + "/" + self.symbol + "_" + ul.period_dic[ self.period] + ".csv" ul.create_folder_if_needed(file_dir + ul.period_dic[self.period] + "/") self.TD.to_csv(whole_path, sep=',')
def save_to_csv(self, file_dir = "./storage/", force = False): # This function saves the TD to a csv file if (self.trimmed == True and force == False): print ("You cannot save the file since you trimmed it, Use force = True") else: ul.create_folder_if_needed(file_dir) whole_path = file_dir + ul.period_dic[self.period] + "/" + \ self.symbolID + "_" + ul.period_dic[self.period] + ".csv" ul.create_folder_if_needed(file_dir + ul.period_dic[self.period] + "/") self.TD.to_csv(whole_path, sep=',')
def get_dir(self, remotedir, localdir = "./", recursive = False, preserve_mtime=False): """ Function that downloads the specified file from the server into the local file name provided. If Recursive it will download all subfolders. """ self.connect() ul.create_folder_if_needed(localdir) if(recursive): self.sftp.get_r(remotedir, localdir, preserve_mtime=preserve_mtime) else: self.sftp.get_d(remotedir, localdir, preserve_mtime=preserve_mtime) self.close()
def get_dir(self, remotedir, localdir="./", recursive=False, preserve_mtime=False): """ Function that downloads the specified file from the server into the local file name provided. If Recursive it will download all subfolders. """ self.connect() ul.create_folder_if_needed(localdir) if (recursive): self.sftp.get_r(remotedir, localdir, preserve_mtime=preserve_mtime) else: self.sftp.get_d(remotedir, localdir, preserve_mtime=preserve_mtime) self.close()
## Import standard libraries import tensorflow as tf # Import libraries from graph_lib import gl import numpy as np import matplotlib.pyplot as plt import utilities_lib as ul import tf_general_model1 as tfm1 plt.close("all") tb_folder = "../TensorBoard/Examples/3_LR_copmlete_Model/" variables_folder = "../TensorBoard/Examples/3_LR_copmlete_Model/Variables/" folder_images = "../pics/Trapying/TensorBoard_Examples/" ul.create_folder_if_needed(tb_folder) ul.create_folder_if_needed(variables_folder) ul.create_folder_if_needed(folder_images) ########################################################################## ################ Load the Data ######################################## ########################################################################## X_data_tr = np.array([1.2,1.8,3.2,3.9,4.7,6.4, 4.4]) Y_data_tr = np.array([0, -1,-2,-3,-4, -5, -9]) X_data_val = np.array([8,9,10,11]) Y_data_val = np.array([-7.1, -7.9,-9.2,-10.1]) #### Hyperparameters ####
## Import TF creared import BBB_LSTM_configs as Bconf import BBB_LSTM_Model as BM import Variational_inferences_lib as VI import util_autoparallel as util import pickle_lib as pkl from graph_lib import gl import numpy as np import matplotlib.pyplot as plt import utilities_lib as ul tb_folder = "../TensorBoard/Examples/1_basic/" ul.create_folder_if_needed(tb_folder) ## Destroy the graph tf.reset_default_graph() ## Build the graph ## We need a global initializer a = tf.constant(5.0) b = tf.constant(6.9) c = tf.multiply(a,b) # We can use normal operators over tensors d = a * c ## Create a Session object to lauch the graph
eta_KL) + "/" video_fotograms_folder_weights = folder_images + "Weights/" video_fotograms_folder_training = folder_images + "Training/" folder_model = "../models/pyTorch/BasicExamples_Bayesian/MLP/" """ ###################### Setting up func ########################### """ ## Set up seeds for more reproducible results random_seed = 0 #time_now.microsecond np.random.seed(random_seed) torch.manual_seed(random_seed) torch.cuda.manual_seed(random_seed) ## Windows and folder management plt.close("all") # Close all previous Windows ul.create_folder_if_needed(folder_images) ul.create_folder_if_needed(video_fotograms_folder_weights) ul.create_folder_if_needed(video_fotograms_folder_training) ul.create_folder_if_needed(folder_model) if (create_video_training): ul.remove_files(video_fotograms_folder_weights) ul.remove_files(video_fotograms_folder_training) """ ######################## LOAD AND PROCESS THE DATA ######################## """ ## Original Numpy data. The last data point is an outlier !! if (linear_data): [X_data_tr, Y_data_tr, X_data_val, Y_data_val] = dl.get_linear_dataset(outlier=False)
folder_images = "../pics/Pytorch/BasicExamples_Bayesian/MLP_classification/"+ str(eta_KL) + "/" video_fotograms_folder_weights = folder_images +"Weights/" video_fotograms_folder_training = folder_images +"Training/" folder_model = "../models/pyTorch/BasicExamples_Bayesian/MLP/" """ ###################### Setting up func ########################### """ ## Set up seeds for more reproducible results random_seed = 0 #time_now.microsecond np.random.seed(random_seed) torch.manual_seed(random_seed) torch.cuda.manual_seed(random_seed) ## Windows and folder management plt.close("all") # Close all previous Windows ul.create_folder_if_needed(folder_images) ul.create_folder_if_needed(video_fotograms_folder_weights) ul.create_folder_if_needed(video_fotograms_folder_training) ul.create_folder_if_needed(folder_model) if(create_video_training): ul.remove_files(video_fotograms_folder_weights) ul.remove_files(video_fotograms_folder_training) """ ######################## LOAD AND PROCESS THE DATA ######################## """ ## Original Numpy data. The last data point is an outlier !! [X_data_tr, Y_data_tr, X_data_val,Y_data_val, enc] = dl.get_toy_classification_data(n_samples=100, centers=3, n_features=2) xgrid_real_func = None; ygrid_real_func = None;
def generate_images_iterations_ll(Xs,mus,covs, Ks ,myDManager, logl,theta_list,model_theta_list,folder_images_gif): # os.remove(folder_images_gif) # Remove previous images if existing """ WARNING: MEANT FOR ONLY 3 Distributions due to the color RGB """ import shutil ul.create_folder_if_needed(folder_images_gif) shutil.rmtree(folder_images_gif) ul.create_folder_if_needed(folder_images_gif) ######## Plot the original data ##### Xdata = np.concatenate(Xs,axis = 1).T colors = ["r","b","g"] K_G,K_W,K_vMF = Ks ### FOR EACH ITERATION for i in range(len(theta_list)): # theta_list indx = i gl.init_figure() ax1 = gl.subplot2grid((1,2), (0,0), rowspan=1, colspan=1) ## Get the relative ll of the Gaussian denoising cluster. ll = myDManager.pdf_log_K(Xdata,theta_list[indx]) N,K = ll.shape # print ll.shape for j in range(N): # For every sample #TODO: Can this not be done without a for ? # Normalize the probability of the sample being generated by the clusters Marginal_xi_probability = gf.sum_logs(ll[j,:]) ll[j,:] = ll[j,:]- Marginal_xi_probability ax1 = gl.scatter(Xdata[j,0],Xdata[j,1], labels = ['EM Evolution. Kg:'+str(K_G)+ ', Kw:' + str(K_W) + ', K_vMF:' + str(K_vMF), "X1","X2"], color = (np.exp(ll[j,1]), np.exp(ll[j,0]), np.exp(ll[j,2])) , ### np.exp(ll[j,2]) alpha = 1, nf = 0) # Only doable if the clusters dont die for k_c in myDManager.clusterk_to_Dname.keys(): k = myDManager.clusterk_to_thetak[k_c] distribution_name = myDManager.clusterk_to_Dname[k_c] # G W if (distribution_name == "Gaussian"): ## Plot the ecolution of the mu #### Plot the Covariance of the clusters ! mean,w,h,theta = bMA.get_gaussian_ellipse_params( mu = theta_list[indx][k][0], Sigma = theta_list[indx][k][1], Chi2val = 2.4477) r_ellipse = bMA.get_ellipse_points(mean,w,h,theta) gl.plot(r_ellipse[:,0], r_ellipse[:,1], ax = ax1, ls = "-.", lw = 3, AxesStyle = "Normal2", legend = ["Kg(%i). pi:%0.2f"%(k, float(model_theta_list[indx][0][0,k]))]) elif(distribution_name == "Watson"): #### Plot the pdf of the distributino ! ## Distribution parameters for Watson kappa = float(theta_list[indx][k][1]); mu = theta_list[-1][k][0] Nsa = 1000 # Draw 2D samples as transformation of the angle Xalpha = np.linspace(0, 2*np.pi, Nsa) Xgrid= np.array([np.cos(Xalpha), np.sin(Xalpha)]) probs = [] # Vector with probabilities for i in range(Nsa): probs.append(np.exp(Wad.Watson_pdf_log(Xgrid[:,i],[mu,kappa]) )) probs = np.array(probs) # Plot it in polar coordinates X1_w = (1 + probs) * np.cos(Xalpha) X2_w = (1 + probs) * np.sin(Xalpha) gl.plot(X1_w,X2_w, alpha = 1, lw = 3, ls = "-.", legend = ["Kw(%i). pi:%0.2f"%(k, float(model_theta_list[indx][0][0,k]))]) elif(distribution_name == "vonMisesFisher"): #### Plot the pdf of the distributino ! ## Distribution parameters for Watson kappa = float(theta_list[indx][k][1]); mu = theta_list[indx][k][0] Nsa = 1000 # Draw 2D samples as transformation of the angle Xalpha = np.linspace(0, 2*np.pi, Nsa) Xgrid= np.array([np.cos(Xalpha), np.sin(Xalpha)]) probs = [] # Vector with probabilities for i in range(Nsa): probs.append(np.exp(vMFd.vonMisesFisher_pdf_log(Xgrid[:,i],[mu,kappa]) )) probs = np.array(probs) probs = probs.reshape((probs.size,1)).T # Plot it in polar coordinates X1_w = (1 + probs) * np.cos(Xalpha) X2_w = (1 + probs) * np.sin(Xalpha) # print X1_w.shape, X2_w.shape gl.plot(X1_w,X2_w, alpha = 1, lw = 3, ls = "-.", legend = ["Kvmf(%i). pi:%0.2f"%(k, float(model_theta_list[indx][0][0,k]))]) gl.set_zoom(xlim = [-6,6], ylim = [-6,6], ax = ax1) ax2 = gl.subplot2grid((1,2), (0,1), rowspan=1, colspan=1) if (indx == 0): gl.add_text(positionXY = [0.1,.5], text = r' Initilization Incomplete LogLike: %.2f'%(logl[0]),fontsize = 15) pass elif (indx >= 1): gl.plot(range(1,np.array(logl).flatten()[1:].size +1),np.array(logl).flatten()[1:(indx+1)], ax = ax2, legend = ["Iteration %i, Incom LL: %.2f"%(indx, logl[indx])], labels = ["Convergence of LL with generated data","Iterations","LL"], lw = 2) gl.scatter(1, logl[1], lw = 2) pt = 0.05 gl.set_zoom(xlim = [0,len(logl)], ylim = [logl[1] - (logl[-1]-logl[1])*pt,logl[-1] + (logl[-1]-logl[1])*pt], ax = ax2) gl.subplots_adjust(left=.09, bottom=.10, right=.90, top=.95, wspace=.2, hspace=0.01) gl.savefig(folder_images_gif +'gif_'+ str(indx) + '.png', dpi = 100, sizeInches = [16, 8], close = "yes",bbox_inches = None) gl.close("all")
train_config.embedding = False eval_config.embedding = False train_config.Y_cardinality = 2 eval_config.Y_cardinality= 2 eval_config.batch_size = 20 ############# PATH WHERE TO SAVE AND LATER LOAD THE OPERATIONS AND PARAMETERS AND RESULTS ############# ## TensorBoard for seeing the intermediate results ! #subprocess.Popen(["tensorboard","--logdir="+ save_path]) save_path += "nl:"+str(train_config.num_layers) +"_nh:" + str(train_config.hidden_size) save_path += "_nt:" + str(train_config.num_steps) + "_Dx:" + str(train_config.X_dim) + "/" utils.create_folder_if_needed(save_path) ##################################################################################### #### TF graph for just creating the models. We create 3 models, one for train, validation tf.reset_default_graph() print ("BUILDING FIRST GRAPH") #def change_random_seed(seed): # global prng # prng = np.random.RandomState(seed) # tf.set_random_seed(seed) # change_random_seed(global_random_seed) build_Graph = 0 train_models = 0 test_models = 0
video_fotograms_folder = folder_images +"video_Bayesian1/" video_fotograms_folder2 = folder_images +"video_Bayesian2/" video_fotograms_folder3 = folder_images +"video_Bayesian3/" video_fotograms_folder4 = folder_images +"video_Bayesian4/" folder_model = "../models/pyTorch/Basic0_Bayesian_RNN/" ################ PLOTTING OPTIONS ################ create_video_training = 1 Step_video = 1 """ ###################### Setting up func ########################### """ ## Windows and folder management plt.close("all") # Close all previous Windows ul.create_folder_if_needed(folder_images) ul.create_folder_if_needed(video_fotograms_folder) ul.create_folder_if_needed(video_fotograms_folder2) ul.create_folder_if_needed(video_fotograms_folder3) ul.create_folder_if_needed(video_fotograms_folder4) ul.create_folder_if_needed(folder_model) if(1): ul.remove_files(video_fotograms_folder) ul.remove_files(video_fotograms_folder2) ul.remove_files(video_fotograms_folder3) ul.remove_files(video_fotograms_folder4) ## Set up seeds for more reproducible results np.random.seed(0) torch.manual_seed(0)
def __init__(self, name = "IoTubes chart", source = None): self.name = name; ##################### ########################### self.source = source; # Sata structure from which we can obtain data self.rt_sampling = None; # Periodic Call temporal to sample data from sources self.rt_plotting = None; # Periodic Call temporal to plot the data, update chart self.period_sampling = 0.150 #*11; self.period_plotting = 0.500 #* 3; ############### Chart related elements ################## self.fig = None; # Figure where we are plotting self.data_axes = None; # Axes where we have the data self.show_window = 300; # Number of samples to show in the window self.size_buffer = 1000; # Maximum number of samples in the bufer ? ## Data Related self.data_buffer = [] # Here we store all the aquited values self.time_buffer = [] self.output_folder = "./IoTubes/" self.images_folder = self.output_folder + "images/" self.disk_folder = self.output_folder + "sensors/" self.reports_folder = self.output_folder + "reports/" self.output_file_index = 0; # Index to create several files ul.create_folder_if_needed(self.output_folder); ##### Specific to the task ###### self.Monitor = Monitor( name = "PH", units = "pH", desired_value = 5.3, range_warning = 2.5, range_stop = 3.2); self.Monitor_time = Monitor( name = "Time", units = "minutes", desired_value = 150, range_warning = 10, range_stop = 30); self.mySFTP = FTP_lib.SFTP_class() self.email_config = email_config; ######## DDBB data ####### self.SQL_DDBB_config = SQL_DDBB_config self.time_now= dt.datetime.now(); ## Assure uniqueness of id, the starting second when it was creted. self.machine_ID = "ID12342" self.piping_ID = "Piping_C7" self.Responsibles = ["Carsten", "Ole P. Ness"]; # FDA Cleaning requirements self.FDA_cleaning_procedure = "FDA53531984EX" self.cleaning_ID = "CKJ" + self.machine_ID +"%i%i%i%i%i%i"%(self.time_now.year,self.time_now.month,self.time_now.day,self.time_now.hour,self.time_now.minute,self.time_now.second) self.first_time_cleaningID = True; self.plots_data = None ## Cleaning Report data self.report = None # Flag for the first time you plot. self.first_plot_flag = True ## Data lock so that the sampling and the plotting do not access the data at the same time self.data_lock = Lock() # If the timer function creates different threads for the task, it might be we f**k up charts. self.plot_lock = Lock() # Not really needed in the end because a Timer() only has a Thread.
def save_to_csv(symbol, dataCSV, file_dir="./storage/"): ul.create_folder_if_needed(file_dir) whole_path = file_dir + symbol + ".csv" dataCSV.to_csv(whole_path, sep=',')
plot_predictions = 0 plot_evolution_loss = 0 create_video_training = 1 plot_weights = 0 Step_video = 1 """ ###################### Setting up func ########################### """ ## Set up seeds for more reproducible results np.random.seed(0) torch.manual_seed(0) torch.cuda.manual_seed(0) ## Windows and folder management plt.close("all") # Close all previous Windows ul.create_folder_if_needed(folder_images) ul.create_folder_if_needed(video_fotograms_folder) ul.create_folder_if_needed(video_fotograms_folder2) ul.create_folder_if_needed(video_fotograms_folder3) ul.create_folder_if_needed(video_fotograms_folder4) ul.create_folder_if_needed(folder_model) if (0): ul.remove_files(video_fotograms_folder) ul.remove_files(video_fotograms_folder2) ul.remove_files(video_fotograms_folder3) ul.remove_files(video_fotograms_folder4) """ ######################## LOAD AND PROCESS THE DATA ######################## """ # load data and make training set
perform_EM = 1 perform_HMM_after_EM = 0 final_clusters_graph = 1 responsibility_graph = 1 ########################################################################## ################# DATA OBTAINING ###################################### ########################################################################## ######## SELECT SOURCE ######## dataSource = "Google" # Hanseatic FxPro GCI Yahoo [storage_folder, info_folder, updates_folder] = ul.get_foldersData(source = dataSource) folder_images = "../pics/Trapying/EM_trading/" ul.create_folder_if_needed(folder_images) ######## SELECT SYMBOLS AND PERIODS ######## symbols = ["XAUUSD","Mad.ITX", "EURUSD"] symbols = ["Alcoa_Inc"] symbols = ["GooG", "Alcoa_Inc"] periods = [5] # Create porfolio and load data cmplist = DBl.read_NASDAQ_companies(whole_path = "../storage/Google/companylist.csv") cmplist.sort_values(by = ['MarketCap'], ascending=[0],inplace = True) symbolIDs = cmplist.iloc[0:30]["Symbol"].tolist() symbol_ID_indx1 = 0 symbol_ID_indx2 = 1
def process_hst(filedir, filetype="new"): ## This funciton reads a hst file and createas a .csv !! if filedir == None: print "Enter a valid filedir (-f)" quit() if filetype != "new" and filetype != "old": print "Enter a valid filetype (valid options are old and new)" quit() ######################################################################## ################ FORMATTING OF THE NAME ############ #### WHERE TO STORE THE CSV !! ######################################################################## filename = filedir.split("/")[-1] original_filename = filename filename = filename.split(".hst")[0] # print filename ## This is of the type : Alcatel-Luc1440.hst ## We detect the period periods = ul.periods flag_detected = 0 # Check in decreasing order of length periods = periods[::-1] # periods = periods[:] for period in periods[::-1]: period_str = str(period) per_len = len(period_str) if (filename[-per_len:] == period_str): flag_detected = 1 print period break # If we did not detect it if (flag_detected != 1): return -1 filename = filename[:-per_len] + "_" + ul.period_dic[period] + '.csv' read = 0 openTime = [] openPrice = [] lowPrice = [] highPrice = [] closePrice = [] volume = [] with open(filedir, 'rb') as f: while True: if read >= HEADER_SIZE: if filetype == "old": buf = f.read(OLD_FILE_STRUCTURE_SIZE) read += OLD_FILE_STRUCTURE_SIZE if filetype == "new": buf = f.read(NEW_FILE_STRUCTURE_SIZE) read += NEW_FILE_STRUCTURE_SIZE if not buf: break if filetype == "old": bar = struct.unpack("<iddddd", buf) openTime.append( time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(bar[0]))) openPrice.append(bar[1]) highPrice.append(bar[3]) lowPrice.append(bar[2]) closePrice.append(bar[4]) volume.append(bar[5]) if filetype == "new": # print filedir bar = struct.unpack("<Qddddqiq", buf) try: openTime.append( time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(bar[0]))) except ValueError: print "Error: " + filedir return -1 openPrice.append(bar[1]) highPrice.append(bar[2]) lowPrice.append(bar[3]) closePrice.append(bar[4]) volume.append(bar[5]) else: buf = f.read(HEADER_SIZE) read += HEADER_SIZE data = { 'Time': openTime, 'Open': openPrice, 'High': highPrice, 'Low': lowPrice, 'Close': closePrice, 'Volume': volume } result = pd.DataFrame.from_dict(data) result = result.set_index('Time') # print result # print filename ## Filepath filepath = filedir.split(original_filename)[0] filepath = "/".join(filepath.split("/")[:-2]) + "/CSVS/" filepath = filepath + ul.period_dic[period] + "/" ul.create_folder_if_needed(filepath) # print filepath result.to_csv(filepath + filename) # , header = False
def generate_images_iterations_ll(Xs, mus, covs, Ks, myDManager, logl, theta_list, model_theta_list, folder_images_gif): # os.remove(folder_images_gif) # Remove previous images if existing """ WARNING: MEANT FOR ONLY 3 Distributions due to the color RGB """ import shutil ul.create_folder_if_needed(folder_images_gif) shutil.rmtree(folder_images_gif) ul.create_folder_if_needed(folder_images_gif) ######## Plot the original data ##### Xdata = np.concatenate(Xs, axis=1).T colors = ["r", "b", "g"] K_G, K_W, K_vMF = Ks ### FOR EACH ITERATION for i in range(len(theta_list)): # theta_list indx = i gl.init_figure() ax1 = gl.subplot2grid((1, 2), (0, 0), rowspan=1, colspan=1) ## Get the relative ll of the Gaussian denoising cluster. ll = myDManager.pdf_log_K(Xdata, theta_list[indx]) N, K = ll.shape # print ll.shape for j in range(N): # For every sample #TODO: Can this not be done without a for ? # Normalize the probability of the sample being generated by the clusters Marginal_xi_probability = gf.sum_logs(ll[j, :]) ll[j, :] = ll[j, :] - Marginal_xi_probability ax1 = gl.scatter( Xdata[j, 0], Xdata[j, 1], labels=[ 'EM Evolution. Kg:' + str(K_G) + ', Kw:' + str(K_W) + ', K_vMF:' + str(K_vMF), "X1", "X2" ], color=(np.exp(ll[j, 1]), np.exp(ll[j, 0]), np.exp(ll[j, 2])), ### np.exp(ll[j,2]) alpha=1, nf=0) # Only doable if the clusters dont die for k_c in myDManager.clusterk_to_Dname.keys(): k = myDManager.clusterk_to_thetak[k_c] distribution_name = myDManager.clusterk_to_Dname[k_c] # G W if (distribution_name == "Gaussian"): ## Plot the ecolution of the mu #### Plot the Covariance of the clusters ! mean, w, h, theta = bMA.get_gaussian_ellipse_params( mu=theta_list[indx][k][0], Sigma=theta_list[indx][k][1], Chi2val=2.4477) r_ellipse = bMA.get_ellipse_points(mean, w, h, theta) gl.plot(r_ellipse[:, 0], r_ellipse[:, 1], ax=ax1, ls="-.", lw=3, AxesStyle="Normal2", legend=[ "Kg(%i). pi:%0.2f" % (k, float(model_theta_list[indx][0][0, k])) ]) elif (distribution_name == "Watson"): #### Plot the pdf of the distributino ! ## Distribution parameters for Watson kappa = float(theta_list[indx][k][1]) mu = theta_list[-1][k][0] Nsa = 1000 # Draw 2D samples as transformation of the angle Xalpha = np.linspace(0, 2 * np.pi, Nsa) Xgrid = np.array([np.cos(Xalpha), np.sin(Xalpha)]) probs = [] # Vector with probabilities for i in range(Nsa): probs.append( np.exp(Wad.Watson_pdf_log(Xgrid[:, i], [mu, kappa]))) probs = np.array(probs) # Plot it in polar coordinates X1_w = (1 + probs) * np.cos(Xalpha) X2_w = (1 + probs) * np.sin(Xalpha) gl.plot(X1_w, X2_w, alpha=1, lw=3, ls="-.", legend=[ "Kw(%i). pi:%0.2f" % (k, float(model_theta_list[indx][0][0, k])) ]) elif (distribution_name == "vonMisesFisher"): #### Plot the pdf of the distributino ! ## Distribution parameters for Watson kappa = float(theta_list[indx][k][1]) mu = theta_list[indx][k][0] Nsa = 1000 # Draw 2D samples as transformation of the angle Xalpha = np.linspace(0, 2 * np.pi, Nsa) Xgrid = np.array([np.cos(Xalpha), np.sin(Xalpha)]) probs = [] # Vector with probabilities for i in range(Nsa): probs.append( np.exp( vMFd.vonMisesFisher_pdf_log( Xgrid[:, i], [mu, kappa]))) probs = np.array(probs) probs = probs.reshape((probs.size, 1)).T # Plot it in polar coordinates X1_w = (1 + probs) * np.cos(Xalpha) X2_w = (1 + probs) * np.sin(Xalpha) # print X1_w.shape, X2_w.shape gl.plot(X1_w, X2_w, alpha=1, lw=3, ls="-.", legend=[ "Kvmf(%i). pi:%0.2f" % (k, float(model_theta_list[indx][0][0, k])) ]) gl.set_zoom(xlim=[-6, 6], ylim=[-6, 6], ax=ax1) ax2 = gl.subplot2grid((1, 2), (0, 1), rowspan=1, colspan=1) if (indx == 0): gl.add_text(positionXY=[0.1, .5], text=r' Initilization Incomplete LogLike: %.2f' % (logl[0]), fontsize=15) pass elif (indx >= 1): gl.plot( range(1, np.array(logl).flatten()[1:].size + 1), np.array(logl).flatten()[1:(indx + 1)], ax=ax2, legend=["Iteration %i, Incom LL: %.2f" % (indx, logl[indx])], labels=[ "Convergence of LL with generated data", "Iterations", "LL" ], lw=2) gl.scatter(1, logl[1], lw=2) pt = 0.05 gl.set_zoom(xlim=[0, len(logl)], ylim=[ logl[1] - (logl[-1] - logl[1]) * pt, logl[-1] + (logl[-1] - logl[1]) * pt ], ax=ax2) gl.subplots_adjust(left=.09, bottom=.10, right=.90, top=.95, wspace=.2, hspace=0.01) gl.savefig(folder_images_gif + 'gif_' + str(indx) + '.png', dpi=100, sizeInches=[16, 8], close="yes", bbox_inches=None) gl.close("all")
X = np.array(data_df.loc[:,input_features_names]) Y = np.array(data_df.loc[:,'Target_clas']).reshape(-1,1) ## Normalize variables !!! from sklearn import preprocessing """ Load the model and the scalers """ folder_model = "../models/" folder_predictions = "../predictions/" key_classifier = "LSVM" # QDA # GNB RF ul.create_folder_if_needed(folder_predictions) scaler_X = pkl.load_pickle(folder_model + "scaler_X" +".pkl")[0] scaler_Y = pkl.load_pickle(folder_model + "scaler_Y" +".pkl")[0] classifier = pkl.load_pickle(folder_model + key_classifier +".pkl")[0] X = scaler_X.transform(X) Y = scaler_Y.transform(Y) Ypred = classifier.predict_proba(X)[:,1] output_df = tut.get_output_df(dates_predictions,Ypred) output_df = output_df[output_df.index > dt.datetime(2015,4,1)] output_df.to_csv(folder_predictions + key_classifier + ".csv") ## Reload predictions for checking loaded_predictions = pd.read_csv(folder_predictions + key_classifier + ".csv",
eval_config.embedding = False train_config.Y_cardinality = 2 eval_config.Y_cardinality = 2 eval_config.batch_size = 20 ############# PATH WHERE TO SAVE AND LATER LOAD THE OPERATIONS AND PARAMETERS AND RESULTS ############# ## TensorBoard for seeing the intermediate results ! #subprocess.Popen(["tensorboard","--logdir="+ save_path]) save_path += "nl:" + str(train_config.num_layers) + "_nh:" + str( train_config.hidden_size) save_path += "_nt:" + str(train_config.num_steps) + "_Dx:" + str( train_config.X_dim) + "/" utils.create_folder_if_needed(save_path) ##################################################################################### #### TF graph for just creating the models. We create 3 models, one for train, validation tf.reset_default_graph() print("BUILDING FIRST GRAPH") #def change_random_seed(seed): # global prng # prng = np.random.RandomState(seed) # tf.set_random_seed(seed) # change_random_seed(global_random_seed) build_Graph = 0 train_models = 0 test_models = 0
## Create dict of classifiers cl_d = dict() cl_d["LR"] = lr; cl_d["LDA"] = lda;cl_d["QDA"] = qda; cl_d["GNB"] = gnb; cl_d["KNN"] = gknn; cl_d["Tree"] = gtree; cl_d["RF"] = rf; cl_d["ERF"] = ert; cl_d["LSVM"] = gsvmr; cl_d["RFSVM"] = sbmrf; # cl_d["PolySVM"] = gsvmp; if (save_model_to_disk): # We save the last model to disk using pickle ! # Lazy way but since non Deep Learning models do not require much # memory in general, it is justified. folder_model = "../models/" key_classifier = "LSVM" # QDA # GNB RF ul.create_folder_if_needed(folder_model) classifier = cl_d[key_classifier] pkl.store_pickle(folder_model + key_classifier +".pkl", [classifier]) pkl.store_pickle(folder_model + "scaler_X" +".pkl", [scaler_X]) pkl.store_pickle(folder_model + "scaler_Y" +".pkl", [scaler_Y]) """ PLOT THE RESULTS """ if (plot_performance_all): def get_class_rate(Y, Ypred): return np.mean(np.equal(Y,Ypred))
Evaluate_Model_Results = 1 # Run the model through the dataset # Option train_validation_dataset = False # train = True, Validation = False bayesian_ensemble = False # Either Bayesian ensemble ## Plotting options database_lengths_plot = 0 database_question_type_plot = 0 model_performances_plot = 0 analyze_example_query = 0 trim_VB_model = True ## Folder folder_images = "../pics/Thesis/Eval_model_dataset/" images_prefix = "Baseline_" # We add this prefix to the image names to avoid collision ul.create_folder_if_needed(folder_images) """ #################### MODEL TO LOAD #################### Information to load the experiments """ #source_path = "../Baseline_model/results/bidaf/" #model_id = "2018-11-28 01:27:02.281959" #source_path = "../data_VB2/CV_bayesian_2/results/bidaf/" #model_id = "2018-12-06 19:58:11.727693" source_path = "../data_VB_1H/results/bidaf/" model_id = "2018-12-07 22:12:05.592914" # 2018-12-08 08:54:34.545417 # "2018-10-10 21:16:33.652684" # "2018-09-30 00:29:50.035864" # "2018-09-26 20:41:06.955179" # "2018-09-27 08:36:18.902846"# "2018-09-26 20:41:06.955179" epoch_i = -1 # Epoch to load
cf_a.VB_similarity_function = True cf_a.VB_highway_layers = True cf_a.VB_span_start_predictor_linear = True cf_a.VB_span_end_predictor_linear = True """ ############## Create folder directories ################ Informatrion about where to save the partial results """ time_now = dt.datetime.now() root_folder = cf_a.results_root_folder pickle_results_path = root_folder + "results/bidaf/" + str( time_now) + "/training_logger/" mode_file_path = root_folder + "results/bidaf/" + str(time_now) + "/models/" ul.create_folder_if_needed(pickle_results_path) ul.create_folder_if_needed(mode_file_path) """ #################### Initialize random generators #################### This will help make results more reproducible but there are at least these sources of uncertainty: - Different GPU's might have different random generators. - Also the generators of GPU and CPU are different - ELMO estates are not reinitialized per batch, so we wil always get somewhat different results every time we execute them. It is recommend it to warm it up after loading by propagating a few samples first. - The CUDA operations are not deterministic """ random_seed = time_now.microsecond np.random.seed(random_seed) torch.manual_seed(random_seed) torch.cuda.manual_seed(random_seed)
cl_d["KNN"] = gknn cl_d["Tree"] = gtree cl_d["RF"] = rf cl_d["ERF"] = ert cl_d["LSVM"] = gsvmr cl_d["RFSVM"] = sbmrf # cl_d["PolySVM"] = gsvmp; if (save_model_to_disk): # We save the last model to disk using pickle ! # Lazy way but since non Deep Learning models do not require much # memory in general, it is justified. folder_model = "../models/" key_classifier = "LSVM" # QDA # GNB RF ul.create_folder_if_needed(folder_model) classifier = cl_d[key_classifier] pkl.store_pickle(folder_model + key_classifier + ".pkl", [classifier]) pkl.store_pickle(folder_model + "scaler_X" + ".pkl", [scaler_X]) pkl.store_pickle(folder_model + "scaler_Y" + ".pkl", [scaler_Y]) """ PLOT THE RESULTS """ if (plot_performance_all): def get_class_rate(Y, Ypred): return np.mean(np.equal(Y, Ypred)) def get_CE(Y, Ypred):
from graph_lib import gl import mpmath as mpm import utilities_lib as ul # Import specific model libraries from scipy import spatial from matplotlib import pyplot as plt from matplotlib import cm as cm import pickle_lib as pkl plt.close("all") # Close all previous Windows ################## Set the output folders ################## folder_data = "./data/artificial/" ul.create_folder_if_needed(folder_data) plot_flag = 0 ### Generation of the easy signal def mean_function(tgrid, f1 = 1, f2 = 5, a1 = 0.4, a2 = 0.2, phi2 = 2*np.pi/7, m = 0.1 ): # This function outputs the function that we want to estimate. """ m = 0.1 # Slope of the linear trend f1 = 1 # Frquency of first sinusoid f2 = 5 # Frequency of second sinusoid a1 = 0.4; # Amplitud of the first sinuoid a2 = 0.2 # Amplitud of the second sinusoid phi2 = 2*np.pi/7 # Phase shifting of the second sinuoid