def main(opts): dataset = opts.dataset embed_dim = int(opts.dimension) # File that contains the edges. Format: source target # Optionally, you can add weights as third column: source target weight edge_f = 'Data/%s.edgelist' % dataset # Specify whether the edges are directed # isDirected = True print "Loading Dataset" # Load graph G = graph_util.loadGraphFromEdgeListTxt(edge_f, directed=False) #G = G.to_directed() embedding = LaplacianEigenmaps(d=embed_dim) print('Num nodes: %d, num edges: %d' % (G.number_of_nodes(), G.number_of_edges())) t1 = time() # Learn embedding - accepts a networkx graph or file with edge list print "Starting Embedding" Y, t = embedding.learn_embedding(graph=G, edge_f=None, is_weighted=True, no_python=True) print(embedding._method_name + ':\n\tTraining time: %f' % (time() - t1)) np_save(writable("Embedding_Results", "jac_" + dataset + str(embed_dim)), Y)
def array2data(arr, adress, other='C', delimiter='\t'): '''Save an array (max 2D for dat/txt) as "*.dat/txt" or "*.npy".''' if adress.endswith('.npy'): np_save(adress, arr) elif adress.endswith('.txt')\ or adress.endswith('.asc')\ or adress.endswith('.dat'): ofi = open(adress, 'w') try: for i in xrange(arr.shape[0]): for j in xrange(arr.shape[1]): ofi.write(str(arr[i, j]) + delimiter) ofi.write('\n') except: for i in xrange(arr.shape[0]): ofi.write(str(arr[i]) + delimiter) ofi.close() elif adress.endswith('.mat'): from scipy.io import savemat if type(arr) == dict: savemat(adress, arr) else: savemat(adress, {other: arr})
def main(opts): dataset = opts.dataset embed_dim = int(opts.dimension) # File that contains the edges. Format: source target # Optionally, you can add weights as third column: source target weight edge_f = 'Data/%s.edgelist' % dataset G = nx.read_edgelist(edge_f) A = nx.adjacency_matrix(G) num_points = A.shape[0] D = np.sum(A, axis=0) D = np.squeeze(np.asarray(D)) A = csr_matrix(A) res = optimize.minimize(partial(num_lap, A=A, n=num_points, dim=embed_dim, D=D, c=1), np.random.rand(num_points * embed_dim), jac=partial(lap_jac, A=A, n=num_points, dim=embed_dim, c=1)) print(res.fun) t1 = time() Y = res.x.reshape(num_points, embed_dim) print(embedding._method_name + ':\n\tTraining time: %f' % (time() - t1)) np_save( writable("Embedding_Results", dataset + str(embed_dim) + opts.cost), Y)
def save(self): if not exists(self.directory): mkdir(self.directory) arr = [self.num_steps, self.action_num] np_save(file=join(self.directory, self.name), arr=arr) np_save(file=join(self.directory, self.name + "_eps"), arr=self.eps)
def save(self): from numpy import save as np_save from os import mkdir from os.path import exists, join if not exists(self.__directory): mkdir(self.__directory) for changed in self.__change_log: path = join(self.__directory, f"memory_{changed:05d}") np_save(file=path, arr=self.__episodes[changed]) self.__change_log.clear() path = join(self.__directory, f"memory_meta") np_save(file=path, arr=[self.__pos, self.__capacity])
def log_transitions(self): if len(self.memory) > 0: basename = self.logdir + "/{}.{}".format( self.environment_name, datetime.now().strftime("%Y-%m-%d-%H-%M-%s")) print("Base Filename: ", basename, flush=True) state, action, reward, next_state, done = zip(*self.memory) np_save(basename + "-state.npy", np_array(state), allow_pickle=False) np_save(basename + "-action.npy", np_array(action), allow_pickle=False) np_save(basename + "-reward.npy", np_array(reward), allow_pickle=False) np_save(basename + "-nextstate.npy", np_array(next_state), allow_pickle=False) np_save(basename + "-done.npy", np_array(done), allow_pickle=False) self.memory.clear()
def learn_manifold(self, X, manifold_out_file_name=None): self.debug_string_out.clear() self.print_and_remember("Learning manifold(" + self.manifold_learner + ")" + str(datetime.now())) learn_time = time() if manifold_out_file_name is not None and isfile( manifold_out_file_name ): # check the learned manifold existance manifold_feats = np_load(manifold_out_file_name, allow_pickle=True) self.print_and_remember("Manifold loaded(" + manifold_out_file_name + ")") elif self.manifold_learner == 'UMAP': manifold_feats = UMAP(random_state=0, metric=self.dist_metric, n_components=self.manifold_dimension, n_neighbors=self.num_of_neighbours, min_dist=float( self.min_dist)).fit_transform(X) elif self.manifold_learner == 'LLE': manifold_feats = LocallyLinearEmbedding( n_components=self.manifold_dimension, n_neighbors=self.num_of_neighbours).fit_transform(X) elif self.manifold_learner == 'tSNE': manifold_feats = TSNE(n_components=self.manifold_dimension, random_state=0, verbose=0).fit_transform(X) elif self.manifold_learner == 'isomap': manifold_feats = Isomap( n_components=self.manifold_dimension, n_neighbors=self.num_of_neighbours).fit_transform(X) self.print_and_remember( "Time to learn manifold: " + str(funcH.getElapsedTimeFormatted(time() - learn_time))) if manifold_out_file_name is not None: np_save(manifold_out_file_name, manifold_feats, allow_pickle=True) self.print_and_remember("Manifold saved(" + manifold_out_file_name + ")") return manifold_feats, self.debug_string_out
def __call__(self, img_processed, img_normalized, img_with_background=None): if not Capture.capture: return Capture.capture = False # When files are listed they will be sorted by subject, then capture number, then capture type filename_basis = GlobalConfig.Paths.data_dir + Capture.subject + '_' + str( Capture.subject_counter).zfill(4) np_save(filename_basis + '_depth.npy', img_processed) cv2.imwrite(filename_basis + '_depth_normalized.png', img_normalized) if not img_with_background is None: np_save(filename_basis + '_depth_withbg.npy', img_with_background) img_with_background = NormalizeFor8Bit(img_with_background) cv2.imwrite(filename_basis + '_depth_withbg.png', img_with_background) Capture.subject_counter += 1 print('Captured "' + filename_basis + '"')
def compute_tsne_positions(activations_per_point: array, perplexity: int, name: str) -> array: """ Computes t-SNE positions from a dataset. `activations_per_point`: n_obvs x n_dims """ logger.info(f"TSNE from data of size {activations_per_point.shape}") t_sne_positions_path = Path(TSNE_SAVE_DIR, f"t-sne positions {name} perp={perplexity}.npy") if t_sne_positions_path.exists(): logger.info("Loading...") t_sne_positions = np_load(t_sne_positions_path) else: logger.info("Computing...") t_sne_positions = TSNE( n_components=2, # 2D perplexity=perplexity, # Recommended args n_iter=1_000, learning_rate=200, method="barnes_hut", ).fit_transform(activations_per_point) np_save(t_sne_positions_path, t_sne_positions) return t_sne_positions
probs = mdl.predict_proba(X_test) y_pred = probs[:,1] cv_scores[c] = qual(y_test, y_pred) tech_data.append(cv_scores.copy()) output.write('%s,%d,%f,%f,%f,%f,%s\n' % ('Guess', d, np.mean(cv_scores), np.std(cv_scores), t, p, args.label)) plot_data.append(tech_data) output.close() names = techniques + ["None", "Guessing"] points = [str(step) for step in dim_steps] # Save off plotting data np_save(path.join(cv_dir, 'plot_data.npy'), array(plot_data)) # Save off additional data output = open(path.join(cv_dir, 'plot_data.txt'), 'w') output.write('#TECHNIQUES,' + ','.join(names) + '\n') output.write('#STEPS,' + ','.join(points) + '\n') output.close() fig, ax = plt.subplots(figsize=(10, 8), dpi=80) plot_data = array(plot_data) lgd = make_grouped_box(ax, plot_data, names, xticklabels=points, legend_pos='outside') ax.set_ylabel('AUC') ax.set_xlabel('Number of Dimensions') ax.set_title('Predicting "%s"' % (metadata_category)) plt.savefig(args.output_file, bbox_extra_artists=(lgd,), bbox_inches='tight')
def test(primaryDataSet, helperDataSet, primaryClass, helperClass, primaryInstances, helperInstances): ''' Inputs : dataSets : List : Datasets for which samples are to be genrated instances : List : Number of instances to be used from original dataset classes : List : Classes for which samples are to be generated Output : File with 1000 compressed images generated by GAN ''' helperClass = primaryClass modelFolder = resultDir + 'models/crossDataSetMMDall'+'/'+primaryDataSet+'/' print (primaryDataSet, helperDataSet, primaryClass, helperClass, primaryInstances, helperInstances, getEpochs(primaryDataSet,primaryInstances)-1) modelFile = modelFolder + primaryDataSet + '_' + helperDataSet + '_' + \ str(primaryClass) + '_' + str(helperClass) + '_' + \ str(primaryInstances) + '_' + str(helperInstances)+'_'+ \ str(getEpochs(primaryDataSet,primaryInstances)-1)+'.pt' print ('Generating examples for Dataset: '+primaryDataSet+ ' Primary Class: '+str(primaryClass)+ ' Helper Class: '+str(helperClass)+ ' Primary Instances: '+str(primaryInstances)+ ' Helper Instances: '+str(helperInstances)+ ' Epochs: '+str(getEpochs(primaryDataSet,primaryInstances))) numOutputChannels = getChannels(primaryDataSet) # load the model learnt during training G = Generator(numInputChannels, numGenFilter, numOutputChannels) G.load_state_dict(torch.load(modelFile)) genImageConcat = np.empty(1) iterations = numOfSamples/batchSize for iteration in range(iterations): noise = torch.FloatTensor(batchSize, numInputChannels, 1, 1) noise.normal_(0,1) if cuda: G = G.cuda() noise = noise.cuda() noiseVariable = Variable(noise) genImage = G(noiseVariable) genImage = genImage.data genImage = genImage.cpu() genImage = genImage.numpy() if iteration==0: genImageConcat = genImage else: genImageConcat = np.concatenate((genImageConcat, genImage), axis=0) if iteration==(iterations-1): # normalize sets image pixels between 0 and 1 genImage = torchvision.utils.make_grid(torch.from_numpy(genImage[:25]), nrow=5, normalize=True) # mapping between 0 to 1 as required by imshow, # otherwise the images are stored in the form -1 to 1 # done through normalize=True genImage = genImage.permute(1,2,0) genImage = genImage.numpy() plt.imshow(genImage, cmap='gray') plotFolder = resultDir+'results'+'/'+'crossDataSetMMDall/samples'+'/'+primaryDataSet+'/' checkAndCreateFolder(plotFolder) plotFile = primaryDataSet + '_' + helperDataSet + '_' + \ str(primaryClass) + '_' + 'all' + '_' + \ str(primaryInstances) + '_' + str(helperInstances) plotPath = plotFolder + plotFile plt.axis('off') plt.savefig(plotPath, bbox_inches='tight') plt.show() resultFolder = resultDir+'results/crossDataSetMMDall'+'/'+'compressed'+'/'+primaryDataSet+'/' checkAndCreateFolder(resultFolder) resultFile = primaryDataSet + '_' + helperDataSet + '_' + \ str(primaryClass) + '_' + 'all' + '_' + \ str(primaryInstances) + '_' + str(helperInstances) + '.npy' resultPath = resultFolder + resultFile # save the image in some format with open(resultPath,'wb+') as fh: genImageConcat = np.squeeze(genImageConcat) np_save(fh, genImageConcat, allow_pickle=False) sync(fh)
def fit_ptsne_model(model: torch.nn.Module, input_points: Dataset, opt: Optimizer, perplexity: Optional[int], n_epochs: int, dev: str, save_dir_path: str, epochs_to_save_after: Optional[int], early_exaggeration: int, early_exaggeration_constant: int, batch_size: int, dist_func_name: str, bin_search_tol: float, bin_search_max_iter: int, min_allowed_sig_sq: float, max_allowed_sig_sq: float, configuration_report: str) -> None: """ Fits a parametric t-SNE model and optionally saves it to the desired directory. Fits either regular or multi-scale t-SNE :param model: nn.Module instance :param input_points: tensor of original points :param opt: optimizer instance :param perplexity: perplexity of a model. If passed None, multi-scale parametric t-SNE model will be trained :param n_epochs: Number of epochs for training :param dev: device for tensors (e.g. "cpu" or "cuda") :param save_dir_path: path to directory to save a trained model to :param epochs_to_save_after: number of epochs to save a model after. If passed None, model won't be saved at all :param early_exaggeration: Number of first training cycles in which exaggeration will be applied :param early_exaggeration_constant: Constant by which p_joint is multiplied in early exaggeration :param batch_size: Batch size for training :param dist_func_name: Name of distance function for distance matrix. Possible names: "euc", "jaccard", "cosine" :param bin_search_tol: A small number - tolerance threshold for binary search :param bin_search_max_iter: Number of max iterations for binary search :param min_allowed_sig_sq: Minimum allowed value for the spread of any distribution in conditional probability matrix :param max_allowed_sig_sq: Maximum allowed value for the spread of any distribution in conditional probability matrix :param configuration_report: Config of the model in string form for report purposes :return: """ model.train() batches_passed = 0 model_name = get_random_string(6) epoch_losses = [] # Function operates with DataLoader train_dl = DataLoader(input_points, batch_size=batch_size, shuffle=True) for epoch in range(n_epochs): train_loss = 0 epoch_start_time = datetime.datetime.now() # For every batch for list_with_batch in tqdm(train_dl): orig_points_batch, _ = list_with_batch # Calculate conditional probability matrix in higher-dimensional space for the batch # Regular parametric t-SNE if perplexity is not None: target_entropy = log2(perplexity) p_cond_in_batch = calculate_optimized_p_cond( orig_points_batch, target_entropy, dist_func_name, bin_search_tol, bin_search_max_iter, min_allowed_sig_sq, max_allowed_sig_sq, dev) if p_cond_in_batch is None: continue p_joint_in_batch = make_joint(p_cond_in_batch) # Multiscale parametric t-SNE else: max_entropy = round(log2(batch_size / 2)) n_different_entropies = 0 mscl_p_joint_in_batch = zeros(batch_size, batch_size).to(device(dev)) for h in range(1, max_entropy): p_cond_for_h = calculate_optimized_p_cond( orig_points_batch, h, dist_func_name, bin_search_tol, bin_search_max_iter, min_allowed_sig_sq, max_allowed_sig_sq, dev) if p_cond_for_h is None: continue n_different_entropies += 1 p_joint_for_h = make_joint(p_cond_for_h) # TODO This fails if the last batch doesn't match the shape of mscl_p_joint_in_batch mscl_p_joint_in_batch += p_joint_for_h p_joint_in_batch = mscl_p_joint_in_batch / n_different_entropies # Apply early exaggeration to the conditional probability matrix if early_exaggeration: p_joint_in_batch *= early_exaggeration_constant early_exaggeration -= 1 batches_passed += 1 opt.zero_grad() # Calculate joint probability matrix in lower-dimensional space for the batch embeddings = model(orig_points_batch) q_joint_in_batch = get_q_joint(embeddings, "euc", alpha=1) # Calculate loss loss = loss_function(p_joint_in_batch, q_joint_in_batch) train_loss += loss.item() # Make an optimization step loss.backward() opt.step() epoch_end_time = datetime.datetime.now() time_elapsed = epoch_end_time - epoch_start_time # Report loss for epoch average_loss = train_loss / batches_passed epoch_losses.append(average_loss) print( f'====> Epoch: {epoch + 1}. Time {time_elapsed}. Average loss: {average_loss:.4f}', flush=True) # Save model and loss history if needed save_path = os.path.join(save_dir_path, f"{model_name}_epoch_{epoch + 1}") if epochs_to_save_after is not None and ( epoch + 1) % epochs_to_save_after == 0: torch.save(model, save_path + ".pt") with open(save_path + ".json", "w") as here: json.dump(json.loads(configuration_report), here) print('Model saved as %s' % save_path, flush=True) if epochs_to_save_after is not None and epoch == n_epochs - 1: epoch_losses = array(epoch_losses) loss_save_path = save_path + "_loss.npy" np_save(loss_save_path, epoch_losses) print("Loss history saved in", loss_save_path, flush=True)
NormW = 4 dir_pri1 = './AT_tT1t_X_Patt' dir_pri2 = '../FilesIN/Time_Courses/only_positives' dir_pri3 = './fRMI_only_positives' if not os.path.exists(dir_pri1): print 'Pas de dossier ' + dir_pri1 quit() if not os.path.exists(dir_pri2): print 'Pas de dossier ' + dir_pri2 quit() if not os.path.exists(dir_pri3): os.mkdir(dir_pri3) t0 = time() pool = multiprocessing.Pool() args = arange(0.02,0.981,0.03).tolist() resultslist = pool.map(whatyouwant, args) resultsarra = resultslist[0] for ll in range(1, len(resultslist)): resultsarra = append(resultsarra, resultslist[ll], axis=0) np_save(dir_pri3 + '/correlations_W%.2f_p%i.npy' %(NormW, patient), resultsarra) print "Simulation took :",time()-t0,"s" print "min= %.2f ; max= %.2f" %(resultsarra.min(), resultsarra.max()) imshow(abs(resultsarra), cmap=get_cmap('Blues'), interpolation='nearest') show()
else: da = ncout.variables['delta_params5'][:] ncout.close() if FLAG_mns: filestr_draws = "draws_" + str(ch) + "_" + str(npop) + ".npy" filestr_ensemble = "ensemble_" + str(ch) + "_" + str(npop) + ".npy" # Load / Generate draws if FLAG_load_draws: draws = np_load(filestr_draws) else: draws = ensemble_func.cov2draws(har, npop) np_save(filestr_draws, draws, allow_pickle=False) # Load / Generate ensemble if FLAG_load_ensemble: ens = np_load(filestr_ensemble, allow_pickle=True).item() else: ens = ensemble_func.draws2ensemble(draws, nens) np_save(filestr_ensemble, ens, allow_pickle=True) ensemble = ens['ensemble'] ensemble_idx = ens['ensemble_idx'] decile = ens['decile'] Z = ens['Z'] Z_norm = ens['Z_norm'] da = ensemble - a_ave # [nens,npar]
def file_np_save(path: str, np_array): Path(path).parent.mkdir(parents=True, exist_ok=True) np_save(path, np_array)
'#################### TC_998' t0 = time() for i in range(nTC): TC1[i * step:(i + 1) * step, :] = np_load(dir_TC1 + dir_sub + 'TC_%i.npy' % i) print 'TC 998 loaded : %.2fs' % (time() - t0) '#################### BO_998' t0 = time() for k in range(N1): Bk = convolve(TC1[:, k], Hrv)[:tmax] for o in range(avgOn): BO1[:, k] += Bk[o::avgOn] BO1 /= avgOn np_save(dir_BO1 + 'BO_%i_' % N1 + name + '.npy', BO1) print 'BO 998 generated : %.2fs' % (time() - t0) '#################### BO_66, FC_B66, FC_B998' t0 = time() for n in range(N1): BO2[:, D998_D66[n]] += BO1[:, n] for t in range(BO1.shape[0]): BO2[t, :] /= AvgN_D66 np_save(dir_BO2 + 'BO_%i_' % N2 + name + '.npy', BO2) FC2 = fastPearsonCorrelation(BO2) del BO2 np_save(dir_FC2 + 'FC_B%i_' % N2 + name + '.npy', FC2) del FC2 FC1 = fastPearsonCorrelation(BO1) del BO1
def main(): x_data, y_data = load_data() ## Set parameter to True for initial download. ## Once data is present, set this to False to ## prevent re-downloading data. ## Plotting the Iris Petal and Sepal length and width. plot_iris_data(x_data, y_data) y_data = one_hot(y_data) #Split data: 80% test set, 20% validation set. i_80 = int(len(y_data) * 0.8) x_train, y_train = x_data[:i_80], y_data[:i_80] x_test, y_test = x_data[i_80:], y_data[i_80:] iris_nn = NN_Model( x_train, ## input data. y_train, ## output data. 3, ## 3 NN layers: Input, hidden-layer, output. [4, 4, 3]) ## num of nodes for each layer. if Grad_Descent_Method: print("\nNeural Network XNOR - using GRADIENT DESCENT ITERATION\n", "#" * 30, "\n") # File location where learned weight is saved. theta_file = CURRENT_PATH + r'/' + 'theta.npy' if LOAD_PREV_THETAS: flat_thetas = np_load(theta_file) iris_nn.unflatten_Thetas(flat_thetas) if CONTINUOUS_TRAINING: iris_nn.train_NN() np_save(theta_file, iris_nn.flatten_Thetas()) else: iris_nn.train_NN() np_save(theta_file, iris_nn.flatten_Thetas()) # Display final cost after learning iterations. print("Final Cost J = ", iris_nn.J_cost(iris_nn.a[-1])) if PLOT_COST: # Plot the J Cost vs. # of iterations. J should coverge as iteration increases. x_axis = range(len(iris_nn.J_cost_values)) y_axis = iris_nn.J_cost_values plt.plot(x_axis, y_axis, label='J_cost vs. # of Iterations') plt.show() # Test model accuracy on Validation/Test set. acc_count = 0 for i in range(len(x_test)): x_input = x_test[i].flatten() y_val = np.argmax(y_test[i]) y_pred = iris_nn.predict(x_input)[0] #print(y_pred, y_val); if y_pred == y_val: acc_count += 1 print("Test Accuraccy = {}".format(acc_count / len(x_test))) return 0
def plotAccuracyBar(dataSet, classifier, primaryInstances, helperInstances, accuracyArray, showImage = 1): fig = plt.figure() ax = fig.add_subplot(111) ind = np.arange(len(primaryInstances)) width = 0.15 delta = 0.02 color = ['black', 'red', 'blue', 'green'] histList = [] legendList = [] for i in range(accuracyArray.shape[0]): hist = ax.bar(ind+i*(width+delta), list(accuracyArray[:,i]), width, color=color[i]) histList.append(hist) if i==0: legendList.append('Original') elif i==1: legendList.append('GAN - 0 Helper Instances') else: legendList.append('GAN - '+str(helperInstances[i-2])+' Helper Instances') print (accuracyArray) ax.set_xlim(-4*width,len(ind)+width) ax.set_ylim(0,1.5) ax.set_ylabel('Accuracy [out of 1]') ax.set_xlabel('Number of Primary Instances') ax.set_title(dataSet) ax.set_xticks(ind+width) xtickNames = ax.set_xticklabels(primaryInstances) plt.setp(xtickNames, rotation=45, fontsize=10) # add to tuple ax.legend( tuple(histList), tuple(legendList) , loc='upper left') ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f')) plotFolderName = 'plots/crossDataSetMMDall/accuracy'+'/'+dataSet+'/' checkAndCreateFolder(plotFolderName) plotFileName = plotFolderName+dataSet+'_'+classifier+'.png' plt.savefig(plotFileName, bbox_inches='tight') plt.show() plt.close() accuracyFolderName = 'plots/crossDataSetMMDall/accuracyValues'+'/'+dataSet+'/' checkAndCreateFolder(accuracyFolderName) accuracyFileName = accuracyFolderName+dataSet+'_'+classifier+'.npy' # save the image in some format with open(accuracyFileName,'wb+') as fh: np_save(fh, accuracyArray, allow_pickle=False) sync(fh)
def _save(self, matrix: array): print(f"Saving t-SNE data for {self.dims} dimensions...") np_save(self.save_path_for(dims=self.dims, distance_name=self.distance_name), matrix)
if __name__ == '__main__': patient = 5 dir_pri1 = './AT_tT1t_S_Patt' dir_pri2 = '../FilesIN/Time_Courses/only_positives' dir_pri3 = './fRMI_only_positives' if not os.path.exists(dir_pri1): print 'Pas de dossier ' + dir_pri1 quit() if not os.path.exists(dir_pri2): print 'Pas de dossier ' + dir_pri2 quit() if not os.path.exists(dir_pri3): os.mkdir(dir_pri3) t0 = time() pool = multiprocessing.Pool() args = arange(0.50,15.01,0.5).tolist() resultslist = pool.map(whatyouwant, args) resultsarra = resultslist[0] for ll in range(1, len(resultslist)): resultsarra = append(resultsarra, resultslist[ll], axis=0) np_save(dir_pri3 + '/correlations_S_p%i.npy' %patient, resultsarra) print "Simulation took :",time()-t0,"s" print "min= %.2f ; max= %.2f" %(resultsarra.min(), resultsarra.max()) imshow(abs(resultsarra), cmap=get_cmap('Blues'), interpolation='nearest') show()
return W if __name__ == '__main__': dir_pri2 = './MatricesG30TVarXAda3' if not os.path.exists(dir_pri2): print 'Pas de dossier ' + dir_pri2 quit() t0 = time() pool = multiprocessing.Pool() args = arange(0.50, 4.501, 0.2).tolist() # normW results = pool.map(whatyouwant, args) results = array(results) np_save(dir_pri2 + '/results.npy', results) print "Simulation took :", time() - t0, "s" W = [] for j in range(results.shape[1]): figure() for i in range(results.shape[0]): subplot(3, 7, i + 1) title(i) imshow(results[i, j], interpolation='nearest') show() mats = input() for i in range(results.shape[0]): if i in mats: W.append(results[i, j])
epochs = 150 dims = 50 eta = 0.1 window_size = 1 minimum_count = 2 for model_name, Word2Vec_model in [ ('Gensim', Word2Vec_gensim), ('Train_on_current', Word2Vec_train_on_current), ('Not_optimised', Word2Vec_no_optimisations), ('No_train_operator_input', Word2Vec_no_train_w_operator_input) ]: for filename in os.listdir(corpus_dir): if filename.endswith('.txt'): fullpath = os.path.join(corpus_dir, filename) logic_examples = SpacedLines(fullpath) models_dict['{}-{}'.format(model_name, filename)] = Word2Vec_model( logic_examples, sg=1, min_count=minimum_count, window=window_size, size=dims, alpha=eta, iter=epochs) print('Model {} file {} complete'.format(model_name, filename)) np_save('models_dict2.npy', models_dict) print('Done!!')
labels_file = args.input_labels output_dir = args.output_dir dim_steps = [int(x) for x in args.dims.split(',')] output_dir = args.output_dir scripts_dir = args.scripts_dir tag = args.tag labels = np_load(labels_file) cv_folds = BalancedKFold(labels, args.num_folds, n_iter=args.cv_iters) i = 0 for k, (training, testing) in enumerate(cv_folds): file_prefix = path.join(output_dir, 'CV_%d_' % k) training_file = file_prefix+'training.npy' np_save(training_file, training) testing_file = file_prefix+'testing.npy' np_save(testing_file, testing) for num_dims in dim_steps: if i >= args.max_scripts: ind = i % args.max_scripts script_file = path.join(scripts_dir, '%d.sh' % ind) add_to_bash_script(script_file, data_matrix_file, labels_file, output_dir, training_file, testing_file, num_dims, k, args.methods) else: script_file = path.join(scripts_dir, '%d.sh' % i) make_bash_script(script_file, data_matrix_file, labels_file, output_dir, training_file, testing_file, num_dims, k, args.methods) i += 1
def save(self, arr, pth): with open(pth, 'wb+') as fh: np_save(fh, arr, allow_pickle=False) sync(fh)
def main(): data = import_urls(redownload_data=False) ## Set parameter to True for initial download. ## Once data is present, set this to False to ## prevent re-downloading data. X_10k, y_10k = None, None X_60k, y_60k = None, None for tag, num_items, raw_data in data: #print(tag, num_items, raw_data.shape, np.min(raw_data), np.max(raw_data)); if num_items == 10000: if tag == r'image': X_10k = raw_data if tag == r'label': y_10k = raw_data if num_items == 60000: if tag == r'image': X_60k = raw_data if tag == r'label': y_60k = raw_data digits_nn = NN_digits( X_10k, ## input data. y_10k, ## output data. 3, ## 3 NN layers: Input, inter-layer, output. [28 * 28, 25, 10]) ## num of nodes for each layer. if Grad_Descent_Method: print("\nNeural Network XNOR - using GRADIENT DESCENT ITERATION\n", "#" * 30, "\n") # File location where learned weight is saved. theta_file = CURRENT_PATH + r'/' + 'theta.npy' if LOAD_PREV_THETAS: flat_thetas = np_load(theta_file) digits_nn.unflatten_Thetas(flat_thetas) if CONTINUOUS_TRAINING: digits_nn.train_NN() np_save(theta_file, digits_nn.flatten_Thetas()) else: digits_nn.train_NN() np_save(theta_file, digits_nn.flatten_Thetas()) # Display final cost after learning iterations. print("Final Cost J = ", digits_nn.J_cost(digits_nn.a[-1])) if PLOT_COST: # Plot the J Cost vs. # of iterations. J should coverge as iteration increases. x_axis = range(digits_nn.J_cost_values) y_axis = digits_nn.J_cost_values plt.plot(x_axis, y_axis, label='J_cost vs. # of Iterations') plt.show() # Pick 10 samples randomly from 60k samples pool. for i in range(0, 10): ith_sample = randint(0, 60000 - 1) x_input = X_60k[ith_sample].flatten() y_val = y_60k[ith_sample] test_res = digits_nn.H_funct(x_input) top_three_answers = "" for val, prob in test_res[1]: top_three_answers += str(val) + ":" + str(prob) + "%, " top_three_answers = top_three_answers[:-2] print("y_val = ", y_val[0], " || Test result = ", test_res[0], "TOP THREE: ", top_three_answers) ## If result is not as expected, display the image. if not test_res[0] == y_val: plot_image(X_60k[ith_sample]) ### Calculate error rate of 60k test samples. ### if COUNT_ERROR: error_cnt = 0 for i in range(60000): x_input = X_60k[i] y_val = y_60k[i] test_res = digits_nn.H_funct(x_input) error_cnt += (test_res[0] != y_val) error_rate = int(error_cnt / 6) / 100 print("ERROR RATE: ", error_rate, "%.") if Minimize_funct_method: print("\nNeural Network XNOR - using fmin_bfgs\n", "#" * 30, "\n") digits_nn.init_thetas() digits_nn.train_NN_with_fmin() for i in range(0, 1000, 10): x_input = X_60k[i].flatten() y_val = y_60k[i] test_res = digits_nn.H_funct(x_input) print("y_val = ", y_val[0], " || Test result = ", test_res, test_res == y_val) print("\n") return 0
def test(dataSet, generatedClass, numOfInstances): ''' Inputs : dataSets : List : Datasets for which samples are to be genrated instances : List : Number of instances to be used from original dataset classes : List : Classes for which samples are to be generated Output : File with 1000 compressed images generated by GAN ''' modelFolder = 'models' + '/' + dataSet modelFile = modelFolder+'/'+dataSet+'_'+str(generatedClass)+'_'+ \ str(numOfInstances)+'_'+str(getEpochs(numOfInstances))+'.pt' print('Generating examples for Dataset: ' + dataSet + ' Class: ' + str(generatedClass) + ' Instances: ' + str(numOfInstances) + ' Epochs: ' + str(getEpochs(numOfInstances))) # load the model leannt during training G = Generator(params.numInputChannels, params.numGenFilter, params.numOutputChannels) G.load_state_dict(torch.load(modelFile)) genImageConcat = np.empty(1) for sample in range(params.numOfSamples): noise = torch.FloatTensor(1, params.numInputChannels, 1, 1) noise.normal_(0, 1) if params.cuda: G = G.cuda() noise = noise.cuda() noiseVariable = Variable(noise) genImage = G(noiseVariable) genImage = genImage.data genImage = genImage.cpu() genImage = genImage.numpy() maxImage = np.max(genImage) minImage = np.min(genImage) #print maxImage, minImage genImage = np.multiply( np.divide((genImage - minImage), (maxImage - minImage)), 255.0) maxImage = np.max(genImage) minImage = np.min(genImage) if sample == 0: genImageConcat = genImage elif sample >= 1: genImageConcat = np.concatenate((genImageConcat, genImage), axis=0) path = '../DCGAN/results'+'/'+'compressed'+'/'+dataSet+'/'+ dataSet + '_' \ + str(generatedClass) + '_' + str(numOfInstances) + '.npy' # save the image in some format with open(path, 'wb+') as fh: genImageConcat = np.squeeze(genImageConcat) np_save(fh, genImageConcat, allow_pickle=False) sync(fh)
if __name__=="__main__": args = interface() num_dims = args.num_dims prefix = 'CV_%d' % (args.cv_fold_id) techniques = get_methods(args.methods) output_dir = args.output_dir data_matrix = np_load(args.input_file) labels = np_load(args.input_labels) testing = np_load(args.testing_vector) training = np_load(args.training_vector) test_labels = labels[testing] training_labels = labels[training] test_matrix = data_matrix[testing, :] training_matrix = data_matrix[training, :] for technique, technique_name in techniques: dim_redux = technique(n_components=num_dims) file_label = '%s_%s_%s_' % (prefix, technique_name, str(num_dims)) file_prefix = path.join(output_dir, file_label) txd_training_matrix = dim_redux.fit_transform(training_matrix, training_labels) txd_test_matrix = dim_redux.transform(test_matrix) np_save(file_prefix + 'txd_test_matrix', txd_test_matrix) np_save(file_prefix + 'txd_training_matrix', txd_training_matrix)
def save(self, data): validate_dtype(data, ndarray) np_save(file=self.path, arr=data, allow_pickle=True)
def fit_model(model: torch.nn.Module, train_dl: DataLoader, val_dl: DataLoader, opt: Optimizer, perplexity: int, n_epochs: int, save_dir_path: str, epochs_to_save_after: int, early_exaggeration: int, early_exaggeration_constant: int, batch_size: int, dist_func_name: str, bin_search_tol: float, bin_search_max_iter: int, min_allowed_sig_sq: float, max_allowed_sig_sq: float, configuration_report: str) -> None: """ Fits t-SNE model :param model: nn.Module instance :param train_dl: data loader with points for training :param val_dl: data loader with points for validation and early stopping :param opt: optimizer instance :param perplexity: perplexity :param n_epochs: Number of epochs for training :param save_dir_path: path to directory to save a trained model to :param epochs_to_save_after: number of epochs to save a model after. If passed None, model won't be saved at all :param early_exaggeration: Number of first training cycles in which exaggeration will be applied :param early_exaggeration_constant: Constant by which p_joint is multiplied in early exaggeration :param batch_size: Batch size for training :param dist_func_name: Name of distance function for distance matrix. Possible names: "euc", "jaccard", "cosine" :param bin_search_tol: Tolerance threshold for binary search to obtain p_cond :param bin_search_max_iter: Number of max iterations for binary search :param min_allowed_sig_sq: Minimal allowed value for squared sigmas :param max_allowed_sig_sq: Maximal allowed value for squared sigmas :param configuration_report: Config of the model in string form for report purposes :return: """ model_name = get_random_string(6) train_batch_losses = [] train_epoch_losses = [] val_batch_losses = [] val_epoch_losses = [] for epoch in range(n_epochs): epoch_start_time = datetime.datetime.now() model.train() for list_with_batch in tqdm(train_dl): orig_points_batch, _ = list_with_batch with torch.no_grad(): p_joint_in_batch = calc_p_joint_in_batch( perplexity, orig_points_batch, dist_func_name, bin_search_tol, bin_search_max_iter, min_allowed_sig_sq, max_allowed_sig_sq) opt.zero_grad() embeddings = model(orig_points_batch) q_joint_in_batch = get_q_joint(embeddings, "euc", alpha=1) if early_exaggeration: p_joint_in_batch *= early_exaggeration_constant early_exaggeration -= 1 loss = loss_function(p_joint_in_batch, q_joint_in_batch) train_batch_losses.append(loss.item()) loss.backward() opt.step() model.eval() for val_list_with_batch in tqdm(val_dl): val_orig_points_batch, _ = val_list_with_batch with torch.no_grad(): p_joint_in_batch_val = calc_p_joint_in_batch( perplexity, val_orig_points_batch, dist_func_name, bin_search_tol, bin_search_max_iter, min_allowed_sig_sq, max_allowed_sig_sq) val_embeddings = model(val_orig_points_batch) q_joint_in_batch_val = get_q_joint(val_embeddings, "euc", alpha=1) loss_val = loss_function(p_joint_in_batch_val, q_joint_in_batch_val) val_batch_losses.append(loss_val.item()) train_epoch_loss = mean(train_batch_losses) train_epoch_losses.append(train_epoch_loss) val_epoch_loss = mean(val_batch_losses) val_epoch_losses.append(val_epoch_loss) train_batch_losses = [] val_batch_losses = [] epoch_end_time = datetime.datetime.now() time_elapsed = epoch_end_time - epoch_start_time # Report loss for epoch print( f'====> Epoch: {epoch + 1}. Time {time_elapsed}. Average loss: {train_epoch_loss:.4f}. Val loss: {val_epoch_loss:.4f}', flush=True) # Save model and loss history if needed save_path = os.path.join(save_dir_path, f"{model_name}_epoch_{epoch + 1}") if epochs_to_save_after is not None and ( epoch + 1) % epochs_to_save_after == 0: torch.save(model, save_path + ".pt") with open(save_path + ".json", "w") as here: json.dump(json.loads(configuration_report), here) print('Model saved as %s' % save_path, flush=True) if epochs_to_save_after is not None and epoch == n_epochs - 1: loss_save_path = save_path + "_loss.npy" np_save( loss_save_path, vstack((array(train_epoch_losses), array(val_epoch_losses)))) print("Loss history saved in", loss_save_path, flush=True)