def plot_decomposition(args): print(f'Reading vectors from `{args.vec_path}`...') embeddings, w2i, i2w = load_vectors(args.vec_path, gensim=args.gensim_format) matrix_path = os.path.join(args.matrix_dir, f'{args.name}') logX = load_matrix(matrix_path + '.logx.npz') fX = load_matrix(matrix_path + '.fx.npz') logX, fX = logX.todense(), fX.todense() plt.imshow(embeddings) plt.savefig(os.path.join('plots', 'emb.pdf')) plt.clf() plt.imshow(embeddings.T) plt.savefig(os.path.join('plots', 'emb.t.pdf')) plt.clf() plt.imshow(logX) plt.savefig(os.path.join('plots', 'logX.pdf')) plt.clf() plt.imshow(fX * logX) plt.savefig(os.path.join('plots', 'fX.logX.pdf')) plt.clf() plt.imshow(embeddings @ embeddings.T) plt.savefig(os.path.join('plots', 'logX_.pdf')) plt.clf()
def load_data(rank): # Loading co-occurrence data. if rank == 0: print('Loading co-occurrence matrices...') logX = load_matrix(matrix_path + '.logx.npz') if rank == 0: print('Loaded logX.') fX = load_matrix(matrix_path + '.fx.npz') if rank == 0: print('Loaded fX.') type = 'sparse' if sparse else 'dense' if rank == 0: print(f'Using {type} cooccurence matrices during training.') if not sparse: logX = logX.todense() fX = fX.todense() return logX, fX
def real_matrix (): matrix_path = "./pickle/matrix/Abby_Watkins.matrix" if not os.path.exists(matrix_path): print "./pickle/matrix/Abby_Watkins.matrix ---- not exits" os.exits() matrix = load_matrix(matrix_path) return matrix
def load_graph(self, path): matrix = util.load_matrix(path) G = nx.Graph() length = len(matrix) for i in range(length): edges_list = [(i, j, matrix[i][j]) for j in range(i+1, length)] G.add_weighted_edges_from(edges_list) self.G = G return None
def load_graph(self, path): matrix = util.load_matrix(path) G = nx.Graph() length = len(matrix) for i in range(length): edges_list = [(i, j, matrix[i][j]) for j in range(i + 1, length)] G.add_weighted_edges_from(edges_list) self.G = G return None
def run(matrix_dir, svd_matrix_dir): if not os.path.exists(svd_matrix_dir): os.makedirs(svd_matrix_dir) for file_name in os.listdir(matrix_dir): name = file_name.split('.')[0] matrix_path = os.path.join(matrix_dir, file_name) matrix = util.load_matrix(matrix_path) svd_matrix = get_topicMatrix(np.array(matrix), len(matrix)) svd_path = os.path.join(svd_matrix_dir, '%s.matrix' % name) util.dump_matrix(svd_matrix, svd_path) return None
def main(): grid = [[]] grid[0].append(util.load_matrix("input.txt")) for i in range(6): print('.') nxt = step2(grid) if h(nxt) == h(grid): break grid = nxt print(h(grid).count("#"))
def run(matrix_dir, svd_matrix_dir): if not os.path.exists(svd_matrix_dir): os.makedirs(svd_matrix_dir) for file_name in os.listdir(matrix_dir): name = file_name.split('.')[0] matrix_path = os.path.join(matrix_dir, file_name) matrix = util.load_matrix(matrix_path) svd_matrix = get_topicMatrix(np.array(matrix), len(matrix)) svd_path = os.path.join(svd_matrix_dir, '%s.matrix' % name) util.dump_matrix(svd_matrix, svd_path) return None
def main(): grid = [] grid.append(util.load_matrix("input.txt")) #while True: for i in range(6): nxt = step2(grid) if h(nxt) == h(grid): break grid = nxt print(h(grid).count("#"))
def run(matrix_dir, cosine_dir, similarity_method): util.makedir(cosine_dir) count = 0 for file_name in os.listdir(matrix_dir): name = file_name.split('.')[0] count += 1 util.write('begin %s: %s' % (count, name)) file_path = os.path.join(matrix_dir, file_name) matrix = util.load_matrix(file_path) # sim_matrix = cosine(matrix) sim_matrix = compute_similarity(matrix, similarity_method) cosine_path = os.path.join(cosine_dir, '%s.matrix' % name) util.dump_matrix(sim_matrix, cosine_path) return None
def run(matrix_dir, category_dir): if not os.path.exists(category_dir): os.makedirs(category_dir) clusterer = GAAClusterer() count = 0 for file_name in os.listdir(matrix_dir): name = file_name.split('.')[0] count += 1 print 'begin %s: %s' % (count, name) file_path = os.path.join(matrix_dir, file_name) matrix = util.load_matrix(file_path) np_matrix = [np.array(row) for row in matrix] print np_matrix result = clusterer.cluster(np_matrix, False, "euc", "mean") category_path = os.path.join(category_dir, '%s.pickle' % name) with open(category_path, 'wb') as fp: pickle.dump(result, fp) return None
def run(matrix_dir, category_dir): if not os.path.exists(category_dir): os.makedirs(category_dir) clusterer = AD_Cluster() count = 0 for file_name in os.listdir(matrix_dir): name = file_name.split('.')[0] count += 1 print 'begin %s: %s' % (count, name) file_path = os.path.join(matrix_dir, file_name) matrix = util.load_matrix(file_path) size = len(matrix) result = [(9999,)]*size # 9999 号类表示 discard, 初始默认所有样本为 discard 状态 realPTs, noisePTs,tmp,angles,real,noise= cutNoise(matrix) print "len of real",len(real) print "len of realPTs",len(realPTs) # --------------------噪声自成一类------------------------- cNum = 0 # 记录已使用类标签数量 for i in noise: result[i] = (cNum,) cNum += 1 # ------------------------------------------------------- np_matrix = [np.array(row) for row in realPTs] print np_matrix result0 = clusterer.cluster(np_matrix, False,None, "euc", "mean") print "len of result0",len(result0) print "len of result",len(result) # ---------------- 聚类结果标签加上原来已用的类标数 ---------- for i, c in enumerate(result0): result[real[i]] = (c[0] + cNum,) category_path = os.path.join(category_dir, '%s.pickle' % name) with open(category_path, 'wb') as fp: pickle.dump(result, fp) # if count > 5: # break return None
def run(matrix_dir, category_dir): if not os.path.exists(category_dir): os.makedirs(category_dir) clusterer = AD_Cluster() count = 0 for file_name in os.listdir(matrix_dir): name = file_name.split('.')[0] count += 1 print 'begin %s: %s' % (count, name) file_path = os.path.join(matrix_dir, file_name) matrix = util.load_matrix(file_path) size = len(matrix) result = [(9999, )] * size # 9999 号类表示 discard, 初始默认所有样本为 discard 状态 realPTs, noisePTs, tmp, angles, real, noise = cutNoise(matrix) print "len of real", len(real) print "len of realPTs", len(realPTs) # --------------------噪声自成一类------------------------- cNum = 0 # 记录已使用类标签数量 for i in noise: result[i] = (cNum, ) cNum += 1 # ------------------------------------------------------- np_matrix = [np.array(row) for row in realPTs] print np_matrix result0 = clusterer.cluster(np_matrix, False, None, "euc", "mean") print "len of result0", len(result0) print "len of result", len(result) # ---------------- 聚类结果标签加上原来已用的类标数 ---------- for i, c in enumerate(result0): result[real[i]] = (c[0] + cNum, ) category_path = os.path.join(category_dir, '%s.pickle' % name) with open(category_path, 'wb') as fp: pickle.dump(result, fp) # if count > 5: # break return None
def demo(): from util import load_matrix,draw_line,draw_2D_noise points = load_matrix("./pickle/matrix/Alice_Gilbreath.matrix") # points = noise_2D_vector_rect(200, 3) # points.append((0,0)) # points.append((0,0)) vectors = [list(e) for e in points] realPTs, noisePTs,tmp,angles,real,noise = cutNoise(vectors) print "realPTs",len(realPTs),"real",len(real) print "noisePTs,",len(noisePTs),"noise",len(noise) print tmp print angles draw_line(tmp) draw_line(angles) draw_2D_noise(realPTs, noisePTs) cluster1 = AD_Cluster() # vectors0 = [np.array(f) for f in points] vectors1 = [np.array(f) for f in realPTs]
def main(args): if torch.cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') print('Device:', device) # Set seed for reproducibility. torch.manual_seed(args.seed) np.random.seed(args.seed) # Construct paths. vocab_path = os.path.join(args.vocab_dir, f'{args.name}.vocab') matrix_path = os.path.join(args.matrix_dir, f'{args.name}') model_path = os.path.join(args.model_dir, f'{args.name}.model.dict') log_path = os.path.join(args.log_dir, 'losses.csv') out_path = os.path.join(args.out_dir, args.name) # Load vocabulary. w2i, i2w = load_vocabulary(vocab_path) vocab_size = len(i2w) print(f'Loaded vocabulary of size {vocab_size}.') sparse = bool(vocab_size > 20000) # Load co-occurrence data. print('Loading co-occurrence matrices...') logX = load_matrix(matrix_path + '.logx.npz') print('Loaded logX.') fX = load_matrix(matrix_path + '.fx.npz') print('Loaded fX.') type = 'sparse' if sparse else 'dense' print(f'Using {type} cooccurence matrices during training.') if not sparse: logX = logX.todense() fX = fX.todense() # Construct model and optimizer. model = GloVe(vocab_size=vocab_size, emb_dim=args.emb_dim, sparse=True).to(device) optimizer = torch.optim.SparseAdam(model.parameters(), lr=args.lr) print('Training...') losses = [] logs = [('step', 'loss')] lr = args.lr epoch = 0 prev_loss = np.inf t0 = time.time() try: for step in range(1, args.num_updates + 1): # Sample a random batch. idx = np.random.randint(0, high=vocab_size, size=(args.batch_size, )) indices = torch.LongTensor(idx).to(device) submat = np.ix_(idx, idx) # used to select the submatrix if sparse: logx = logX[submat].todense() weights = fX[submat].todense() else: logx = logX[submat] weights = fX[submat] logx = torch.FloatTensor(logx).to(device) weights = torch.FloatTensor(weights).to(device) # Forward pass loss = model(indices, logx, weights) del indices, logx, weights # free some memory # Update parameters optimizer.zero_grad() loss.backward() optimizer.step() # Bookkeeping losses.append(loss.item()) if step % args.print_every == 0: ls = losses[-args.print_every:] avg_loss = sum(ls) / args.print_every logs.append((step, avg_loss)) print( '| epoch {:4d} | step {:6d} | loss {:.4f} | pairs/sec {:.1f} | lr {:.1e}' .format( epoch, step, avg_loss, args.print_every * args.batch_size / (time.time() - t0), lr)) t0 = time.time() if args.use_schedule: if avg_loss >= prev_loss: lr /= 4.0 optimizer = torch.optim.SparseAdam(model.parameters(), lr=lr) prev_loss = avg_loss if step % args.save_every == 0: with open(log_path, 'w') as f: writer = csv.writer(f) writer.writerows(logs) k, _ = divmod(step * args.batch_size, vocab_size) if k > epoch: epoch = k except KeyboardInterrupt: print('-' * 89) print('Exiting from training early') # Get the learned embeddings from the model. embeddings = model.embedding.weight.data.cpu().numpy() if args.gensim_format: out_path += f'.{args.emb_dim}d.gensim.txt' else: out_path += f'.{args.emb_dim}d.txt' print(f'Writing vectors to `{out_path}`...') write_vectors(embeddings, out_path, args.emb_dim, i2w, gensim=True)
def main(args): """create word vector :param file_path: path of corpus :param window_size: window size :param shift: num of samples in w2v skip-gram negative-sampling(sgns) :param dim: the size of wordvec WV = [vocab_size, dim] """ logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO) logging.info(f"[INFO] args: {args}") logging.info("[INFO] Loading dictionary...") id_to_word, word_to_id = load_pickle(args.pickle_id2word) vocab_size = len(id_to_word) logging.debug(f"[DEBUG] vocab: {vocab_size} words") if args.cooccur_pretrained is not None: logging.info("[INFO] Loading pre-trained co-occur matrix...") C = load_matrix(args.cooccur_pretrained, len(id_to_word)) else: logging.info("[INFO] Creating co-occur matrix...") C = create_co_matrix(args.file_path, word_to_id, vocab_size, args.window_size) # threshold by min_count if args.threshold: C = threshold_cooccur(C, threshold=args.threshold) os.makedirs("model", exist_ok=True) c_name = "model/C_w-{}".format(args.window_size) with open(c_name, "w") as wp: for id, cooccur_each in enumerate(C): cooccur_nonzero = [ f"{id}:{c}" for id, c in enumerate(cooccur_each) if c > 0 ] wp.write(f"{id}\t{' '.join(cooccur_nonzero)}\n") if args.sppmi_pretrained is not None: logging.info("[INFO] Loading pre-trained sppmi matrix...") M = load_matrix(args.sppmi_pretrained, len(id_to_word)) else: logging.info("[INFO] Computing sppmi matrix...") # use smoothing or not in computing sppmi M = sppmi(C, args.shift, has_abs_dis=args.has_abs_dis, has_cds=args.has_cds) m_name = "model/SPPMI_w-{}_s-{}".format(args.window_size, args.shift) with open(m_name, "w") as wp: for id, sppmi_each in enumerate(M): sppmi_nonzero = [ f"{id}:{m}" for id, m in enumerate(sppmi_each) if m > 0 ] wp.write(f"{id}\t{' '.join(sppmi_nonzero)}\n") logging.info("[INFO] Calculating word vector...") try: from scipy.sparse.linalg import svds U, S, V = svds(coo_matrix(M), k=args.dim) except: U, S, V = np.linalg.svd(coo_matrix(M)) word_vec = np.dot(U, np.sqrt(np.diag(S))) wv_name = "model/WV_d-{}_w-{}_s-{}".format(args.dim, args.window_size, args.shift) np.save(wv_name, word_vec[:, :args.dim]) return