def main(argv): if (len(argv) != 2): print('usage: extract_embeddings.py datadir') exit(1) np.random.seed(123) data_dir = argv[1] fname_vocab = os.path.join(data_dir, 'vocab.pickle') alphabet = cPickle.load(open(fname_vocab)) words = alphabet.keys() print "Vocab size: ", len(alphabet) for fname in [ 'embeddings/aquaint+wiki.txt.gz.ndim=50.bin', ]: word2vec = load_bin_vec(fname, words) ndim = len(word2vec[word2vec.keys()[0]]) print 'embedding dim: ', ndim random_words_count = 0 vocab_emb = np.zeros((len(alphabet) + 1, ndim)) for word, idx in alphabet.iteritems(): word_vec = word2vec.get(word, None) if word_vec is None: word_vec = np.random.uniform(-0.25, 0.25, ndim) random_words_count += 1 vocab_emb[idx] = word_vec print "Using zero vector as random" print 'random_words_count', random_words_count print vocab_emb.shape outfile = os.path.join(data_dir, 'emb_{}.npy'.format(os.path.basename(fname))) print outfile np.save(outfile, vocab_emb)
def main(train, test, ngram, we): ''' Given output path (out), liblinear path (liblinear), Given ngram string rule (like "123"), ngram ''' global times times = 0 print 'loading data...' train_df = read_dataset(train) test_df = read_dataset(test) print 'cleaning data and add lex indicators...' train_df[['text_lex', 'lex_ws']] = train_df.apply(add_lex_indicator, axis=1) test_df[['text_lex', 'lex_ws']] = test_df.apply(add_lex_indicator, axis=1) print "using train to build pos and neg dic..." pos_dic, neg_dic = build_dict(train_df, ngram) print "computing log-count ratio r..." dic, r, v = compute_ratio(pos_dic, neg_dic) print 'loading word embedding...' word2vec = load_bin_vec(we) print "building train and test features --- ngram part..." train_df.sort_index(inplace=True) test_df.sort_index(inplace=True) X_train_ngram, y_train = process_files_ngram(train_df, dic, r, v, ngram) X_test_ngram, y_test = process_files_ngram(test_df, dic, r, v, ngram) print "building train and test features --- pos embedding part..." X_train_embed = process_files_wemb(train_df, word2vec) X_test_embed = process_files_wemb(test_df, word2vec) print "combining log-count ratio and pos embedding features..." train_f = sp.hstack((X_train_ngram, X_train_embed), format='csr') test_f = sp.hstack((X_test_ngram, X_test_embed), format='csr') print "running model..." basemodel = LogisticRegression() f_score = model_run(basemodel, train_f, test_f, y_train, y_test) print '##############f_score is: ', f_score print 'model ended.'
def main(): folder = sys.argv[1] print "Folder: %s" % folder np.random.seed(123) data_dirs = [ os.path.join(folder, 'TRAIN'), ] for data_dir in data_dirs: fname_vocab = os.path.join(data_dir, 'vocab.pickle') alphabet = cPickle.load(open(fname_vocab)) words = alphabet.keys() print "Vocab size", len(alphabet) for fname in [ 'embeddings/aquaint+wiki.txt.gz.ndim=50.bin', ]: word2vec = load_bin_vec(fname, words) ndim = len(word2vec[word2vec.keys()[0]]) print 'ndim', ndim random_words_count = 0 vocab_emb = np.zeros((len(alphabet) + 1, ndim)) for word, idx in alphabet.iteritems(): word_vec = word2vec.get(word, None) if word_vec is None: word_vec = np.random.uniform(-0.25, 0.25, ndim) random_words_count += 1 vocab_emb[idx] = word_vec print "Using zero vector as random" print 'random_words_count', random_words_count print vocab_emb.shape outfile = os.path.join( data_dir, 'emb_{}.npy'.format(os.path.basename(fname))) print outfile np.save(outfile, vocab_emb)
def find_keyword(args): print "loading data" if args.word_vec_file == '': w2v_file = 'GoogleNews-vectors-negative300.bin' keywords = load_words() vocab = keywords w2v = load_bin_vec(w2v_file, vocab) else: w2v = cPickle.load(open(args.word_vec_file, "r")) print "finish loading data" W = dict2Mat(w2v) kmeans = KMeans(n_clusters=args.keyword_num, random_state=0).fit(W) # save index to file cPickle.dump(kmeans.labels_, open(args.idx_save_file, "wb")) # get center vectors ctr_vecs = np.zeros(shape=(args.keyword_num, W.shape[1])) for i in range(args.keyword_num): ctr_vecs[i] = np.mean(W[kmeans.labels_ == i], axis=0) cPickle.dump(ctr_vecs, open('test.p', "wb")) print "center vecters saved" # save center words # get index of the closest vector to center vectors nbrs = NearestNeighbors(n_neighbors=1, algorithm=args.tree_algo).fit(W) distances, indices = nbrs.kneighbors(ctr_vecs) indices = np.reshape(indices, (len(indices))) # print words to file f_landmark = open(args.word_save_file, 'w') for i in range(args.keyword_num): print >> f_landmark, w2v.items()[indices[i]][0] f_landmark.close() print 'landmark words saved' # save words for vectors in W f_words = open(args.dict_file, 'w') for i in range(W.shape[0]): print >> f_words, w2v.items()[i][0] f_words.close() print 'words saved' print 'all done'
def dump_embedding(outdir, embeddingfile, alphabet): words = alphabet.keys() print "Vocab size: ", len(alphabet) word2vec = load_bin_vec(embeddingfile, words) ndim = len(word2vec[word2vec.keys()[0]]) print 'embedding dim: ', ndim random_words_count = 0 np.random.seed(321) vocab_emb = np.zeros((len(alphabet) + 1, ndim)) dummy_word_emb = np.random.uniform(-0.25, 0.25, ndim) for word, idx in alphabet.iteritems(): word_vec = word2vec.get(word, None) if word_vec is None: word_vec = np.random.uniform(-0.25, 0.25, ndim) #word_vec = dummy_word_emb #word_vec = np.zeros(ndim) random_words_count += 1 vocab_emb[idx] = word_vec print "Using zero vector as random" print 'random_words_count', random_words_count print 'vocab_emb.shape', vocab_emb.shape outfile = os.path.join(outdir, 'emb_{}.npy'.format(os.path.basename(embeddingfile))) print 'saving embedding file', outfile np.save(outfile, vocab_emb)
FLAGS.max_aspect_len, FLAGS.max_context_len, FLAGS.train_data, FLAGS.pre_processed) test_aspects, test_contexts, test_labels, test_aspect_lens, \ test_context_lens, test_aspect_texts, test_context_texts = load_data(FLAGS.test_fname, FLAGS.word2id, FLAGS.max_aspect_len, FLAGS.max_context_len, FLAGS.test_data, FLAGS.pre_processed) print('Loading pre-trained word vectors ...') if FLAGS.embedding == 'glove': FLAGS.word2vec = load_word_embeddings(FLAGS.embedding_fname, FLAGS.embedding_dim, FLAGS.word2id) else: FLAGS.word2vec = load_bin_vec(FLAGS.embedding_fname, FLAGS.embedding_dim, FLAGS.word2id) # Building lexicon embedding lex_list = get_lex_file_list(FLAGS.lex_path) train = zip(train_aspect_texts, train_context_texts) test = zip(test_aspect_texts, test_context_texts) lex = LexHelper(lex_list, train, test, max_aspect_len=FLAGS.max_aspect_len, max_context_len=FLAGS.max_context_len) train_context_lex, train_aspect_lex, test_context_lex, test_aspect_lex, FLAGS.lex_dim = lex.build_lex_embeddings( ) train_data = zip(train_aspects, train_contexts, train_labels, train_aspect_lens, train_context_lens, train_aspect_lex,
print word2dfs.items()[:10] ######### alphabet = Alphabet(start_feature_id=0) alphabet.add('UNKNOWN_WORD_IDX') add_to_vocab(answers, alphabet) add_to_vocab(questions, alphabet) basename = os.path.basename(train) cPickle.dump(alphabet, open(os.path.join(outdir, 'vocab.pickle'), 'w')) print "alphabet", len(alphabet) embeddings_location = 'embeddings/aquaint+wiki.txt.gz.ndim=50.bin' word2vec = load_bin_vec(embeddings_location, alphabet.keys()) dummy_word_idx = alphabet.fid q_max_sent_length = max(map(lambda x: len(x), questions)) a_max_sent_length = max(map(lambda x: len(x), answers)) print 'q_max_sent_length', q_max_sent_length print 'a_max_sent_length', a_max_sent_length # Convert dev and test sets for fname in [train, dev, test]: print fname # qids, questions, answers, labels = load_data(fname, stoplist) qids, questions, answers, labels = load_data(fname) overlap_feats = compute_overlap_features(questions,
def main(train, test, ngram, we): ''' Given output path (out), liblinear path (liblinear), Given ngram string rule (like "123"), ngram ''' global times times = 0 print 'loading data...' train_df = read_dataset(train) test_df = read_dataset(test) print 'cleaning data and add lex indicators...' train_df[['text_lex', 'lex_ws']] = train_df.apply(add_lex_indicator, axis=1) test_df[['text_lex', 'lex_ws']] = test_df.apply(add_lex_indicator, axis=1) print "using train to build token count dict for each class..." counters = build_counters(train_df, ngram) print "computing log-count ratio r..." dic, r, v = compute_ratios(counters) print 'loading word embedding...' word2vec = load_bin_vec(we) print "building train and test features --- ngram part..." train_df.sort_index(inplace=True) test_df.sort_index(inplace=True) y_train = train_df['y'] y_test = test_df['y'] X_train_ngram = process_files_ngram(train_df, dic, r, v, ngram) X_test_ngram = process_files_ngram(test_df, dic, r, v, ngram) print "building train and test features --- pos embedding part..." X_train_embed = process_files_wemb(train_df, word2vec) X_test_embed = process_files_wemb(test_df, word2vec) print "combining log-count ratio and pos embedding features..." X_train = sp.hstack((X_train_ngram, X_train_embed), format='csr') X_test = sp.hstack((X_test_ngram, X_test_embed), format='csr') print "running model..." basemod = LogisticRegression() #tunning paramater step especially for multiclass classifier c and class_weight cv = 10 param_grid = [{ 'C': [1, 0.1], 'class_weight': [{ 1: 1, -1: 1 }, { 1: 0.9, -1: 1 }, { 1: 1, -1: 0.9 }] }] f_score = model_run(basemod, param_grid, cv, X_train, X_test, y_train, y_test) print 'f_score is: ', f_score