cluster = []
# Read all sentence for this meaning
for j in range(0,int(num_sentences)):
word_pos = f.readline()
contexts = f.readline().split()
context_idxs = map2vocab(vocab, contexts)
example = word + '_' + str(i) + '.' + str(cnt)
test_data[word].append((example, word_idx, context_idxs))
cluster.append(example)
cnt += 1
ground_truth_clusters_dict[word][i] = cluster
line = f.readline()
#word_cnt += 1
#if word_cnt > 2: break
return ground_truth_clusters_dict, test_data
if __name__ == '__main__':
# read embedding files
embedding_filename = sys.argv[1]
context_embedding_filename = sys.argv[2]
print("Using embedding file: %s" % (embedding_filename))
vocab, embeddings = read_embedding_file(embedding_filename)
_, context_embeddings = read_embedding_file(context_embedding_filename)
ground_truth_clusters_dict, test_data = read_wwsi_file(vocab)
print("read wwsi file")
iw2v_clusters_dict = clean_clusters(get_clusters(test_data, vocab, embeddings, context_embeddings))
rand_index = avg_adj_rand_index(ground_truth_clusters_dict, iw2v_clusters_dict)
print 'avg_rand_index: ', rand_index
elif opt in ("-c", "--context"):
for a in arg.split('.'):
context_to_plot.append(a.split(','))
# extract dim and sparsity penalties from file names
arr = process_embeddings_dir(rootDir)
for input_embedding_file, context_embedding_file, sparsity, dim_penalty in arr:
print 'Context embeddings file: ', context_embedding_file
print 'Input embeddings file: ', input_embedding_file
print 'sparsity penalty: ', str(sparsity)
print 'dimension penalty: ', str(dim_penalty)
for i, c in enumerate(context_to_plot):
print str(i) + ': context to plot: ', " ".join(c)
print "loading embeddings and vocabulary..."
w_vocab, w_embeddings = read_embedding_file(input_embedding_file)
# reduce words to sum over
w_vocab = w_vocab[:k]
w_embeddings = w_embeddings[:k]
c_vocab, c_embeddings = read_embedding_file(context_embedding_file)
# if a sentence is specified, get embeddings
context_avg_vecs = []
context_all_vecs = []
for idx_s, s in context_to_plot:
word_counter = 0
for idx_w, word in enumerate(s):
if idx_w == 0:
context_avg_vecs.append(np.zeros(len(c_embeddings[0])))
context_all_vecs.append([])
vocab_idx = -1
try:
elif opt in ("-r", "--rootDir"):
rootDir = arg
elif opt in ("-w", "--words"):
words_to_plot = arg.split(',')
elif opt in ("-n", "--filename"):
filename = arg
# extract dim and sparsity penalties from file names
arr = process_embeddings_dir(rootDir)
for input_embedding_file, context_embedding_file, sparsity, dim_penalty in arr:
print 'Input embeddings file: ', input_embedding_file
print 'Context embeddings file: ', context_embedding_file
print 'words_to_plot: ', ", ".join(words_to_plot)
print 'sparsity penalty: ', str(sparsity)
print 'dimension penalty: ', str(dim_penalty)
print "loading embeddings and vocabulary..."
in_vocab_all, in_embeddings_all = read_embedding_file(
input_embedding_file)
in_vocab = in_vocab_all[:k]
in_embeddings = in_embeddings_all[:k]
out_vocab_all, out_embeddings_all = read_embedding_file(
context_embedding_file)
out_vocab = out_vocab_all[:k]
out_embeddings = out_embeddings_all[:k]
plot(in_vocab_all, in_embeddings_all, words_to_plot,
"heatmap_input.png")
plot(in_vocab_all, out_embeddings_all, words_to_plot,
"heatmap_context.png")
for opt, arg in opts:
if opt == '-h':
print help_message
sys.exit()
elif opt in ("-i", "--ifile"):
input_embedding_file = arg
elif opt in ("-w", "--words"):
words_to_plot = arg.split(',')
elif opt in ("-k", "--numNeighbors"):
num_of_nns_to_get = int(arg)
print 'Input embeddings file: ', input_embedding_file
print 'words_to_plot: ', ", ".join(words_to_plot)
print "loading embeddings and vocabulary..."
in_vocab, in_embeddings = read_embedding_file(input_embedding_file)
in_vocab = in_vocab[:k]
in_embeddings = in_embeddings[:k]
d = len(in_embeddings[0])
for plot_idx, word_to_plot in enumerate(words_to_plot):
in_word_idx = in_vocab.index(word_to_plot)
word_in_embedding = in_embeddings[in_word_idx]
nn_idxs = get_nearest_neighbors(word_in_embedding, in_word_idx,
in_embeddings, num_of_nns_to_get)
t = []
for i, idx in enumerate(nn_idxs):
t.append([i + 1, in_vocab[idx]])