def main(args):
# location of input-output
data_dir = args.input
train_loc = os.path.join(data_dir, "train.csv")
test_loc = os.path.join(data_dir, "test.csv")
train = pd.read_csv(train_loc)
test = pd.read_csv(test_loc)
nlp = spacy.load("en_core_web_lg")
def extract_bow(data, text_col, id_col, uniq_tokens=None):
documents = {}
sent = {}
if uniq_tokens is None:
uniq_tokens = {}
for i, line in tqdm(data.iterrows(), total=data.shape[0]):
# TODO: remove after debugging
sent[line[id_col]] = line[text_col]
if i == 1000:
# TODO: remove after experiments
break
text = nlp(line[text_col])
tokens = [t for t in text if t.is_alpha and not t.is_stop]
orths = {t.text: t.orth for t in tokens}
words = Counter(t.text for t in tokens if t.text in nlp.vocab)
sorted_words = sorted(words)
documents[line[id_col]] = (line[id_col], [
orths[t] for t in sorted_words
], np.array([words[t] for t in sorted_words], dtype=np.float32))
return documents, uniq_tokens, sent
tid1_nlp, uniq_tokens, tid1_sent = extract_bow(train,
text_col="title1_en",
id_col="tid1")
tid2_nlp, uniq_tokens, tid2_sent = extract_bow(train,
text_col="title2_en",
id_col="tid2",
uniq_tokens=uniq_tokens)
class SpacyEmbeddings(object):
def __getitem__(self, item):
return nlp.vocab[item].vector
from wmd import TailVocabularyOptimizer
tid1_calc = WMD(SpacyEmbeddings(),
tid1_nlp,
vocabulary_min=10,
vocabulary_optimizer=TailVocabularyOptimizer(1.))
tid2_calc = WMD(SpacyEmbeddings(), tid2_nlp, vocabulary_min=3)
def word_mover_distance(word_embedding_dict_source,
word_embedding_dict_target):
""" Calculate euclidean distance between two dictionaries of arrays.
"""
try:
source = np.array(word_embedding_dict_source, dtype=np.float32)
target = np.array(word_embedding_dict_target, np.float32)
embeddings = np.concatenate((source, target))
source_len = source.shape[0]
target_len = target.shape[0]
source_words = np.array([i for i in range(source_len)],
dtype=np.int32)
target_words = np.array(
[source_len + i for i in range(target_len)], dtype=np.int32)
source_weights = np.array([1 for i in range(source_len)],
dtype=np.int32)
target_weights = np.array([1 for i in range(target_len)],
dtype=np.int32)
nbow = {
"source": ("source", source_words, source_weights),
"target": ("target", target_words, target_weights)
}
calc = WMD(embeddings, nbow, vocabulary_min=2)
return calc.nearest_neighbors("source", 1)[0][1]
except (ValueError, IndexError):
return 0
def calc_smd(input_f, output_f="", WORD_REP='elmo', METRIC='sms'):
if WORD_REP == "elmo":
MODEL = ElmoEmbedder()
inF = open(input_f, 'r')
inLines = inF.readlines()
inF.close()
#print("Found", len(inLines), "documents")
token_doc_list, text_doc_list = tokenize_texts(inLines, WORD_REP, tokenize=True)
count = 0
results_list = []
for doc_id in range(len(token_doc_list)):
doc = token_doc_list[doc_id]
text = text_doc_list[doc_id]
# transform doc to ID list, both words and/or sentences. get ID dict that maps to emb
[ref_ids, hyp_ids], rep_map = get_embeddings(doc, text, WORD_REP, MODEL, METRIC)
# get D values
[ref_id_list, hyp_id_list], [ref_d, hyp_d] = get_weights([ref_ids, hyp_ids], METRIC)
# format doc as expected: {id: (id, ref_id_list, ref_d)}
doc_dict = {"0": ("ref", ref_id_list, ref_d), "1": ("hyp", hyp_id_list, hyp_d)}
calc = WMD(rep_map, doc_dict, vocabulary_min=1)
try:
dist = calc.nearest_neighbors(str(0), k=1, early_stop=1)[0][1] # how far is hyp from ref?
except Exception as e:
print(e)
sim = math.exp(-dist) # switch to similarity
results_list.append(sim)
if doc_id == int((len(token_doc_list) / 10.) * count):
print(str(count * 10) + "% done with calculations")
count += 1
if output_f != "":
print_score(inLines, output_f, results_list)
else:
print("Results: ", np.mean(results_list))
return 'Done!'
def compute_q(self, f_df, q_df, return_f_nbow=False):
logger.info('Computing question wmds')
f_nbow = {
row.Index: self.nbowify(row.Index, row.original)
for row in f_df.itertuples()
}
nb_facts = len(f_nbow)
q_nbow = {
row.Index + nb_facts: self.nbowify(row.Index + nb_facts,
row.original)
for row in q_df.itertuples()
}
merged_fnbow = copy.copy(f_nbow)
merged_fnbow.update(q_nbow)
q_calc = WMD(SpacyEmbeddings(self.nlp),
merged_fnbow,
vocabulary_min=1,
verbosity=logging.WARNING)
q_calc.cache_centroids()
q_closest = pd.Series(
np.array([
i for i, _ in q_calc.nearest_neighbors(
idx, k=self.config.nearest_k_visible) if i < nb_facts
]) for idx in tqdm(q_nbow.keys(), desc='Question wmd...'))
return (q_closest, f_nbow) if return_f_nbow else q_closest
def get_similar_bugs(self, query):
query = self.text_preprocess(self.get_text(query))
words = [
word for word in set(chain(query, *self.corpus))
if word in self.w2vmodel.wv
]
indices, words = zip(*sorted(((index, word) for (
index, _), word in zip(self.dictionary.doc2bow(words), words))))
query = dict(self.tfidf[self.dictionary.doc2bow(query)])
query = [(new_index, query[dict_index])
for new_index, dict_index in enumerate(indices)
if dict_index in query]
documents = [
dict(self.tfidf[self.dictionary.doc2bow(document)])
for document in self.corpus
]
documents = [[(new_index, document[dict_index])
for new_index, dict_index in enumerate(indices)
if dict_index in document] for document in documents]
embeddings = np.array([self.w2vmodel.wv[word] for word in words],
dtype=np.float32)
nbow = dict(((index, list(chain([None], zip(*document))))
for index, document in enumerate(documents)
if document != []))
nbow["query"] = tuple([None] + list(zip(*query)))
distances = WMD(embeddings, nbow,
vocabulary_min=1).nearest_neighbors("query")
return [
self.bug_ids[distance[0]] for distance in distances
if self.bug_ids[distance[0]] != query["id"]
]
def get_distance(self, query1, query2):
query1 = self.text_preprocess(self.get_text(query1))
query2 = self.text_preprocess(self.get_text(query2))
words = [
word for word in set(chain(query1, query2, *self.corpus))
if word in self.w2vmodel.wv
]
indices, words = zip(*sorted(((index, word) for (
index, _), word in zip(self.dictionary.doc2bow(words), words))))
query1 = dict(self.tfidf[self.dictionary.doc2bow(query1)])
query2 = dict(self.tfidf[self.dictionary.doc2bow(query2)])
query1 = [(new_index, query1[dict_index])
for new_index, dict_index in enumerate(indices)
if dict_index in query1]
query2 = [(new_index, query2[dict_index])
for new_index, dict_index in enumerate(indices)
if dict_index in query2]
embeddings = np.array([self.w2vmodel.wv[word] for word in words],
dtype=np.float32)
nbow = {}
nbow["query1"] = tuple([None] + list(zip(*query1)))
nbow["query2"] = tuple([None] + list(zip(*query2)))
distances = WMD(embeddings, nbow,
vocabulary_min=1).nearest_neighbors("query1")
return distances[0][1]
def calculate_similarity(candidate, next_id, emb):
s = time.time()
can_doc = calculator.nlp(candidate[essay_field])
similarities = []
next_id, emb, can_id_list, can_weights = calculator.get_embeddings_ids_weights(
can_doc, next_id, emb, method)
nbow = {"hypothesis": ("hypothesis", can_id_list, can_weights)}
for id, item in processed_refs.items():
ref_weights = item["weights"]
ref_id_list = item["id_list"]
nbow[id] = (id, ref_id_list, ref_weights)
calc = WMD(emb, nbow, vocabulary_min=1)
# print("NBOW")
# print(nbow)
distances = calc.nearest_neighbors("hypothesis",
k=len(processed_refs),
early_stop=1)
for id, dist in distances:
similarity = np.exp(-dist)
similarities.append({
"candidate_id": candidate[id_field],
"reference_id": id,
"similarity": similarity,
"dist": dist,
"score": candidate[score_field]
})
print("Time taken for candidate " + str(candidate[id_field]) + " is " +
str(time.time() - s))
return similarities
def get_similarity_dist(self, candidate, reference, method):
emb, nbow = self.get_emb_nbow(candidate, reference, method)
# print("emb:", emb.keys())
# print("nbow:", nbow)
calc = WMD(emb, nbow, vocabulary_min=1)
dist = calc.nearest_neighbors("reference", k=1, early_stop=1)
# print("Dist:", dist)
dist = dist[0][1]
similarity = np.exp(-dist)
return similarity, dist
def __init__(self,
id2vec=None,
df=None,
nbow=None,
verbosity=logging.DEBUG,
wmd_cache_centroids=True,
wmd_kwargs=None,
gcs_bucket=None,
repo2nbow_kwargs=None,
initialize_environment=True):
if initialize_environment:
initialize()
self._log = logging.getLogger("similar_repos")
self._log.setLevel(verbosity)
if gcs_bucket:
backend = create_backend(args="bucket=" + gcs_bucket)
else:
backend = create_backend()
if id2vec is None:
self._id2vec = Id2Vec(log_level=verbosity, backend=backend)
else:
assert isinstance(id2vec, Id2Vec)
self._id2vec = id2vec
self._log.info("Loaded id2vec model: %s", self._id2vec)
if df is None:
if df is not False:
self._df = DocumentFrequencies(log_level=verbosity,
backend=backend)
else:
self._df = None
self._log.warning("Disabled document frequencies - you will "
"not be able to query custom repositories.")
else:
assert isinstance(df, DocumentFrequencies)
self._df = df
self._log.info("Loaded document frequencies: %s", self._df)
if nbow is None:
self._nbow = NBOW(log_level=verbosity, backend=backend)
else:
assert isinstance(nbow, NBOW)
self._nbow = nbow
self._log.info("Loaded nBOW model: %s", self._nbow)
self._repo2nbow = Repo2nBOW(self._id2vec,
self._df,
log_level=verbosity,
**(repo2nbow_kwargs or {}))
self._log.info("Creating the WMD engine...")
self._wmd = WMD(self._id2vec.embeddings,
self._nbow,
verbosity=verbosity,
**(wmd_kwargs or {}))
if wmd_cache_centroids:
self._wmd.cache_centroids()
def get_sim(doc, text, wordrep, model, metric):
[ref_ids, hyp_ids], rep_map = get_embeddings(doc, text, wordrep, model, metric)
[ref_id_list, hyp_id_list], [ref_d, hyp_d] = get_weights([ref_ids, hyp_ids], metric)
# format doc as expected: {id: (id, ref_id_list, ref_d)}
doc_dict = {"0": ("ref", ref_id_list, ref_d), "1": ("hyp", hyp_id_list, hyp_d)}
calc = WMD(rep_map, doc_dict, vocabulary_min=1)
try:
dist = calc.nearest_neighbors(str(0), k=1, early_stop=1)[0][1] # how far is hyp from ref?
except:
return 0.0
sim = math.exp(-dist) # switch to similarity
return sim
def calc_smd(inLines, model):
global nlp
nlp = model
#print("Found", len(inLines), "documents")
# TODO: rewrite this
token_doc_list, text_doc_list = tokenize_texts(inLines)
results_list = []
for doc_id in range(len(token_doc_list)):
doc = token_doc_list[doc_id]
# TODO: rewrite this
text = text_doc_list[doc_id]
# transform doc to ID list, both words and/or sentences. get ID dict that maps to emb
# TODO: rewrite this
try:
[ref_ids, hyp_ids], rep_map = get_embeddings(doc, text)
except ValueError:
print(inLines[doc_id])
print('ValueError: max() arg is an empty sequence; get_embeddings')
continue
# get D values
[ref_id_list, hyp_id_list], [ref_d,
hyp_d] = get_weights([ref_ids, hyp_ids])
# format doc as expected: {id: (id, ref_id_list, ref_d)}
doc_dict = {
"0": ("ref", ref_id_list, ref_d),
"1": ("hyp", hyp_id_list, hyp_d)
}
calc = WMD(rep_map, doc_dict, vocabulary_min=1)
try:
dist = calc.nearest_neighbors(
str(0), k=1, early_stop=1)[0][1] # how far is hyp from ref?
sim = math.exp(-dist) # switch to similarity
except IndexError:
print(
'dist = calc.nearest_neighbors(str(0), k=1, early_stop=1)[0][1]'
)
print('IndexError: list index out of range')
print(inLines[doc_id])
continue
except UnboundLocalError:
print('dist could not be calculated')
print(inLines[doc_id])
continue
except ValueError:
print('Too little vocabulary')
print(inLines[doc_id])
continue
results_list.append((inLines[doc_id], sim))
return score_list(results_list)
def __init__(self, id2vec=None, df=None, nbow=None, prune_df_threshold=1,
verbosity=logging.DEBUG, wmd_cache_centroids=True, wmd_kwargs=None,
gcs_bucket=None, repo2nbow_kwargs=None, initialize_environment=True):
if initialize_environment:
initialize()
self._log = logging.getLogger("similar_repos")
self._log.setLevel(verbosity)
if gcs_bucket:
backend = create_backend(args="bucket=" + gcs_bucket)
else:
backend = create_backend()
if id2vec is None:
self._id2vec = Id2Vec(log_level=verbosity).load(backend=backend)
else:
assert isinstance(id2vec, Id2Vec)
self._id2vec = id2vec
self._log.info("Loaded id2vec model: %s", self._id2vec)
if df is None:
if df is not False:
self._df = DocumentFrequencies(log_level=verbosity).load(backend=backend)
else:
self._df = None
self._log.warning("Disabled document frequencies - you will "
"not be able to query custom repositories.")
else:
assert isinstance(df, DocumentFrequencies)
self._df = df
if self._df is not None:
self._df = self._df.prune(prune_df_threshold)
self._log.info("Loaded document frequencies: %s", self._df)
if nbow is None:
self._nbow = NBOW(log_level=verbosity).load(backend=backend)
else:
assert isinstance(nbow, NBOW)
self._nbow = nbow
self._log.info("Loaded nBOW model: %s", self._nbow)
self._repo2nbow = Repo2nBOW(
self._id2vec, self._df, log_level=verbosity, **(repo2nbow_kwargs or {}))
assert self._nbow.dep("id2vec")["uuid"] == self._id2vec.meta["uuid"]
if len(self._id2vec) != self._nbow.matrix.shape[1]:
raise ValueError("Models do not match: id2vec has %s tokens while nbow has %s" %
(len(self._id2vec), self._nbow.matrix.shape[1]))
self._log.info("Creating the WMD engine...")
self._wmd = WMD(self._id2vec.embeddings, self._nbow,
verbosity=verbosity, **(wmd_kwargs or {}))
if wmd_cache_centroids:
self._wmd.cache_centroids()
def __init__(self,
id2vec=None,
df=None,
nbow=None,
prune_df_threshold=1,
wmd_cache_centroids=True,
wmd_kwargs: Dict[str, Any] = None,
languages: Tuple[List, bool] = (None, False),
engine_kwargs: Dict[str, Any] = None):
backend = create_backend()
if id2vec is None:
self._id2vec = Id2Vec().load(backend=backend)
else:
assert isinstance(id2vec, Id2Vec)
self._id2vec = id2vec
self._log.info("Loaded id2vec model: %s", self._id2vec)
if df is None:
if df is not False:
self._df = DocumentFrequencies().load(backend=backend)
else:
self._df = None
self._log.warning("Disabled document frequencies - you will "
"not be able to query custom repositories.")
else:
assert isinstance(df, DocumentFrequencies)
self._df = df
if self._df is not None:
self._df = self._df.prune(prune_df_threshold)
self._log.info("Loaded document frequencies: %s", self._df)
if nbow is None:
self._bow = BOW().load(backend=backend)
else:
assert isinstance(nbow, BOW)
self._bow = nbow
self._log.info("Loaded BOW model: %s", self._bow)
assert self._bow.get_dep("id2vec")["uuid"] == self._id2vec.meta["uuid"]
if len(self._id2vec) != self._bow.matrix.shape[1]:
raise ValueError(
"Models do not match: id2vec has %s tokens while nbow has %s" %
(len(self._id2vec), self._bow.matrix.shape[1]))
self._log.info("Creating the WMD engine...")
self._wmd = WMD(self._id2vec.embeddings, self._bow, **(wmd_kwargs
or {}))
if wmd_cache_centroids:
self._wmd.cache_centroids()
self._languages = languages
self._engine_kwargs = engine_kwargs
def compute_f(self, f_df, f_nbow=None):
logger.info('Computing fact wmds')
f_nbow = {
row.Index: self.nbowify(row.Index, row.original)
for row in f_df.itertuples()
} if f_nbow is None else f_nbow
f_calc = WMD(SpacyEmbeddings(self.nlp),
f_nbow,
vocabulary_min=1,
verbosity=logging.WARNING)
f_calc.cache_centroids()
f_closest = pd.Series(
np.array([
i for i, _ in f_calc.nearest_neighbors(
idx, k=self.config.nearest_k_visible)
]) for idx in tqdm(f_nbow.keys(), desc='Fact wmd...'))
return f_closest
def fit_wme_model(self, d_max=6, r=1024):
self._r = r
possible_words = list(self.word_mapping)
nbow = {}
for i in range(r):
d = random.sample(range(1, d_max + 1), 1)[0]
random_doc = random.sample(possible_words, d)
doc_embeddings = [self.word_mapping[word] for word in random_doc]
document, idf_ids = zip(*[(word.glove_id, word.idf_id)
for word in doc_embeddings])
words = np.array(document, dtype=np.uint32)
idf_weights = np.array(
[self.tf_idf_model.idf_[idf_id] for idf_id in idf_ids],
dtype=np.float32)
weights = idf_weights
doc_id = '#' + str(i + 1)
nbow[doc_id] = (doc_id, words, weights)
self.wmd = WMD(embeddings=self.glove_model.word_vectors.astype(
np.float32),
nbow=nbow,
vocabulary_min=1)
def calc_smd(opts, output_f=""):
inF = open(opts.input_file, 'r')
inLines = inF.readlines()
inF.close()
print("Found", len(inLines), "documents")
token_doc_list, text_doc_list = tokenize_texts(inLines)
count = 0
results_list = []
for doc_id in range(len(token_doc_list)):
doc = token_doc_list[doc_id]
text = text_doc_list[doc_id]
# transform doc to ID list, both words and/or sentences. get ID dict that maps to emb
[ref_ids, hyp_ids], rep_map = get_embeddings(doc, text)
# get D values
[ref_id_list, hyp_id_list], [ref_d,
hyp_d] = get_weights([ref_ids, hyp_ids])
# format doc as expected: {id: (id, ref_id_list, ref_d)}
doc_dict = {
"0": ("ref", ref_id_list, ref_d),
"1": ("hyp", hyp_id_list, hyp_d)
}
calc = WMD(rep_map, doc_dict, vocabulary_min=1)
try:
dist = calc.nearest_neighbors(
str(0), k=1, early_stop=1)[0][1] # how far is hyp from ref?
except:
print(doc, text)
sim = math.exp(-dist) # switch to similarity
results_list.append(sim)
if doc_id == int((len(token_doc_list) / 10.) * count):
print(str(count * 10) + "% done with calculations")
count += 1
# added by wchen to compute correlation scores with human annotated scores
hscoreF = open(opts.score_file, 'r')
hscoreLines = hscoreF.readlines()
hscoreF.close()
compute_corrs(opts, results_list, hscoreLines)
def calc_smd(ref, hyp, model):
global nlp
nlp = model
doc, text = tokenize_texts([ref, hyp])
count = 0
results_list = []
# transform doc to ID list, both words and/or sentences. get ID dict that maps to emb
[ref_ids, hyp_ids], rep_map = get_embeddings(doc, text)
# get D values
[ref_id_list, hyp_id_list], [ref_d,
hyp_d] = get_weights([ref_ids, hyp_ids])
# format doc as expected: {id: (id, ref_id_list, ref_d)}
doc_dict = {
"0": ("ref", ref_id_list, ref_d),
"1": ("hyp", hyp_id_list, hyp_d)
}
calc = WMD(rep_map, doc_dict, vocabulary_min=1)
try:
dist = calc.nearest_neighbors(
str(0), k=1, early_stop=1)[0][1] # how far is hyp from ref?
except:
print(doc, text)
sim = math.exp(-dist) # switch to similarity
return sim
# List of page names we will fetch from Wikipedia and query for similarity
titles = sys.argv[1:] or ["Germany", "Spain", "Google"]
documents = {}
for title in titles:
print("fetching", title)
pages = requests.get(
"https://en.wikipedia.org/w/api.php?action=query&format=json&titles=%s"
"&prop=extracts&explaintext" % title).json()["query"]["pages"]
print("parsing", title)
text = nlp(next(iter(pages.values()))["extract"])
tokens = [t for t in text if t.is_alpha and not t.is_stop]
words = Counter(t.text for t in tokens)
orths = {t.text: t.orth for t in tokens}
sorted_words = sorted(words)
documents[title] = (title, [orths[t] for t in sorted_words],
numpy.array([words[t] for t in sorted_words],
dtype=numpy.float32))
# Hook in WMD
class SpacyEmbeddings(object):
def __getitem__(self, item):
return nlp.vocab[item].vector
calc = WMD(SpacyEmbeddings(), documents)
print("calculating")
# Germany shall be closer to Spain than to Google
for title, relevance in calc.nearest_neighbors(titles[0]):
print("%24s\t%s" % (title, relevance))
# pip3 install wmd
# https://github.com/src-d/wmd-relax
# 论文： http://www.cs.cornell.edu/~kilian/papers/wmd_metric.pdf
import time
import numpy
from wmd import WMD
import pickle
embeddings = numpy.array([[0.1, 1], [1, 0.1], [0.8, 0.7]], dtype=numpy.float32)
nbow = { # key: 序号， 向量， 权重；
"first": ("#1", [0, 1, 2], numpy.array([1.5, 0.3, 0.5],
dtype=numpy.float32)),
"你好": ("#3", [1, 2], numpy.array([1.3, 0.5], dtype=numpy.float32)),
"second": ("#2", [0, 1], numpy.array([0.75, 0.15], dtype=numpy.float32))
}
calc = WMD(embeddings, nbow, vocabulary_min=2)
origin = "first"
print(calc.nearest_neighbors(origin))
model_file = '/home/gswewf/yhb/model/wx_vector_char.pkl'
with open(model_file, "rb") as f:
w2v_model = pickle.load(f, encoding='iso-8859-1') # 此处耗内存 60.8 MiB
words_list = []
w_emb = []
for word, emb in w2v_model.items():
words_list.append(word)
w_emb.append(emb)
from jieba.analyse.tfidf import TFIDF
text = nlp(doctext)
tokens = [t for t in text if t.is_alpha and not t.is_stop]
words = Counter(t.text for t in tokens)
orths = {t.text: t.orth for t in tokens}
sorted_words = sorted(words)
documents[title] = (title, [orths[t] for t in sorted_words],
numpy.array([words[t] for t in sorted_words],
dtype=numpy.float32))
# Hook in WMD
class SpacyEmbeddings(object):
def __getitem__(self, item):
return nlp.vocab[item].vector
embeddings = SpacyEmbeddings()
vocabulary_min = 10
calc = WMD(embeddings, documents, vocabulary_min=vocabulary_min)
print("calculating")
# Germany shall be closer to Spain than to Google
neigbors_of_germany = calc.nearest_neighbors(titles[0])
for title, relevance in neigbors_of_germany:
print("%24s\t%s" % (title, relevance))