else:
continue
if ( m_score > 0.0 ):
machine_score.append(m_score)
human_score.append(float(p[2]))
p_val, p_rel = sci.stats.spearmanr(human_score, machine_score)
print "Simple Linear Approach", p_val
if __name__ == "__main__":
word_vectors = nlp.read_word_vectors(VECTOR_DIR + VECTOR_NAME)
word_pairs = nlp.read_csv(CSV_DIR + CSV_NAME)
vocab = []
for p in word_pairs:
vocab.append(p[0].lower())
vocab.append(p[1].lower())
vocab = list(set(vocab))
for w in vocab:
word_hypernyms[w] = nlp.read_hypernyms(w)
word_hyponyms[w] = nlp.read_hyponyms(w)
word_synonyms[w] = nlp.read_synonyms(w)
senses[w] = nlp.read_senses(w)
for s in senses[w]:
sense_vectors[s] = np.zeros(VECTOR_DIM)
sense_hypernyms[s] = nlp.read_hypernyms_by_sense(s)
sense_hyponyms[s] = nlp.read_hyponyms_by_sense(s)
sense_synonyms[s] = nlp.read_synonyms_by_sense(s)
sense_vectors[s] = nlp.get_pooling(s, sense_hypernyms,sense_synonyms,sense_hyponyms, word_vectors, VECTOR_DIM)
if ( word_vectors.has_key(w)):
sense_vectors[s] = sense_vectors[s] + word_vectors[w]
word_pool[w] = nlp.get_pooling(w, word_hypernyms, word_synonyms, word_hyponyms, word_vectors, VECTOR_DIM)
test_sense_vectors()
else:
up_time = 0
# learning_rate = learning_rate - 0.005
if (p_rel >= 80.0 or up_time > 4):
break
print "this time error", sum_error
print "former time error", former_sum_error
print "up_time", up_time
print "learning_rate", learning_rate
print "pearson", sci.stats.pearsonr(human_s, machine_s)
former_sum_error = sum_error
if __name__ == "__main__":
print "read vector & score"
word_vectors = nlplib.read_word_vectors("./test_vector/100_3.vec")
word_pair_score = nlplib.read_csv("./csv/R&G-65.csv")
######Read hypernyms
print "get hypernyms etc."
for w_pair_score in word_pair_score:
word_dictionary.append(w_pair_score[0])
word_dictionary.append(w_pair_score[1])
# remove dumplicated word, for we are searching hypernyms, hyponyms, synonyms according to the dictionary.
word_dictionary = list(set(word_dictionary))
for w in word_dictionary:
word_hypernyms[w] = nlplib.read_hypernyms(w)
word_hyponyms[w] = nlplib.read_hyponyms(w)
word_synonyms[w] = nlplib.read_synonyms(w)
print "start training"
cnn_training()
result = 0.5 * result
return result
if __name__ == "__main__":
print "VECTOR:", VECTOR_NAME
word_vectors = nlp.read_word_vectors(VECTOR_DIR + VECTOR_NAME)
word_pairs = nlp.read_csv(CSV_DIR + CSV_NAME)
print "CSV", CSV_NAME
vocab = []
for p in word_pairs:
vocab.append(p[0].lower())
vocab.append(p[1].lower())
vocab = list(set(vocab))
for w in vocab:
word_hypernyms[w] = nlp.read_hypernyms(w)
word_hyponyms[w] = nlp.read_hyponyms(w)
word_synonyms[w] = nlp.read_synonyms(w)
senses[w] = nlp.read_senses(w)
for s in senses[w]:
sense_vectors[s] = np.zeros(VECTOR_DIM)
sense_hypernyms[s] = nlp.read_hypernyms_by_sense(s)
sense_hyponyms[s] = nlp.read_hyponyms_by_sense(s)
sense_synonyms[s] = nlp.read_synonyms_by_sense(s)
sense_pool[s] = nlp.get_pooling(s, sense_hypernyms, sense_synonyms,
sense_hyponyms, word_vectors,
VECTOR_DIM)
if (word_vectors.has_key(w)):
sense_pool[s] = sense_pool[s] + word_vectors[w]
sense_vectors[s] = sense_pool[s]
for l in s.lemmas():
vocab = list(set(vocab))
word_vectors = nlp_lib.read_word_vectors(VECTOR_DIR + VECTOR_NAME)
#read word senses
for w in vocab:
word_senses[w] = nlp_lib.read_senses(w)
#read word senses' hyponyms
for w in vocab:
for s in word_senses[w]:
word_sense_hyponyms[s] = nlp_lib.read_hyponyms_by_sense(s)
word_sense_hypernyms[s] = nlp_lib.read_hypernyms_by_sense(s)
word_sense_synonyms[s] = nlp_lib.read_synonyms_by_sense(s)
word_sense_full_pooling[s] = get_full_pooling_sense(s, w)
#read for retrofitting
for w in vocab:
word_hyponyms[w] = nlp_lib.read_hyponyms(w)
word_hypernyms[w] = nlp_lib.read_hypernyms(w)
word_synonyms[w] = nlp_lib.read_synonyms(w)
for w in vocab:
print "word",w
if ( word_vectors.has_key(w) ):
print "has a vector"
else:
print "no vector for it"
for w in vocab:
word_final_vectors[w] = get_pooling(w)
training_sense_vectors()
#calculate pearson similarity
test_sense_vectors()
