def eval_analogies(frame): filename = get_support_data_filename('google-analogies/questions-words.txt') quads = read_google_analogies(filename) vocab = [ standardized_uri('en', word) for word in wordfreq.top_n_list('en', 200000) ] wrap = VectorSpaceWrapper(frame=frame) vecs = np.vstack([wrap.get_vector(word) for word in vocab]) tframe = pd.DataFrame(vecs, index=vocab) total = 0 correct = 0 seen_mistakes = set() for quad in quads: prompt = quad[:3] answer = quad[3] vector = analogy_func(frame, *prompt) similar = similar_to_vec(tframe, vector) result = None for match in similar.index: if match not in prompt: result = match break if result == answer: correct += 1 else: if result not in seen_mistakes: print( "%s : %s :: %s : [%s] (should be %s)" % (quad[0], quad[1], quad[2], result, answer) ) seen_mistakes.add(result) total += 1 low, high = proportion_confint(correct, total) return pd.Series([correct / total, low, high], index=['acc', 'low', 'high'])
def __init__(self, ablate=()): self.wrap = VectorSpaceWrapper( get_external_data_filename('numberbatch-20180108-biased.h5'), use_db=False) self.cache = {} self.wp_db = None self.sme = None self.queries = None self.phrases = None self.svm = None self.ablate = ablate self.feature_methods = [ self.direct_relatedness_features, self.sme_features, self.wikipedia_relatedness_features, self.wordnet_relatedness_features, self.phrase_hit_features ] self.feature_names = [ 'ConceptNet vector relatedness', 'SME: RelatedTo', 'SME: (x IsA a)', 'SME: (x HasA a)', 'SME: (x PartOf a)', 'SME: (x CapableOf a)', 'SME: (x UsedFor a)', 'SME: (x HasContext a)', 'SME: (x HasProperty a)', 'SME: (x AtLocation a)', 'SME: (a PartOf x)', 'SME: (a AtLocation x)', 'Wikipedia lead sections', 'WordNet relatedness', 'Google Ngrams', ]
def evaluate(frame, subset='val'): """ Evaluate a DataFrame containing term vectors on its ability to predict term relatedness, according to MEN-3000, RW, MTurk-771, and WordSim-353. Use a VectorSpaceWrapper to fill missing vocabulary from ConceptNet. Return a Series containing these labeled results. """ # Make subset names consistent with other datasets if subset == 'dev': subset = 'val' elif subset == 'all': # for the final evaluation, use just the test data subset = 'test' filename = get_support_data_filename('story-cloze/cloze_test_spring2016_%s.tsv' % subset) vectors = VectorSpaceWrapper(frame=frame) total = 0 correct = 0 for sentences, answers in read_cloze(filename): text = ' '.join(sentences) right_answer, wrong_answer = answers probe_vec = vectors.text_to_vector('en', text) right_vec = vectors.text_to_vector('en', right_answer) wrong_vec = vectors.text_to_vector('en', wrong_answer) right_sim = cosine_similarity(probe_vec, right_vec) wrong_sim = cosine_similarity(probe_vec, wrong_vec) if right_sim > wrong_sim: correct += 1 total += 1 # print("%+4.2f %s / %s / %s" % (right_sim - wrong_sim, text, right_answer, wrong_answer)) low, high = proportion_confint(correct, total) return pd.Series([correct / total, low, high], index=['acc', 'low', 'high'])
def evaluate(frame, subset='val'): """ Evaluate a DataFrame containing term vectors on its ability to predict term relatedness, according to MEN-3000, RW, MTurk-771, and WordSim-353. Use a VectorSpaceWrapper to fill missing vocabulary from ConceptNet. Return a Series containing these labeled results. """ # Make subset names consistent with other datasets if subset == 'dev': subset = 'val' elif subset == 'all': # for the final evaluation, use just the test data subset = 'test' filename = get_support_data_filename( 'story-cloze/cloze_test_spring2016_%s.tsv' % subset) vectors = VectorSpaceWrapper(frame=frame) total = 0 correct = 0 for sentences, answers in read_cloze(filename): text = ' '.join(sentences) right_answer, wrong_answer = answers probe_vec = vectors.text_to_vector('en', text) right_vec = vectors.text_to_vector('en', right_answer) wrong_vec = vectors.text_to_vector('en', wrong_answer) right_sim = cosine_similarity(probe_vec, right_vec) wrong_sim = cosine_similarity(probe_vec, wrong_vec) if right_sim > wrong_sim: correct += 1 total += 1 # print("%+4.2f %s / %s / %s" % (right_sim - wrong_sim, text, right_answer, wrong_answer)) low, high = proportion_confint(correct, total) return pd.Series([correct / total, low, high], index=['acc', 'low', 'high'])
def test_missing_language(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() # The frame contains no Esperanto, of course, so the out-of-vocabulary # mechanism will fail. We should simply get no results, not crash. similarity = vectors.similar_terms('/c/eo/ekzemplo') eq_(len(similarity), 0)
def test_match_prefix(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() term = '/c/en/figure_skate' expected_prefix_matches = [('/c/en/figure', 0.0033333333333333335), ('/c/en/figure skater', 0.0033333333333333335), ('/c/en/figure skating', 0.0033333333333333335)] prefix_matches = vectors._match_prefix(term=term, prefix_weight=0.01) eq_(expected_prefix_matches, prefix_matches)
def test_cache_with_oov(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() # check the vector of all zeros is returned if the term is not present ok_(not vectors.get_vector('/c/en/test', oov_vector=False).any()) # If include_neighbors=True, the neighbor of 'test' in ConceptNet ('trial') # will be used to approximate its vector ok_(vectors.get_vector('/c/en/test', oov_vector=True).any())
def test_expand_terms(multi_ling_frame): vectors = VectorSpaceWrapper(frame=multi_ling_frame) vectors.load() term = [('/c/en/ski_jumper', 1.0)] expanded_terms = vectors.expand_terms(terms=term, oov_vector=True) expected_expanded_terms = [ ('/c/en/ski_jumper', 0.9900990099009901), ('/c/en/ski_jumping', 0.009900990099009901), ] assert expected_expanded_terms == expanded_terms
def test_match_prefix(simple_frame): vectors = VectorSpaceWrapper(frame=simple_frame) vectors.load() term = '/c/en/figure_skate' expected_prefix_matches = [ ('/c/en/figure', 0.0033333333333333335), ('/c/en/figure skater', 0.0033333333333333335), ('/c/en/figure skating', 0.0033333333333333335), ] prefix_matches = vectors._match_prefix(term=term, prefix_weight=0.01) assert expected_prefix_matches == prefix_matches
def test_match_prefix(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() term = '/c/en/figure_skate' expected_prefix_matches = [ ('/c/en/figure', 0.0033333333333333335), ('/c/en/figure skater', 0.0033333333333333335), ('/c/en/figure skating', 0.0033333333333333335), ] prefix_matches = vectors._match_prefix(term=term, prefix_weight=0.01) eq_(expected_prefix_matches, prefix_matches)
def test_lookup_neighbors(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() term = '/c/pl/skoki_narciarskie' neighbors = vectors._find_neighbors(term=term, limit_per_term=10, weight=1.0) expected_neighbors = [('/c/en/ski_jumping', 0.02), ('http://pl.dbpedia.org/resource/Skoki_narciarskie', 0.01), ('/c/en/ski_jumping', 0.01), ('/c/en/ski_jumping', 0.005)] eq_(expected_neighbors, neighbors)
def test_similar_terms_filter(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() ok_( '/c/pl/kombinacja' in vectors.similar_terms('/c/en/nordic_combined', filter='/c/pl', limit=1).index ) ok_( '/c/en/present' in vectors.similar_terms('/c/en/gift', filter='/c/en/present', limit=1).index )
def test_expand_terms(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() term = [('/c/en/ski_jumper', 1.0)] expanded_terms = vectors.expand_terms(terms=term, limit_per_term=2, oov_vector=True) expected_expanded_terms = [ ('/c/en/ski_jumper', 0.9523809523809523), ('/c/pt/saltadores_de_esqui', 0.019047619047619046), ('/c/pl/skoczek_narciarski', 0.019047619047619046), ('/c/en/ski_jumping', 0.009523809523809523), ] eq_(expected_expanded_terms, expanded_terms)
def test_lookup_neighbors(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() term = '/c/pl/skoki_narciarskie' neighbors = vectors._find_neighbors(term=term, limit_per_term=10, weight=1.0) expected_neighbors = { ('/c/en/ski_jumping', 0.02), ('/c/en/ski_jumping', 0.01), ('http://pl.dbpedia.org/resource/Skoki_narciarskie', 0.01), ('/c/de/skispringen', 0.01), ('/c/en/ski_jumping', 0.005), } eq_(expected_neighbors, set(neighbors))
def test_load(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() ok_(vectors.frame is not None) ok_(vectors.small_frame is not None) ok_(all(label.startswith('/c/en/') for label in vectors.frame.index)) ok_(vectors.frame.index.is_monotonic_increasing) ok_(vectors.small_frame.shape[1] <= 100) ok_(vectors._trie is not None) # test there are no transformations to raw terms other than adding the # English tag ok_('/c/en/figure skater' in vectors.frame.index) # no underscore ok_('/c/en/Island' in vectors.frame.index) # no case folding
def test_load(simple_frame): vectors = VectorSpaceWrapper(frame=simple_frame) vectors.load() assert vectors.frame is not None assert vectors.small_frame is not None assert all(label.startswith('/c/en/') for label in vectors.frame.index) assert vectors.frame.index.is_monotonic_increasing assert vectors.small_frame.shape[1] <= 100 assert vectors._trie is not None # test there are no transformations to raw terms other than adding the # English tag assert '/c/en/figure skater' in vectors.frame.index # no underscore assert '/c/en/Island' in vectors.frame.index # no case folding
def test_expand_terms(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() term = [('/c/en/ski_jumper', 1.0)] expanded_terms = vectors.expand_terms(terms=term, limit_per_term=2, oov_vector=True) # /c/en/bounder and /c/en/skier from neighbor search # /c/en/ski_jumping from prefix match expected_expanded_terms = [('/c/en/ski_jumper', 0.9523809523809523), ('/c/en/bounder', 0.019047619047619046), ('/c/en/skier', 0.019047619047619046), ('/c/en/ski_jumping', 0.009523809523809523)] eq_(expected_expanded_terms, expanded_terms)
def eval_pairwise_analogies(frame, eval_filename, subset='all', weight_direct=0.35, weight_transpose=0.65): total = 0 correct = 0 wrap = VectorSpaceWrapper(frame=frame) for idx, (prompt, choices, answer) in enumerate(read_turney_analogies(eval_filename)): # Enable an artificial training/test split if subset == 'all' or (subset == 'dev') == (idx % 2 == 0): a1, b1 = prompt choice_values = [] for choice in choices: a2, b2 = choice choice_values.append( pairwise_analogy_func(wrap, a1, b1, a2, b2, weight_direct, weight_transpose)) our_answer = np.argmax(choice_values) if our_answer == answer: correct += 1 total += 1 low, high = proportion_confint(correct, total) return pd.Series([correct / total, low, high], index=['acc', 'low', 'high'])
class RelatednessClassifier(AttributeClassifier): def __init__(self): self.wrap = VectorSpaceWrapper( get_external_data_filename('numberbatch-20180108-biased.h5'), use_db=False) self.cache = {} def get_vector(self, uri): if uri in self.cache: return self.cache[uri] else: vec = normalize_vec(self.wrap.get_vector(uri)) self.cache[uri] = vec return vec def get_similarity(self, uri1, uri2): return self.get_vector(uri1).dot(self.get_vector(uri2)) def direct_relatedness(self, example): match1 = max(0, self.get_similarity(example.node1(), example.att_node()))**0.5 match2 = max(0, self.get_similarity(example.node2(), example.att_node()))**0.5 return match1 - match2 def classify(self, examples, mode): return np.array( [self.direct_relatedness(example) > .1 for example in examples])
def evaluate(frame, subset='dev'): """ Evaluate a DataFrame containing term vectors on its ability to predict term relatedness, according to MEN-3000, RW, MTurk-771, and WordSim-353. Use a VectorSpaceWrapper to fill missing vocabulary from ConceptNet. Return a Series containing these labeled results. """ if subset == 'all': men_subset = 'test' else: men_subset = subset vectors = VectorSpaceWrapper(frame=frame) men_score = spearman_evaluate(vectors, read_men3000(men_subset)) rw_score = spearman_evaluate(vectors, read_rw(subset)) mturk_score = spearman_evaluate(vectors, read_mturk()) ws_score = spearman_evaluate(vectors, read_ws353()) ws_es_score = spearman_evaluate(vectors, read_ws353_multilingual('es'), language='es') ws_ro_score = spearman_evaluate(vectors, read_ws353_multilingual('ro'), language='ro') results = empty_comparison_table() results.loc['men3000'] = men_score results.loc['rw'] = rw_score results.loc['mturk'] = mturk_score results.loc['ws353'] = ws_score results.loc['ws353-es'] = ws_es_score results.loc['ws353-ro'] = ws_ro_score return results
def evaluate(frame, subset='dev', semeval_scope='global'): """ Evaluate a DataFrame containing term vectors on its ability to predict term relatedness, according to MEN-3000, RW, MTurk-771, WordSim-353, and Semeval2017-Task2. Use a VectorSpaceWrapper to fill missing vocabulary from ConceptNet. Return a Series containing these labeled results. """ if subset == 'all': men_subset = 'test' else: men_subset = subset vectors = VectorSpaceWrapper(frame=frame) men_score = measure_correlation(spearmanr, vectors, read_men3000(men_subset)) rw_score = measure_correlation(spearmanr, vectors, read_rw(subset)) mturk_score = measure_correlation(spearmanr, vectors, read_mturk()) simlex_score = measure_correlation(spearmanr, vectors, read_simlex()) gur350_score = measure_correlation(spearmanr, vectors, read_gurevych('350')) zg222_score = measure_correlation(spearmanr, vectors, read_gurevych('222')) ws_score = measure_correlation(spearmanr, vectors, read_ws353()) ws_es_score = measure_correlation(spearmanr, vectors, read_ws353_multilingual('es')) ws_ro_score = measure_correlation(spearmanr, vectors, read_ws353_multilingual('ro')) pku500_score = measure_correlation(spearmanr, vectors, read_pku500()) jsim_score = measure_correlation(spearmanr, vectors, read_jsim()) results = empty_comparison_table() results.loc['men3000'] = men_score results.loc['rw'] = rw_score results.loc['mturk'] = mturk_score results.loc['simlex'] = simlex_score results.loc['gur350-de'] = gur350_score results.loc['zg222-de'] = zg222_score results.loc['ws353'] = ws_score results.loc['ws353-es'] = ws_es_score results.loc['ws353-ro'] = ws_ro_score results.loc['pku500-zh'] = pku500_score results.loc['jsim-ja'] = jsim_score if semeval_scope == 'global': results.loc['semeval17-2a'] = evaluate_semeval_monolingual_global(vectors) results.loc['semeval17-2b'] = evaluate_semeval_crosslingual_global(vectors) else: languages = ['en', 'de', 'es', 'it', 'fa'] for lang in languages: results.loc['semeval-2a-{}'.format(lang)] = evaluate_semeval_monolingual( vectors, lang ) for lang1, lang2 in combinations(languages, 2): results.loc[ 'semeval-2b-{}-{}'.format(lang1, lang2) ] = evaluate_semeval_crosslingual(vectors, lang1, lang2) return results
def test_vector_space_wrapper_filter(): wrap = VectorSpaceWrapper(frame=TEST_FRAME) wrap.load() ok_('/c/pl/kombinacja' in wrap.similar_terms('/c/en/nordic_combined', filter='/c/pl', limit=1).index) ok_('/c/en/present' in wrap.similar_terms('/c/en/gift', filter='/c/en/present', limit=1).index)
def test_similar_terms_filter(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() ok_('/c/pl/kombinacja' in vectors.similar_terms( '/c/en/nordic_combined', filter='/c/pl', limit=1).index) ok_('/c/en/present' in vectors.similar_terms( '/c/en/gift', filter='/c/en/present', limit=1).index)
def test_similar_terms(simple_frame): """ Check if VectorSpaceWrapper's index is sorted and its elements are concepts. """ vectors = VectorSpaceWrapper(frame=simple_frame) vectors.load() assert ('/c/en/figure skating' in vectors.similar_terms('/c/en/figure skating', limit=3).index) assert ('/c/en/figure skater' in vectors.similar_terms('/c/en/figure skating', limit=3).index) assert ('/c/en/figure' in vectors.similar_terms('/c/en/figure skating', limit=3).index)
def test_similar_terms(): """ Check if VectorSpaceWrapper's index is sorted and its elements are concepts. """ vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() ok_('/c/en/figure skating' in vectors.similar_terms('/c/en/figure skating', limit=3).index) ok_('/c/en/figure skater' in vectors.similar_terms('/c/en/figure skating', limit=3).index) ok_('/c/en/figure' in vectors.similar_terms('/c/en/figure skating', limit=3).index)
def test_vector_space_wrapper(): """ Check if VectorSpaceWrapper's index is sorted and its elements are concepts. """ wrap = VectorSpaceWrapper(frame=TEST_FRAME) wrap.load() ok_(all(is_term(label) for label in wrap.frame.index)) ok_(wrap.frame.index.is_monotonic_increasing) # test there are no transformations to raw terms other than adding the english tag ok_('/c/en/figure skater' in wrap.frame.index) # no underscore ok_('/c/en/Island' in wrap.frame.index) # no case folding # test index_prefix_range ok_(wrap.index_prefix_range('/c/en/figure') == (3, 6)) ok_(wrap.index_prefix_range('/c/en/skating') == (0, 0)) # test_similar_terms ok_('/c/en/figure skating' in wrap.similar_terms('/c/en/figure skating', limit=3).index) ok_('/c/en/figure skater' in wrap.similar_terms('/c/en/figure skating', limit=3).index) ok_('/c/en/figure' in wrap.similar_terms('/c/en/figure skating', limit=3).index)
def measure_bias(frame): """ Return a DataFrame that measures biases in a semantic space, on four data sets: - Gender - Fine-grained ethnicity - Coarse-grained ethnicity - Religious beliefs """ vsw = VectorSpaceWrapper(frame=frame) vsw.load() gender_binary_axis = normalize_vec( get_category_axis(frame, FEMALE_WORDS) - get_category_axis(frame, MALE_WORDS)) gender_bias_numbers = [] for female_biased_word, male_biased_word in GENDER_BIAS_PAIRS: female_biased_uri = standardized_uri('en', female_biased_word) male_biased_uri = standardized_uri('en', male_biased_word) diff = normalize_vec( vsw.get_vector(female_biased_uri) - vsw.get_vector(male_biased_uri)).dot(gender_binary_axis) gender_bias_numbers.append(diff) mean = np.mean(gender_bias_numbers) sem = scipy.stats.sem(gender_bias_numbers) gender_bias = pd.Series([mean, mean - sem * 2, mean + sem * 2], index=['bias', 'low', 'high']) stereotype_vecs_1 = get_vocabulary_vectors(frame, PEOPLE_BY_ETHNICITY) stereotype_vecs_2 = get_vocabulary_vectors(frame, ETHNIC_STEREOTYPE_TERMS) fine_ethnic_bias = correlation_bias(stereotype_vecs_1, stereotype_vecs_2) stereotype_vecs_1 = get_vocabulary_vectors(frame, COARSE_ETHNICITY_TERMS) stereotype_vecs_2 = get_vocabulary_vectors(frame, ETHNIC_STEREOTYPE_TERMS) coarse_ethnic_bias = correlation_bias(stereotype_vecs_1, stereotype_vecs_2) stereotype_vecs_1 = pd.DataFrame( np.vstack( [get_category_axis(frame, names) for names in ETHNIC_NAME_SETS])) stereotype_vecs_2 = get_vocabulary_vectors(frame, ETHNIC_STEREOTYPE_TERMS) name_ethnic_bias = correlation_bias(stereotype_vecs_1, stereotype_vecs_2) stereotype_vecs_1 = get_vocabulary_vectors(frame, PEOPLE_BY_BELIEF) stereotype_vecs_2 = get_vocabulary_vectors(frame, BELIEF_STEREOTYPE_TERMS) belief_bias = correlation_bias(stereotype_vecs_1, stereotype_vecs_2) return pd.DataFrame({ 'gender': gender_bias, 'ethnicity-fine': fine_ethnic_bias, 'ethnicity-coarse': coarse_ethnic_bias, 'ethnicity-names': name_ethnic_bias, 'beliefs': belief_bias }).T
def _load_vectors(): frame = load_hdf(resource_filename('codenames', 'data/mini.h5')) selections = [ label for label in frame.index if label.startswith('/c/en/') and '_' not in label and '#' not in label and wordfreq.zipf_frequency(label[6:], 'en') > 3.0 ] # Make sure all the words in Codenames are represented wordlist = [ standardized_uri('en', line.strip()) for line in open( resource_filename('codenames', 'data/codenames-words.txt')) ] additions = [word for word in wordlist if word not in selections] selections += additions frame = l2_normalize_rows(frame.loc[selections].astype('f')) return VectorSpaceWrapper(frame=frame)
def test_similar_terms_filter(multi_ling_frame): vectors = VectorSpaceWrapper(frame=multi_ling_frame) vectors.load() assert ('/c/pl/kombinacja' in vectors.similar_terms('/c/en/nordic_combined', filter='/c/pl', limit=1).index) assert ('/c/en/present' in vectors.similar_terms('/c/en/gift', filter='/c/en/present', limit=1).index)
def test_similar_terms(): """ Check if VectorSpaceWrapper's index is sorted and its elements are concepts. """ vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() ok_( '/c/en/figure skating' in vectors.similar_terms('/c/en/figure skating', limit=3).index ) ok_( '/c/en/figure skater' in vectors.similar_terms('/c/en/figure skating', limit=3).index ) ok_('/c/en/figure' in vectors.similar_terms('/c/en/figure skating', limit=3).index)
from conceptnet5.vectors.query import VectorSpaceWrapper from conceptnet5.query import field_match, VALID_KEYS from conceptnet5.db.query import MAX_GROUP_SIZE from conceptnet5.nodes import standardized_concept_uri from conceptnet5.nodes import ld_node import itertools VECTORS = VectorSpaceWrapper() FINDER = VECTORS.finder VALID_KEYS = VALID_KEYS # re-export CONTEXT = [ "http://api.conceptnet.io/ld/conceptnet5.5/context.ld.json", "http://api.conceptnet.io/ld/conceptnet5.5/pagination.ld.json" ] def success(response): response['@context'] = CONTEXT return response def error(response, status, details): response['@context'] = CONTEXT response['error'] = {'status': status, 'details': details} return response def make_query_url(url, items): str_items = ['{}={}'.format(*item) for item in items] if not str_items: return url
def evaluate(frame, analogy_filename, subset='test', tune_analogies=True, scope='global', google_vocab_size=200000): """ Run SAT and Semeval12-2 evaluations. Required parameters: frame a DataFrame containing term vectors analogy_filename the filename of Turney's SAT evaluation data Optional parameters: subset (string, default 'test') a subset of a data to evaluate on, either 'test' or 'dev' tune_analogies (boolean, default True) tune the weights in eval_pairwise_analogies() semeval_scope (string, default 'global') 'global' to get the average of the results across all subclasses of semeval12-2, or another string to get the results broken down by a subclass (1a, 1b, etc.) """ vectors = VectorSpaceWrapper(frame=frame) results = empty_comparison_table() if tune_analogies: sat_weights = optimize_weights(eval_pairwise_analogies, vectors, analogy_filename) semeval_weights = optimize_weights(eval_semeval2012_global, vectors) else: sat_weights = (0.35, 0.65) semeval_weights = (0.3, 0.35) sat_results = eval_pairwise_analogies(vectors, analogy_filename, sat_weights[0], sat_weights[1], subset) results.loc['sat-analogies'] = sat_results for gsubset in ['semantic', 'syntactic']: google_results = eval_google_analogies(vectors, subset=gsubset, vocab_size=google_vocab_size) results.loc['google-%s' % gsubset] = google_results # There's no meaningful "all" subset for semeval12, because the dev and # test data are stored entirely separately. Just use "test". if subset == 'dev': semeval12_subset = 'dev' else: semeval12_subset = 'test' if scope == 'global': maxdiff_score, spearman_score = eval_semeval2012_global( vectors, semeval_weights[0], semeval_weights[1], semeval12_subset) results.loc['semeval12-spearman'] = spearman_score results.loc['semeval12-maxdiff'] = maxdiff_score else: for subclass in product(range(1, 11), 'a b c d e f g h i j'): subclass = ''.join([str(element) for element in subclass]) try: maxdiff_score, spearman_score = eval_semeval2012_analogies( vectors, semeval_weights[0], semeval_weights[1], semeval12_subset, subclass) results.loc['semeval12-{}-spearman'.format( subclass)] = spearman_score results.loc['semeval12-{}-maxdiff'.format( subclass)] = maxdiff_score except FileNotFoundError: continue bats_results = [] for category in product('DEIL', range(1, 11)): category = ''.join([str(element) for element in category]) quads = read_bats(category) category_results = eval_open_vocab_analogies(vectors, quads) bats_results.append((category, category_results)) if scope == 'global': average_scores = [] for interval in ['acc', 'low', 'high']: average_scores.append( np.mean([result[interval] for name, result in bats_results])) results.loc['bats'] = pd.Series(average_scores, index=['acc', 'low', 'high']) else: for name, result in bats_results: results.loc['bats-{}'.format(''.join(name))] = result return results
def test_index_prefix_range(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) vectors.load() eq_(vectors._index_prefix_range('/c/en/figure'), (3, 6)) eq_(vectors._index_prefix_range('/c/en/skating'), (0, 0))
def test_englishify(): vectors = VectorSpaceWrapper(frame=TEST_FRAME) eq_(vectors._englishify('/c/sv/harry_potter'), '/c/en/harry_potter')
def test_cache_with_oov(multi_ling_frame): vectors = VectorSpaceWrapper(frame=multi_ling_frame) vectors.load() # check the vector of all zeros is returned if the term is not present assert not vectors.get_vector('/c/en/test', oov_vector=False).any()