def test_basic_corpus_nd(specified_test_corpus): calls = [({ 'query': specified_test_corpus.find('mata'), 'max_distance': 1 }, 1.0), ({ 'query': specified_test_corpus.find('nata'), 'max_distance': 2 }, 3.0), ({ 'query': specified_test_corpus.find('mata'), 'algorithm': 'phono_edit_distance', 'max_distance': 3 }, 1.0)] with CanonicalVariantContext(specified_test_corpus, 'transcription', 'type') as c: for kwargs, v in calls: result = neighborhood_density(c, **kwargs) assert (abs(result[0] - v) < 0.0001) with CanonicalVariantContext(specified_test_corpus, 'spelling', 'type') as c: result = neighborhood_density( c, **{ 'query': specified_test_corpus.find('mata'), 'max_distance': 1 }) assert (abs(result[0] - 1.0) < 0.0001)
def test_basic_phonoprob(unspecified_test_corpus): expected = { 'atema': 0.1011173499, 'enuta': 0.1011173499, 'ta': 0.4353651056, 'mata': 0.1881114313, 'nata': 0.1782478499 } with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: for k, v in expected.items(): res = phonotactic_probability_vitevitch( c, unspecified_test_corpus.find(k), 'unigram') assert (abs(v - res) < 0.0001) expected = { 'atema': 0.0798992942, 'enuta': 0.0798992942, 'ta': 0.1507780332, 'mata': 0.0626335622, 'nata': 0.0494821204 } with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: for k, v in expected.items(): res = phonotactic_probability_vitevitch( c, unspecified_test_corpus.find(k), 'bigram') assert (abs(v - res) < 0.0001)
def __init__(self, corpus, parent=None): QWidget.__init__(self, parent) self.corpus = corpus self.type_context = CanonicalVariantContext(self.corpus, 'transcription', 'type') self.token_context = CanonicalVariantContext(self.corpus, 'transcription', 'token') layout = QHBoxLayout() layout.setAlignment(Qt.AlignTop) self.segments = InventoryBox('Segments', self.corpus.inventory) self.segments.setExclusive(True) for b in self.segments.btnGroup.buttons(): b.clicked.connect(self.summarizeSegment) layout.addWidget(self.segments) self.detailFrame = QFrame() layout.addWidget(self.detailFrame) self.setLayout(layout)
def test_relative_minpair(unspecified_test_corpus): calls = [ ({ 'segment': 's', 'relative_count': True }, 0.013888), ({ 'segment': 's', 'relative_count': False }, 0.11111), ({ 'segment': 'n', 'relative_count': True }, 0.0123457), ({ 'segment': 'n', 'relative_count': False }, 0.11111), ({ 'segment': 'o', 'relative_count': True }, 0), ({ 'segment': 'o', 'relative_count': False }, 0), ] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: for kwargs, v in calls: assert (abs(relative_minpair_fl(c, **kwargs) - v) < 0.0001) calls = [({ 'segment': 's', 'relative_count': True }, 0.01587), ({ 'segment': 's', 'relative_count': False }, 0.11111), ({ 'segment': 'n', 'relative_count': True }, 0), ({ 'segment': 'n', 'relative_count': False }, 0), ({ 'segment': 'o', 'relative_count': True }, 0), ({ 'segment': 'o', 'relative_count': False }, 0)] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type', frequency_threshold=3) as c: for kwargs, v in calls: assert (abs(relative_minpair_fl(c, **kwargs) - v) < 0.0001)
def test_minpair(unspecified_test_corpus): calls = [({'segment_pairs':[('s','ʃ')], 'relative_count':True},0.125), ({'segment_pairs':[('s','ʃ')], 'relative_count':False},1), ({'segment_pairs':[('m','n')], 'relative_count':True},0.11111), ({'segment_pairs':[('m','n')], 'relative_count':False},1), ({'segment_pairs':[('e','o')], 'relative_count':True},0), ({'segment_pairs':[('e','o')], 'relative_count':False},0), ({'segment_pairs':[('s','ʃ'), ('m','n'), ('e','o')], 'relative_count':True},0.14286), ({'segment_pairs':[('s','ʃ'), ('m','n'), ('e','o')], 'relative_count':False},2),] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: for kwargs, v in calls: print(kwargs) assert(abs(minpair_fl(c, **kwargs)[0]-v) < 0.0001) calls = [({'segment_pairs':[('s','ʃ')], 'relative_count':True},0.14286), ({'segment_pairs':[('s','ʃ')], 'relative_count':False},1), ({'segment_pairs':[('m','n')], 'relative_count':True},0), ({'segment_pairs':[('m','n')], 'relative_count':False},0), ({'segment_pairs':[('e','o')], 'relative_count':True},0), ({'segment_pairs':[('e','o')], 'relative_count':False},0), ({'segment_pairs':[('s','ʃ'), ('m','n'), ('e','o')], 'relative_count':True},0.09091), ({'segment_pairs':[('s','ʃ'), ('m','n'), ('e','o')], 'relative_count':False},1)] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type', frequency_threshold = 3) as c: for kwargs, v in calls: print(kwargs) assert(abs(minpair_fl(c, **kwargs)[0]-v) < 0.0001)
def test_freqalt(specified_test_corpus): with CanonicalVariantContext(specified_test_corpus, 'transcription', 'type') as c: result = calc_freq_of_alt(c,'s','ʃ','khorsi', min_rel = -15, phono_align=True) assert(result==(8,3,0.375)) result = calc_freq_of_alt(c,'s','ʃ','khorsi', min_rel = -6, phono_align=True) assert(result==(8,0,0)) result = calc_freq_of_alt(c,'s','ʃ','khorsi', min_rel = -6, phono_align=False) assert(result==(8,2,0.25)) result = calc_freq_of_alt(c,'s','ʃ','khorsi', min_rel = -15, phono_align=False) assert(result==(8,7,0.875)) result = calc_freq_of_alt(c,'s','ʃ','edit_distance', max_rel = 2, phono_align=True) assert(result==(8,2,0.25)) result = calc_freq_of_alt(c,'s','ʃ','edit_distance', max_rel = 2, phono_align=False) assert(result==(8,2,0.25)) result = calc_freq_of_alt(c,'s','ʃ','phono_edit_distance', max_rel = 6, phono_align=True) assert(result==(8,2,0.25)) result = calc_freq_of_alt(c,'s','ʃ','phono_edit_distance', max_rel = 6, phono_align=False) assert(result==(8,2,0.25)) with CanonicalVariantContext(specified_test_corpus, 'transcription', 'token') as c: result = calc_freq_of_alt(c,'s','ʃ','khorsi', min_rel = -15, phono_align=True) assert(result==(8,3,0.375)) result = calc_freq_of_alt(c,'s','ʃ','khorsi', min_rel = -6, phono_align=True) assert(result==(8,2,0.25)) result = calc_freq_of_alt(c,'s','ʃ','khorsi', min_rel = -15, phono_align=False) assert(result==(8,7,0.875)) result = calc_freq_of_alt(c,'s','ʃ','khorsi', min_rel = -6, phono_align=False) assert(result==(8,3,0.375)) result = calc_freq_of_alt(c,'s','ʃ','edit_distance', max_rel = 4, phono_align=True) assert(result==(8,3,0.375)) result = calc_freq_of_alt(c,'s','ʃ','edit_distance', max_rel = 4, phono_align=False) assert(result==(8,6,0.75)) result = calc_freq_of_alt(c,'s','ʃ','phono_edit_distance', max_rel = 20, phono_align=True) assert(result==(8,3,0.375)) result = calc_freq_of_alt(c,'s','ʃ','phono_edit_distance', max_rel = 20, phono_align=False) assert(result==(8,6,0.75))
def test_prod_type(specified_test_corpus): seg1 = 's' seg2 = 'ʃ' expected = { "-voc": 0.0, "+voc,+high": 0.863120568566631, "+voc,-high": 0.9852281360342515, "#": 0.0 } env_list = [] expected_envs = {} for k, v in expected.items(): if k != '#': segs = specified_test_corpus.features_to_segments(k) else: segs = k env = EnvironmentFilter(['s', 'ʃ'], None, [segs]) env_list.append(env) expected_envs[env] = v expected_envs["AVG"] = 0.9241743523004413 type_or_token = 'type' tier = 'transcription' with CanonicalVariantContext(specified_test_corpus, tier, type_or_token) as c: result = calc_prod(c, env_list, all_info=False) for k, v in result.items(): assert (expected_envs[k] - v < 0.001)
def test_mass_fl(unspecified_test_corpus): #This needs to be updated so that it deterministically passes calls = [({'algorithm':'minpair', 'relative_count':True}, ([(('s', 'ʃ'), 0.125), (('m', 'n'), 0.1111111111111111), (('i', 't'), 0.0), (('t', 'u'), 0.0), (('m', 't'), 0.0), (('i', 'u'), 0.0), (('e', 'o'), 0.0), (('n', 'o'), 0.0), (('i', 'ʃ'), 0.0), (('u', 'ɑ'), 0.0), (('m', 'ʃ'), 0.0), (('m', 'ɑ'), 0.0), (('t', 'ʃ'), 0.0), (('e', 'n'), 0.0), (('o', 't'), 0.0), (('e', 'ɑ'), 0.0), (('n', 'u'), 0.0), (('n', 't'), 0.0), (('o', 'ʃ'), 0.0), (('e', 'u'), 0.0), (('s', 't'), 0.0), (('ɑ', 'ʃ'), 0.0), (('n', 's'), 0.0), (('e', 's'), 0.0), (('i', 's'), 0.0), (('m', 'u'), 0.0), (('e', 'i'), 0.0), (('i', 'n'), 0.0), (('i', 'o'), 0.0), (('i', 'm'), 0.0), (('n', 'ɑ'), 0.0), (('t', 'ɑ'), 0.0), (('s', 'ɑ'), 0.0), (('s', 'u'), 0.0), (('i', 'ɑ'), 0.0), (('o', 's'), 0.0), (('e', 'ʃ'), 0.0), (('u', 'ʃ'), 0.0), (('m', 'o'), 0.0), (('e', 'm'), 0.0), (('o', 'u'), 0.0), (('n', 'ʃ'), 0.0), (('e', 't'), 0.0), (('o', 'ɑ'), 0.0), (('m', 's'), 0.0)])), ({'algorithm':'minpair', 'relative_count':False}, ([(('s', 'ʃ'), 1.0), (('m', 'n'), 1.0), (('i', 't'), 0.0), (('t', 'u'), 0.0), (('m', 't'), 0.0), (('i', 'u'), 0.0), (('e', 'o'), 0.0), (('n', 'o'), 0.0), (('i', 'ʃ'), 0.0), (('u', 'ɑ'), 0.0), (('m', 'ʃ'), 0.0), (('m', 'ɑ'), 0.0), (('t', 'ʃ'), 0.0), (('e', 'n'), 0.0), (('o', 't'), 0.0), (('e', 'ɑ'), 0.0), (('n', 'u'), 0.0), (('n', 't'), 0.0), (('o', 'ʃ'), 0.0), (('e', 'u'), 0.0), (('s', 't'), 0.0), (('ɑ', 'ʃ'), 0.0), (('n', 's'), 0.0), (('e', 's'), 0.0), (('i', 's'), 0.0), (('m', 'u'), 0.0), (('e', 'i'), 0.0), (('i', 'n'), 0.0), (('i', 'o'), 0.0), (('i', 'm'), 0.0), (('n', 'ɑ'), 0.0), (('t', 'ɑ'), 0.0), (('s', 'ɑ'), 0.0), (('s', 'u'), 0.0), (('i', 'ɑ'), 0.0), (('o', 's'), 0.0), (('e', 'ʃ'), 0.0), (('u', 'ʃ'), 0.0), (('m', 'o'), 0.0), (('e', 'm'), 0.0), (('o', 'u'), 0.0), (('n', 'ʃ'), 0.0), (('e', 't'), 0.0), (('o', 'ɑ'), 0.0), (('m', 's'), 0.0)]))] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: for kwargs, v in calls: for result,prediction in zip(all_pairwise_fls(c, **kwargs), v): assert(abs(result[1]-prediction[1]) < 0.0001)
def test_identical(specified_test_corpus): #Test 1, things that are identical with CanonicalVariantContext(specified_test_corpus, 'transcription', 'type') as c: seg1_entropy, seg2_entropy, distance, ur, is_spurious = KL( c, 's', 's', 'b') assert (distance == 0.0) assert (seg1_entropy == seg2_entropy)
def test_deltah(unspecified_test_corpus): type_calls = [({'segment_pairs':[('s','ʃ')]},0.02547695), ({'segment_pairs':[('m','n')]},0.02547695), ({'segment_pairs':[('e','o')]},0), ({'segment_pairs':[('s','ʃ'), ('m','n'), ('e','o')]},0.05284),] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: for kwargs, v in type_calls: assert(abs(deltah_fl(c, **kwargs)-v) < 0.0001) type_calls = [({'segment_pairs':[('s','ʃ')], 'prevent_normalization':True},0.09953567), ({'segment_pairs':[('m','n')], 'prevent_normalization':True},0.09953567), ({'segment_pairs':[('e','o')], 'prevent_normalization':True},0), ({'segment_pairs':[('s','ʃ'), ('m','n'), ('e','o')], 'prevent_normalization':True},0.206450877),] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: for kwargs, v in type_calls: assert(abs(deltah_fl(c, **kwargs)-v) < 0.0001) type_calls = [({'segment_pairs':[('s','ʃ')]},0.035015954), ({'segment_pairs':[('m','n')]},0), ({'segment_pairs':[('e','o')]},0), ({'segment_pairs':[('s','ʃ'), ('m','n'), ('e','o')]},0.035015954)] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type', frequency_threshold = 3) as c: for kwargs, v in type_calls: assert(abs(deltah_fl(c, **kwargs)-v) < 0.0001) token_calls = [({'segment_pairs':[('s','ʃ')]},0.08305), ({'segment_pairs':[('m','n')]},0.002314), ({'segment_pairs':[('e','o')]},0.0), ({'segment_pairs':[('s','ʃ'), ('m','n'), ('e','o')]},0.0853641),] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: for kwargs, v in token_calls: assert(abs(deltah_fl(c, **kwargs)-v) < 0.0001)
def test_basic_corpus_mutation_minpairs(specified_test_corpus): calls = [({'query':Word(**{'transcription': ['s', 'ɑ', 't', 'ɑ']}), },2)] with CanonicalVariantContext(specified_test_corpus, 'transcription', 'type') as c: for kwargs,v in calls: result = find_mutation_minpairs(c, **kwargs) assert(result[0] == v) assert(sorted(result[1]) == sorted(['n.ɑ.t.ɑ', 'm.ɑ.t.ɑ']))
def test_allophones(specified_test_corpus): #Test 2, things are supposed to be allophones with CanonicalVariantContext(specified_test_corpus, 'transcription', 'type') as c: seg1_entropy, seg2_entropy, distance, ur, is_spurious = KL( c, 's', 'ʃ', 'b') assert (abs(distance - 0.15113518339295337) < 0.001) assert (abs(seg1_entropy - 0.035000140096702444) < 0.001) assert (abs(seg2_entropy - 0.06074393445793598) < 0.001)
def test_pseudo_allophones(specified_test_corpus): #Test 3, things that are allophones by coincidence with CanonicalVariantContext(specified_test_corpus, 'transcription', 'type') as c: seg1_entropy, seg2_entropy, distance, ur, is_spurious = KL( c, 's', 'ɑ', 'b') assert (abs(distance - 0.23231302100802534) < 0.001) assert (abs(seg1_entropy - 0.03500014009670246) < 0.001) assert (abs(seg2_entropy - 0.07314589775440267) < 0.001)
def test_default(specified_test_corpus): #Test 4, things that have no assumed relationship with CanonicalVariantContext(specified_test_corpus, 'transcription', 'type') as c: seg1_entropy, seg2_entropy, distance, ur, is_spurious = KL( c, 's', 'm', 'b') assert (abs(distance - 0.14186314747884132) < 0.001) assert (abs(seg1_entropy - 0.035000140096702444) < 0.001) assert (abs(seg2_entropy - 0.06074393445793598) < 0.001)
def test_spelling(unspecified_test_corpus): expected = [(unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('atema'),0), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('enuta'),4), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('mashomisi'),7), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('mata'),3), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('nata'),3), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('sasi'),5), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('shashi'),6), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('shisata'),6), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('shushoma'),6), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('ta'),3), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('tatomi'),3), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('tishenishu'),9), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('toni'),4), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('tusa'),3), (unspecified_test_corpus.find('atema'),unspecified_test_corpus.find('ʃi'),5)] expected.sort(key=lambda t:t[1]) with CanonicalVariantContext(unspecified_test_corpus, 'spelling', 'type') as c: calced = string_similarity(c,unspecified_test_corpus.find('atema'),'edit_distance') calced.sort(key=lambda t:t[1]) for i, v in enumerate(expected): assert(calced[i] == v) expected = [(unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('atema'),5), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('enuta'),5), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('mashomisi'),6), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('mata'),3), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('nata'),3), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('sasi'),0), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('shashi'),2), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('shisata'),5), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('shushoma'),6), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('ta'),3), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('tatomi'),4), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('tishenishu'),8), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('toni'),3), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('tusa'),3), (unspecified_test_corpus.find('sasi'),unspecified_test_corpus.find('ʃi'),3)] expected.sort(key=lambda t:t[1]) with CanonicalVariantContext(unspecified_test_corpus, 'spelling', 'type') as c: calced = string_similarity(c,unspecified_test_corpus.find('sasi'),'edit_distance') calced.sort(key=lambda t:t[1]) for i, v in enumerate(expected): assert(calced[i] == v)
def test_relative_deltah(unspecified_test_corpus): type_calls = [({'segment':'s'},0.00283), ({'segment':'n'},0.00283), ({'segment':'o'},0),] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: for kwargs, v in type_calls: assert(abs(relative_deltah_fl(c, **kwargs)[0]-v) < 0.0001) type_calls = [({'segment':'s'}, 0.00389), ({'segment':'n'},0), ({'segment':'o'},0),] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type', frequency_threshold = 3) as c: for kwargs, v in type_calls: assert(abs(relative_deltah_fl(c, **kwargs)[0]-v) < 0.0001) token_calls = [({'segment':'s'},0.009227777), ({'segment':'n'},0.0002571111), ({'segment':'o'},0),] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: for kwargs, v in token_calls: assert(abs(relative_deltah_fl(c, **kwargs)[0]-v) < 0.0001)
def test_freq_base_transcription_type(unspecified_test_corpus): expected = { '#': 30, 'ɑ': 16, 'e': 3, 'i': 10, 'm': 6, 'n': 4, 'o': 4, 's': 5, 'ʃ': 9, 't': 11, 'u': 4, 'total': 102 } with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: freq_base = c.get_frequency_base() assert (freq_base == expected)
def test_freq_base_transcription_token(unspecified_test_corpus): expected = { '#': 1158, 'ɑ': 466, 'e': 118, 'i': 429, 'm': 156, 'n': 142, 'o': 171, 's': 318, 'ʃ': 540, 't': 271, 'u': 265, 'total': 4034 } with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: freq_base = c.get_frequency_base() assert (freq_base == expected)
def test_freq_base_spelling_type(unspecified_test_corpus): expected = { '#': 30, 'a': 16, 'e': 3, 'h': 8, 'i': 10, 'm': 6, 'n': 4, 'o': 4, 's': 13, 'ʃ': 1, 't': 11, 'u': 4, 'total': 110 } with CanonicalVariantContext(unspecified_test_corpus, 'spelling', 'type') as c: freq_base = c.get_frequency_base() assert (freq_base == expected)
def test_pointwise_mi(unspecified_test_corpus): with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: calls = [({ 'corpus_context': c, 'query': ('e', 'm') }, 2.7319821866519507), ({ 'corpus_context': c, 'query': ('t', 'n'), 'in_word': True }, 0.5849625007211564), ({ 'corpus_context': c, 'query': ('e', 'm'), 'halve_edges': True }, 2.7319821866519507)] for c, v in calls: result = pointwise_mi(**c) assert (abs(result - v) < 0.0001)
def test_freq_base_spelling_token(unspecified_test_corpus): expected = { '#': 1158, 'a': 466, 'e': 118, 'h': 538, 'i': 429, 'm': 156, 'n': 142, 'o': 171, 's': 856, 'ʃ': 2, 't': 271, 'u': 265, 'total': 4572 } with CanonicalVariantContext(unspecified_test_corpus, 'spelling', 'token') as c: freq_base = c.get_frequency_base() assert (freq_base == expected)
def test_relative_deltah(unspecified_test_corpus): type_calls = [ ({ 'segment': 's' }, 0.014814), ({ 'segment': 'n' }, 0.014814), ({ 'segment': 'o' }, 0), ] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: for kwargs, v in type_calls: assert (abs(relative_deltah_fl(c, **kwargs) - v) < 0.0001) type_calls = [ ({ 'segment': 's' }, 0.0185185), ({ 'segment': 'n' }, 0), ({ 'segment': 'o' }, 0), ] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type', frequency_threshold=3) as c: for kwargs, v in type_calls: assert (abs(relative_deltah_fl(c, **kwargs) - v) < 0.0001) token_calls = [ ({ 'segment': 's' }, 0.0275487), ({ 'segment': 'n' }, 0.000767), ({ 'segment': 'o' }, 0), ] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: for kwargs, v in token_calls: assert (abs(relative_deltah_fl(c, **kwargs) - v) < 0.0001) token_calls = [({ 'segment': 's' }, 0.027837), ({ 'segment': 'n' }, 0), ({ 'segment': 'o' }, 0)] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token', frequency_threshold=3) as c: for kwargs, v in token_calls: assert (abs(relative_deltah_fl(c, **kwargs) - v) < 0.0001)
def test_error(specified_test_corpus): #Test 5, things not in the corpus with pytest.raises(ValueError): with CanonicalVariantContext(specified_test_corpus, 'transcription', 'type') as c: KL(c, 's', '!', '')
def test_mass_relate_spelling_type(unspecified_test_corpus): expected = [ (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('atema'), 11.0766887), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('enuta'), -14.09489383), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('mashomisi'), -18.35890071), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('mata'), -6.270847817), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('nata'), -8.494720336), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('sasi'), -13.57140897), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shashi'), -18.17657916), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shisata'), -13.51516925), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shushoma'), -16.90806783), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('ta'), -8.717863887), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tatomi'), -13.53912249), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tishenishu'), -28.78151269), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('toni'), -15.17933206), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tusa'), -13.53067344), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('ʃi'), -17.53815687), ] expected.sort(key=lambda t: t[2]) expected.reverse() with CanonicalVariantContext(unspecified_test_corpus, 'spelling', 'type') as c: calced = string_similarity(c, unspecified_test_corpus.find('atema'), 'khorsi') for i, v in enumerate(expected): assert (abs(calced[i][2] - v[2]) < 0.0001) expected = [ (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('atema'), -13.57140897), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('enuta'), -15.36316844), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('mashomisi'), -16.92481569), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('mata'), -10.28799462), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('nata'), -10.69345973), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('sasi'), 7.323034009), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shashi'), -8.971692634), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shisata'), -10.26267682), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shushoma'), -20.30229654), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('ta'), -6.088289546), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tatomi'), -15.73786189), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tishenishu'), -25.52902026), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('toni'), -11.13974683), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tusa'), -5.449867265), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('ʃi'), -7.54617756), ] expected.sort(key=lambda t: t[2]) expected.reverse() with CanonicalVariantContext(unspecified_test_corpus, 'spelling', 'type') as c: calced = string_similarity(c, unspecified_test_corpus.find('sasi'), 'khorsi') for i, v in enumerate(expected): assert (abs(calced[i][2] - v[2]) < 0.0001)
def test_mass_relate_spelling_token(unspecified_test_corpus): expected = [(unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('atema'), 12.9671688), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('enuta'), -16.49795651), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('mashomisi'), -17.65533907), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('mata'), -7.337667817), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('nata'), -9.088485208), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('sasi'), -13.8251823), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shashi'), -17.52074498), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shisata'), -12.59737574), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shushoma'), -14.82488063), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('ta'), -9.8915809), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tatomi'), -14.6046824), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tishenishu'), -27.61147254), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('toni'), -16.14809881), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tusa'), -13.8308605), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('ʃi'), -22.4838445)] expected.sort(key=lambda t: t[2]) expected.reverse() with CanonicalVariantContext(unspecified_test_corpus, 'spelling', 'token') as c: calced = string_similarity(c, unspecified_test_corpus.find('atema'), 'khorsi') for i, v in enumerate(expected): assert (abs(calced[i][2] - v[2]) < 0.0001) expected = [(unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('atema'), -13.8251823), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('enuta'), -14.48366705), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('mashomisi'), -16.62778969), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('mata'), -10.46022702), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('nata'), -10.55425597), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('sasi'), 6.832376308), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shashi'), -7.235843913), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shisata'), -9.913037922), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shushoma'), -19.77169406), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('ta'), -5.382988852), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tatomi'), -16.07045316), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tishenishu'), -24.92713472), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('toni'), -11.39132061), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tusa'), -5.172159875), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('ʃi'), -10.12650306)] expected.sort(key=lambda t: t[2]) expected.reverse() with CanonicalVariantContext(unspecified_test_corpus, 'spelling', 'token') as c: calced = string_similarity(c, unspecified_test_corpus.find('sasi'), 'khorsi') for i, v in enumerate(expected): assert (abs(calced[i][2] - v[2]) < 0.0001)
def test_mass_relate_transcription_token(unspecified_test_corpus): expected = [ (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('atema'), 12.10974787), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('enuta'), -15.29756722), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('mashomisi'), -16.05808867), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('mata'), -8.574032654), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('nata'), -6.823215263), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('sasi'), -14.77671518), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shashi'), -13.71767966), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shisata'), -11.34309371), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shushoma'), -11.19329949), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('ta'), -9.205644162), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tatomi'), -13.74726148), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tishenishu'), -23.12247048), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('toni'), -15.1191937), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tusa'), -13.79217439), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('ʃi'), -15.68503325), ] expected.sort(key=lambda t: t[2]) expected.reverse() with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: calced = string_similarity(c, unspecified_test_corpus.find('atema'), 'khorsi') for i, v in enumerate(expected): assert (abs(calced[i][2] - v[2]) < 0.0001) expected = [ (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('atema'), -14.77671518), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('enuta'), -15.43519993), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('mashomisi'), -13.96361833), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('mata'), -11.58324408), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('nata'), -11.67727303), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('sasi'), 8.126877557), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shashi'), -9.734809346), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shisata'), -7.840021077), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shushoma'), -15.95332831), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('ta'), -6.848974285), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tatomi'), -16.85050186), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tishenishu'), -20.51761446), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('toni'), -12.51433768), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tusa'), -4.829191506), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('ʃi'), -5.994066536), ] expected.sort(key=lambda t: t[2]) expected.reverse() with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: calced = string_similarity(c, unspecified_test_corpus.find('sasi'), 'khorsi') for i, v in enumerate(expected): assert (abs(calced[i][2] - v[2]) < 0.0001)
def test_basic_corpus_probs(unspecified_test_corpus): with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: prob_dict = c.get_phone_probs(1, log_count=False, probability=False) expected = { (('i', ), 1): 3, (('s', ), 2): 3, (('ʃ', ), 6): 1, (('t', ), 3): 1, (('m', ), 3): 1, (('s', ), 0): 1, (('o', ), 1): 1, (('u', ), 1): 2, (('u', ), 2): 1, (('n', ), 4): 1, (('o', ), 3): 3, (('ʃ', ), 2): 4, (('m', ), 4): 3, (('n', ), 0): 1, (('t', ), 0): 5, (('ʃ', ), 0): 4, (('e', ), 3): 1, (('ɑ', ), 5): 2, (('m', ), 0): 2, (('t', ), 1): 1, (('u', ), 7): 1, (('t', ), 4): 1, (('ɑ', ), 1): 7, (('i', ), 7): 1, (('t', ), 2): 3, (('s', ), 6): 1, (('ɑ', ), 3): 4, (('i', ), 5): 3, (('e', ), 0): 1, (('i', ), 3): 3, (('n', ), 1): 1, (('n', ), 2): 1, (('ɑ', ), 0): 1, (('ɑ', ), 4): 2, (('e', ), 2): 1, 'total': { 0: 15, 1: 15, 2: 13, 3: 13, 4: 7, 5: 5, 6: 2, 7: 2 } } for k, v in expected.items(): if k == 'total': for k2, v2 in v.items(): assert (prob_dict[k][k2] == v2) continue assert (prob_dict[k] == v) with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: prob_dict = c.get_phone_probs(1, log_count=False, probability=True) for k, v in expected.items(): if k == 'total': continue assert (prob_dict[k] == v / expected['total'][k[1]]) with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: prob_dict = c.get_phone_probs(1, log_count=True, probability=True) expected = { (('ɑ', ), 0): 0.0587828456, (('t', ), 1): 0.0587828456, #atema (('e', ), 2): 0.0668038019, (('m', ), 3): 0.0668038019, (('ɑ', ), 4): 0.2544134544, (('e', ), 0): 0.0587828456, (('n', ), 1): 0.0587828456, #enuta (('u', ), 2): 0.0668038019, (('t', ), 3): 0.0668038019, (('t', ), 0): 0.4333449434, (('ɑ', ), 1): 0.4373852679, #ta (('m', ), 0): 0.0564463785, (('t', ), 2): 0.0928330493, #mata (('ɑ', ), 3): 0.1657810293, (('n', ), 0): 0.0169920531 #nata } for k, v in expected.items(): assert (abs(prob_dict[k] - v) < 0.0001) with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: prob_dict = c.get_phone_probs(2, log_count=True, probability=True) expected = { (('ɑ', 't'), 0): 0.0587828456, (('t', 'e'), 1): 0.0668038019, #atema (('e', 'm'), 2): 0.0668038019, (('m', 'ɑ'), 3): 0.1272067272, (('e', 'n'), 0): 0.0587828456, (('n', 'u'), 1): 0.0668038019, #enuta (('u', 't'), 2): 0.0668038019, (('t', 'ɑ'), 3): 0.1272067272, (('t', 'ɑ'), 0): 0.1507780332, #ta (('m', 'ɑ'), 0): 0.0564463785, (('ɑ', 't'), 1): 0.0928330493, #mata (('t', 'ɑ'), 2): 0.0386212588, (('n', 'ɑ'), 0): 0.0169920531 #nata } #print(list(prob_dict.keys())) for k, v in expected.items(): assert (abs(prob_dict[k] - v) < 0.0001)
def test_mass_relate_transcription_type(unspecified_test_corpus): expected = [ (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('atema'), 10.54988612), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('enuta'), -13.35737022), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('mashomisi'), -16.64202823), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('mata'), -5.95476627), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('nata'), -8.178638789), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('sasi'), -14.85026877), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shashi'), -13.67469544), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shisata'), -12.0090178), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('shushoma'), -12.51154463), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('ta'), -8.296421824), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tatomi'), -13.01231991), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tishenishu'), -23.85818691), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('toni'), -14.54716897), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('tusa'), -13.85402179), (unspecified_test_corpus.find('atema'), unspecified_test_corpus.find('ʃi'), -14.60340869), ] expected.sort(key=lambda t: t[2]) expected.reverse() with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: calced = string_similarity(c, unspecified_test_corpus.find('atema'), 'khorsi') for i, v in enumerate(expected): assert (abs(calced[i][2] - v[2]) < 0.0001) expected = [ (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('atema'), -14.85026877), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('enuta'), -16.64202823), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('mashomisi'), -12.94778139), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('mata'), -11.67221494), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('nata'), -12.07768004), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('sasi'), 8.812614836), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shashi'), -11.93742415), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shisata'), -7.90637444), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('shushoma'), -18.22899329), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('ta'), -7.683230889), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tatomi'), -16.91136117), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tishenishu'), -21.83498509), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('toni'), -12.52396715), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('tusa'), -5.239146233), (unspecified_test_corpus.find('sasi'), unspecified_test_corpus.find('ʃi'), -6.943894326), ] expected.sort(key=lambda t: t[2]) expected.reverse() with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: calced = string_similarity(c, unspecified_test_corpus.find('sasi'), 'khorsi') for i, v in enumerate(expected): assert (abs(calced[i][2] - v[2]) < 0.0001)
def test_deltah(unspecified_test_corpus): type_calls = [ ({ 'segment_pairs': [('s', 'ʃ')] }, 0.13333), ({ 'segment_pairs': [('m', 'n')] }, 0.13333), ({ 'segment_pairs': [('e', 'o')] }, 0), ({ 'segment_pairs': [('s', 'ʃ'), ('m', 'n'), ('e', 'o')] }, 0.26667), ] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type') as c: for kwargs, v in type_calls: assert (abs(deltah_fl(c, **kwargs) - v) < 0.0001) type_calls = [({ 'segment_pairs': [('s', 'ʃ')] }, 0.16667), ({ 'segment_pairs': [('m', 'n')] }, 0), ({ 'segment_pairs': [('e', 'o')] }, 0), ({ 'segment_pairs': [('s', 'ʃ'), ('m', 'n'), ('e', 'o')] }, 0.16667)] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'type', frequency_threshold=3) as c: for kwargs, v in type_calls: assert (abs(deltah_fl(c, **kwargs) - v) < 0.0001) token_calls = [ ({ 'segment_pairs': [('s', 'ʃ')] }, 0.24794), ({ 'segment_pairs': [('m', 'n')] }, 0.00691), ({ 'segment_pairs': [('e', 'o')] }, 0), ({ 'segment_pairs': [('s', 'ʃ'), ('m', 'n'), ('e', 'o')] }, 0.25485), ] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token') as c: for kwargs, v in token_calls: assert (abs(deltah_fl(c, **kwargs) - v) < 0.0001) token_calls = [ ({ 'segment_pairs': [('s', 'ʃ')] }, 0.25053), ({ 'segment_pairs': [('m', 'n')] }, 0), ({ 'segment_pairs': [('e', 'o')] }, 0), ({ 'segment_pairs': [('s', 'ʃ'), ('m', 'n'), ('e', 'o')] }, 0.25053), ] with CanonicalVariantContext(unspecified_test_corpus, 'transcription', 'token', frequency_threshold=3) as c: for kwargs, v in token_calls: assert (abs(deltah_fl(c, **kwargs) - v) < 0.0001)