def test_lev3(self): for t in TEXTS: s1, s2 = t[0], t[1] d = fClassicalLevDistance()(s1, s2) m1 = DistanceMatrix() fClassicalLevDistance().fill_matrix(s1, s2, m1) m2 = DistanceMatrix() fLevDistance().fill_matrix(s1, s2, m2) m3 = DistanceMatrix() fLevDistance2().fill_matrix(s1, s2, m3) m4 = DistanceMatrix(111) fLevDistanceDiag(len(s1) - 1).fill_matrix(s1, s2, m4) m5 = fLevDistanceDiag(len(s1) - 1).matrix(s1, s2) # C-code m6 = c_lev_distance.fLevDistance().matrix(s1, s2) m6_1 = DistanceMatrix() c_lev_distance.fLevDistance().fill_matrix(s1, s2, m6_1) m7 = c_lev_distance.fLevDistanceDiag(len(s1) - 1).matrix(s1, s2) m7_1 = DistanceMatrix(def_value = 111) c_lev_distance.fLevDistanceDiag(len(s1) - 1).fill_matrix(s1, s2, m7_1) self.assertEqual(m1, m2) self.assertEqual(m2, m3) self.assertEqual(m3, m4) self.assertEqual(m4, m5) self.assertEqual(m5, m6) self.assertEqual(m6, m7) self.assertEqual(m6, m6_1) self.assertEqual(m7, m7_1) # test C and Python diagonal m8 = c_lev_distance.fLevDistanceDiag(2).matrix(s1, s2) m9 = fLevDistanceDiag(2).matrix(s1, s2) self.assertEqual(m6, m7)
def test_lev3(self): for t in TEXTS: s1, s2 = t[0], t[1] d = fClassicalLevDistance()(s1, s2) m1 = DistanceMatrix() fClassicalLevDistance().fill_matrix(s1, s2, m1) m2 = DistanceMatrix() fLevDistance().fill_matrix(s1, s2, m2) m3 = DistanceMatrix() fLevDistance2().fill_matrix(s1, s2, m3) m4 = DistanceMatrix(111) fLevDistanceDiag(len(s1) - 1).fill_matrix(s1, s2, m4) m5 = fLevDistanceDiag(len(s1) - 1).matrix(s1, s2) # C-code m6 = c_lev_distance.fLevDistance().matrix(s1, s2) m6_1 = DistanceMatrix() c_lev_distance.fLevDistance().fill_matrix(s1, s2, m6_1) m7 = c_lev_distance.fLevDistanceDiag(len(s1) - 1).matrix(s1, s2) m7_1 = DistanceMatrix(def_value=111) c_lev_distance.fLevDistanceDiag(len(s1) - 1).fill_matrix( s1, s2, m7_1) self.assertEqual(m1, m2) self.assertEqual(m2, m3) self.assertEqual(m3, m4) self.assertEqual(m4, m5) self.assertEqual(m5, m6) self.assertEqual(m6, m7) self.assertEqual(m6, m6_1) self.assertEqual(m7, m7_1) # test C and Python diagonal m8 = c_lev_distance.fLevDistanceDiag(2).matrix(s1, s2) m9 = fLevDistanceDiag(2).matrix(s1, s2) self.assertEqual(m6, m7)
def test_ldistance(self): ed = fClassicalLevDistance() self.assertEqual(ed("Text", "Text"), 0) ed = fClassicalLevDistance() self.assertEqual(ed("Texmt", "sdText"), 3) ed = fLevDistance() self.assertEqual(ed("Text", "Text"), 0) ed = fLevDistance() self.assertEqual(ed("Texmt", "sdText"), 3) em = [ [1, 2, 2, 3, 4, 5 ], [2, 2, 3, 2, 3, 4 ], [3, 3, 3, 3, 2, 3 ], [4, 4, 4, 4, 3, 3 ], [5, 5, 5, 5, 4, 3 ] ] m = DistanceMatrix() fClassicalLevDistance().fill_matrix("Texmt", "sdText", m) for row in range(len(em)): for col in range(len(em[row])): self.assertEqual(em[row][col], m.get(row,col)) m = fLevDistance().fill_matrix("Texmt", "sdText", DistanceMatrix()) for row in range(len(em)): for col in range(len(em[row])): self.assertEqual(em[row][col], m.get(row,col)) mp = [(4, 5), (3, 4), (2, 4), (1, 3), (0, 2), (0, 1), (0, 0)] i = 0 for e in fMinPath()(m): self.assertEqual(e, mp[i]) i+=1 f = fDistanceMatch(fLevDistance) self.assertEqual(f("Text 1", "Text 2"), True) f = fDistanceMatch(fLevDistance) self.assertEqual(f("Text 1", "Tessxt 2"), False)
def test_ldistance(self): ed = fClassicalLevDistance() self.assertEqual(ed("Text", "Text"), 0) ed = fClassicalLevDistance() self.assertEqual(ed("Texmt", "sdText"), 3) ed = fLevDistance() self.assertEqual(ed("Text", "Text"), 0) ed = fLevDistance() self.assertEqual(ed("Texmt", "sdText"), 3) em = [[1, 2, 2, 3, 4, 5], [2, 2, 3, 2, 3, 4], [3, 3, 3, 3, 2, 3], [4, 4, 4, 4, 3, 3], [5, 5, 5, 5, 4, 3]] m = DistanceMatrix() fClassicalLevDistance().fill_matrix("Texmt", "sdText", m) for row in range(len(em)): for col in range(len(em[row])): self.assertEqual(em[row][col], m.get(row, col)) m = fLevDistance().fill_matrix("Texmt", "sdText", DistanceMatrix()) for row in range(len(em)): for col in range(len(em[row])): self.assertEqual(em[row][col], m.get(row, col)) mp = [(4, 5), (3, 4), (2, 4), (1, 3), (0, 2), (0, 1), (0, 0)] i = 0 for e in fMinPath()(m): self.assertEqual(e, mp[i]) i += 1 f = fDistanceMatch(fLevDistance) self.assertEqual(f("Text 1", "Text 2"), True) f = fDistanceMatch(fLevDistance) self.assertEqual(f("Text 1", "Tessxt 2"), False)
def test_lev1(self): s1 = s2 = "abcdef gk lmn" self.assertEqual(fLevDistance()(s1, s2), 0) self.assertEqual(fLevDistance2()(s1, s2), 0) self.assertEqual(fClassicalLevDistance()(s1, s2), 0) self.assertEqual(fLevDistanceDiag()(s1, s2), 0) # C code testing self.assertEqual(c_lev_distance.fLevDistance()(s1, s2), 0) self.assertEqual(c_lev_distance.fLevDistanceDiag()(s1, s2), 0) # unicode strings (contains Greek characters) s1 = s2 = u"abcdef \u03A0 gk \u03A3 lmn \u03A9" self.assertEqual(c_lev_distance.fLevDistance()(s1, s2), 0) self.assertEqual(c_lev_distance.fLevDistanceDiag()(s1, s2), 0)
def test_lev1(self): s1 = s2 = "abcdef gk lmn" self.assertEqual(fLevDistance()(s1, s2) , 0) self.assertEqual(fLevDistance2()(s1, s2) , 0) self.assertEqual(fClassicalLevDistance()(s1, s2) , 0) self.assertEqual(fLevDistanceDiag()(s1, s2) , 0) # C code testing self.assertEqual(c_lev_distance.fLevDistance()(s1, s2) , 0) self.assertEqual(c_lev_distance.fLevDistanceDiag()(s1, s2) , 0) # unicode strings (contains Greek characters) s1 = s2 = u"abcdef \u03A0 gk \u03A3 lmn \u03A9" self.assertEqual(c_lev_distance.fLevDistance()(s1, s2) , 0) self.assertEqual(c_lev_distance.fLevDistanceDiag()(s1, s2) , 0)
def test_lev2(self): for t in TEXTS: s1, s2 = t[0], t[1] d = fClassicalLevDistance()(s1, s2) self.assertEqual(fLevDistance()(s1, s2) , d) self.assertEqual(fLevDistance2()(s1, s2) , d) # C-code self.assertEqual(c_lev_distance.fLevDistance()(s1, s2) , d) self.assertEqual(c_lev_distance.fLevDistanceDiag(len(s1))(s1, s2) , d) # diagonal levenshtein self.assertEqual(fLevDistanceDiag(len(s1))(s1, s2) , d) # test diag bounds handling self.assertEqual(fLevDistanceDiag(10*len(s1))(s1, s2) , d) self.assertEqual(c_lev_distance.fLevDistanceDiag(10*len(s1))(s1, s2) , d)
def test_lev2(self): for t in TEXTS: s1, s2 = t[0], t[1] d = fClassicalLevDistance()(s1, s2) self.assertEqual(fLevDistance()(s1, s2), d) self.assertEqual(fLevDistance2()(s1, s2), d) # C-code self.assertEqual(c_lev_distance.fLevDistance()(s1, s2), d) self.assertEqual( c_lev_distance.fLevDistanceDiag(len(s1))(s1, s2), d) # diagonal levenshtein self.assertEqual(fLevDistanceDiag(len(s1))(s1, s2), d) # test diag bounds handling self.assertEqual(fLevDistanceDiag(10 * len(s1))(s1, s2), d) self.assertEqual( c_lev_distance.fLevDistanceDiag(10 * len(s1))(s1, s2), d)
t = time.time() #prepare_data() #step1_fetch_text(os.path.join(data_dir, "xml")) #step2_run_tnt(os.path.join(data_dir, "txt")) #step3_calc_lev_distance() #step4_calc_minpath() #step5_calc_minpath_dt() #step6_cleantnt_txt() #step5_calc_minpath_dt("cleaned-tnt", "txt", size_limit=1000) from gravity.tae.match.lev_distance import fLevDistanceDiag #s1 = "Song about Alice dream" #s2 = "Song al?ce dream" s1 = " abc d dfg rer klm" s2 = "abc dfg klm" m = DistanceMatrix(111) #fLevDistanceDiag(1).fill_matrix(s2, s1, m) fLevDistance().fill_matrix(s2, s1, m) print m.toString(s2, s1, fMinPath()(m)) print "Elapsed time: ", (time.time() - t)
step1_fetch_text(os.path.join(data_dir, "xml")) step2_run_tnt(os.path.join(data_dir, "txt")) step6_cleantnt_txt() t = time.time() #prepare_data() #step1_fetch_text(os.path.join(data_dir, "xml")) #step2_run_tnt(os.path.join(data_dir, "txt")) #step3_calc_lev_distance() #step4_calc_minpath() #step5_calc_minpath_dt() #step6_cleantnt_txt() #step5_calc_minpath_dt("cleaned-tnt", "txt", size_limit=1000) from gravity.tae.match.lev_distance import fLevDistanceDiag #s1 = "Song about Alice dream" #s2 = "Song al?ce dream" s1 = " abc d dfg rer klm" s2 = "abc dfg klm" m = DistanceMatrix(111) #fLevDistanceDiag(1).fill_matrix(s2, s1, m) fLevDistance().fill_matrix(s2, s1, m) print m.toString(s2, s1, fMinPath()(m)) print "Elapsed time: ", (time.time() - t)