def test_discounted_levenshtein_aligmnent(self): """Test abydos.distance.DiscountedLevenshtein.alignment.""" self.assertEqual(self.cmp.alignment('', ''), (0, '', '')) self.assertEqual(self.cmp.alignment('a', 'a'), (0.0, 'a', 'a')) self.assertEqual(self.cmp.alignment('ab', 'ab'), (0.0, 'ab', 'ab')) self.assertEqual(self.cmp.alignment('', 'a'), (1.0, '-', 'a')) self.assertEqual( self.cmp.alignment('', 'ab'), (1.845793595028118, '--', 'ab') ) self.assertEqual(self.cmp.alignment('a', 'c'), (1.0, 'a', 'c')) self.assertEqual(self.cmp.alignment('abc', 'ac'), (1.0, 'abc', 'a-c')) self.assertEqual( self.cmp.alignment('abbc', 'ac'), (1.845793595028118, 'abbc', 'a--c'), ) self.assertEqual( self.cmp.alignment('abbc', 'abc'), (0.8457935950281179, 'abbc', 'ab-c'), ) self.assertEqual( DiscountedLevenshtein(mode='osa').alignment('Niall', 'Naill'), (0.8457935950281179, 'Niall', 'Naill'), ) self.assertEqual( self.cmp.alignment('abcd', 'efgh'), (3.594032108779918, 'abcd', 'efgh'), ) self.assertEqual( self.cmp.alignment('Nigel', 'Niall'), (1.5940321087799176, 'Nigel', 'Niall'), ) self.assertEqual( self.cmp.alignment('Niall', 'Nigel'), (1.5940321087799176, 'Niall', 'Nigel'), ) self.assertEqual( self.cmp.alignment('Colin', 'Coiln'), (1.5940321087799176, 'Coli-n', 'Co-iln'), ) self.assertEqual( self.cmp.alignment('Coiln', 'Colin'), (1.5940321087799176, 'Coil-n', 'Co-lin'), ) self.assertEqual( self.cmp.alignment('cccColin', 'Colin'), (2.594032108779918, 'cccColin', '---Colin'), ) self.assertEqual( self.cmp.alignment('Colin', 'cccColin'), (2.594032108779918, '---Colin', 'cccColin'), )
def __init__(self, model='latin'): self.model = model self.impH = input_helpers.InputHelper() self.ST = syllable_tokenizer.SyllableTokenizer() # Phonetic Encoder self.pe = Ainsworth() # Soundex Firstname Algorithm self.pshp_soundex_first = PSHPSoundexFirst() # String Distance algorithms self.algos = [ IterativeSubString(), BISIM(), DiscountedLevenshtein(), Prefix(), LCSstr(), MLIPNS(), Strcmp95(), MRA(), Editex(), SAPS(), FlexMetric(), JaroWinkler(mode='Jaro'), HigueraMico(), Sift4(), Eudex(), ALINE(), Covington(), PhoneticEditDistance() ] self.algo_names = [ 'iterativesubstring', 'bisim', 'discountedlevenshtein', 'prefix', 'lcsstr', 'mlipns', 'strcmp95', 'mra', 'editex', 'saps', 'flexmetric', 'jaro', 'higueramico', 'sift4', 'eudex', 'aline', 'covington', 'phoneticeditdistance' ] # extract model tarball into directory if doesnt exist model_dir = os.path.join(os.path.dirname(__file__), "models", self.model) if not os.path.exists(model_dir): os.makedirs(model_dir) tar = tarfile.open( os.path.join(os.path.dirname(__file__), "models", self.model + ".tar.gz"), "r:gz") tar.extractall(model_dir) tar.close() # String Distance Pipeline (Level 0/Base Model) self.baseModel = joblib.load(os.path.join(model_dir, 'base.pkl')) # Character Embedding Network (Level 0/Base Model) self.vocab = preprocess.VocabularyProcessor( max_document_length=15, min_frequency=0).restore(os.path.join(model_dir, 'vocab')) siamese_model = os.path.join(model_dir, 'siamese') graph = tf.Graph() with graph.as_default() as graph: self.sess = tf.Session() with self.sess.as_default(): # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph( '{}.meta'.format(siamese_model)) self.sess.run(tf.global_variables_initializer()) saver.restore(self.sess, siamese_model) # Get the placeholders from the graph by name self.input_x1 = graph.get_operation_by_name('input_x1').outputs[0] self.input_x2 = graph.get_operation_by_name('input_x2').outputs[0] self.dropout_keep_prob = graph.get_operation_by_name( 'dropout_keep_prob').outputs[0] self.prediction = graph.get_operation_by_name( 'output/distance').outputs[0] self.sim = graph.get_operation_by_name( 'accuracy/temp_sim').outputs[0] # Logreg (Level 1/Meta Model) self.metaModel = joblib.load(os.path.join(model_dir, 'meta.pkl')) # seen names (mapping dict from raw name to processed name) self.seen_names = {} # seen pairs (mapping dict from name pair tuple to similarity) self.seen_pairs = {}
#Featurizer packges import unidecode from abydos.distance import (IterativeSubString, BISIM, DiscountedLevenshtein, Prefix, LCSstr, MLIPNS, Strcmp95, MRA, Editex, SAPS, FlexMetric, JaroWinkler, HigueraMico, Sift4, Eudex, ALINE, PhoneticEditDistance) from abydos.phonetic import PSHPSoundexFirst, Ainsworth from abydos.phones import * import re from sklearn.preprocessing import MinMaxScaler # Featurizer pshp_soundex_first = PSHPSoundexFirst() pe = Ainsworth() iss = IterativeSubString() bisim = BISIM() dlev = DiscountedLevenshtein() prefix = Prefix() lcs = LCSstr() mlipns = MLIPNS() strcmp95 = Strcmp95() mra = MRA() editex = Editex() saps = SAPS() flexmetric = FlexMetric() jaro = JaroWinkler(mode='Jaro') higuera_mico = HigueraMico() sift4 = Sift4() eudex = Eudex() aline = ALINE() phonetic_edit = PhoneticEditDistance() algos = [iss, bisim, dlev, prefix, lcs, mlipns, strcmp95, mra, editex, saps, flexmetric, jaro, higuera_mico, sift4, eudex,
def __init__(self, model='latin', prefilter=True, allow_alt_surname=True, allow_initials=True, allow_missing_components=True): # user-provided parameters self.model = model self.allow_alt_surname = allow_alt_surname self.allow_initials = allow_initials self.allow_missing_components = allow_missing_components self.prefilter = prefilter if self.prefilter: self.refined_soundex = { 'b': 1, 'p': 1, 'f': 2, 'v': 2, 'c': 3, 'k': 3, 's': 3, 'g': 4, 'j': 4, 'q': 5, 'x': 5, 'z': 5, 'd': 6, 't': 6, 'l': 7, 'm': 8, 'n': 8, 'r': 9 } # verify user-supplied class arguments model_dir = self.validate_parameters() self.impH = input_helpers.InputHelper() # Phonetic Encoder self.pe = Ainsworth() # Soundex Firstname Algorithm self.pshp_soundex_first = PSHPSoundexFirst() # Soundex Lastname Algorithm self.pshp_soundex_last = PSHPSoundexLast() # String Distance algorithms self.algos = [IterativeSubString(), BISIM(), DiscountedLevenshtein(), Prefix(), LCSstr(), MLIPNS(), Strcmp95(), MRA(), Editex(), SAPS(), FlexMetric(), JaroWinkler(mode='Jaro'), HigueraMico(), Sift4(), Eudex(), ALINE(), CovingtonGuard(), PhoneticEditDistance()] self.algo_names = ['iterativesubstring', 'bisim', 'discountedlevenshtein', 'prefix', 'lcsstr', 'mlipns', 'strcmp95', 'mra', 'editex', 'saps', 'flexmetric', 'jaro', 'higueramico', 'sift4', 'eudex', 'aline', 'covington', 'phoneticeditdistance'] # String Distance Pipeline (Level 0/Base Model) self.baseModel = joblib.load(os.path.join(model_dir, 'base.pkl')) # Character Embedding Network (Level 0/Base Model) self.vocab = preprocess.VocabularyProcessor(max_document_length=15, min_frequency=0).restore( os.path.join(model_dir, 'vocab')) siamese_model = os.path.join(model_dir, 'siamese') # start tensorflow session graph = tf.Graph() with graph.as_default() as graph: self.sess = tf.Session() if tf.__version__[0] == '1' else tf.compat.v1.Session() with self.sess.as_default(): # Load the saved meta graph and restore variables if tf.__version__[0] == '1': saver = tf.train.import_meta_graph('{}.meta'.format(siamese_model)) self.sess.run(tf.global_variables_initializer()) else: saver = tf.compat.v1.train.import_meta_graph('{}.meta'.format(siamese_model)) self.sess.run(tf.compat.v1.global_variables_initializer()) saver.restore(self.sess, siamese_model) # Get the placeholders from the graph by name self.input_x1 = graph.get_operation_by_name('input_x1').outputs[0] self.input_x2 = graph.get_operation_by_name('input_x2').outputs[0] self.dropout_keep_prob = graph.get_operation_by_name('dropout_keep_prob').outputs[0] self.prediction = graph.get_operation_by_name('output/distance').outputs[0] self.sim = graph.get_operation_by_name('accuracy/temp_sim').outputs[0] # Logreg (Level 1/Meta Model) self.metaModel = joblib.load(os.path.join(model_dir, 'meta.pkl')) # seen names (mapping dict from raw name to processed name) self.seen_names = {} # seen pairs (mapping dict from name pair tuple to similarity) self.seen_pairs = {} # user scores (mapping dict from name pair tuple to similarity) self.user_scores = {}
class DiscountedLevenshteinTestCases(unittest.TestCase): """Test DiscountedLevenshtein functions. abydos.distance.DiscountedLevenshtein """ cmp = DiscountedLevenshtein() cmp_coda = DiscountedLevenshtein(discount_from='coda') def test_discounted_levenshtein_aligmnent(self): """Test abydos.distance.DiscountedLevenshtein.alignment.""" self.assertEqual(self.cmp.alignment('', ''), (0, '', '')) self.assertEqual(self.cmp.alignment('a', 'a'), (0.0, 'a', 'a')) self.assertEqual(self.cmp.alignment('ab', 'ab'), (0.0, 'ab', 'ab')) self.assertEqual(self.cmp.alignment('', 'a'), (1.0, '-', 'a')) self.assertEqual( self.cmp.alignment('', 'ab'), (1.845793595028118, '--', 'ab') ) self.assertEqual(self.cmp.alignment('a', 'c'), (1.0, 'a', 'c')) self.assertEqual(self.cmp.alignment('abc', 'ac'), (1.0, 'abc', 'a-c')) self.assertEqual( self.cmp.alignment('abbc', 'ac'), (1.845793595028118, 'abbc', 'a--c'), ) self.assertEqual( self.cmp.alignment('abbc', 'abc'), (0.8457935950281179, 'abbc', 'ab-c'), ) self.assertEqual( DiscountedLevenshtein(mode='osa').alignment('Niall', 'Naill'), (0.8457935950281179, 'Niall', 'Naill'), ) self.assertEqual( self.cmp.alignment('abcd', 'efgh'), (3.594032108779918, 'abcd', 'efgh'), ) self.assertEqual( self.cmp.alignment('Nigel', 'Niall'), (1.5940321087799176, 'Nigel', 'Niall'), ) self.assertEqual( self.cmp.alignment('Niall', 'Nigel'), (1.5940321087799176, 'Niall', 'Nigel'), ) self.assertEqual( self.cmp.alignment('Colin', 'Coiln'), (1.5940321087799176, 'Coli-n', 'Co-iln'), ) self.assertEqual( self.cmp.alignment('Coiln', 'Colin'), (1.5940321087799176, 'Coil-n', 'Co-lin'), ) self.assertEqual( self.cmp.alignment('cccColin', 'Colin'), (2.594032108779918, 'cccColin', '---Colin'), ) self.assertEqual( self.cmp.alignment('Colin', 'cccColin'), (2.594032108779918, '---Colin', 'cccColin'), ) def test_discounted_levenshtein_dist_abs(self): """Test abydos.distance.DiscountedLevenshtein.dist_abs.""" # Base cases self.assertEqual(self.cmp.dist_abs('', ''), 0.0) self.assertEqual(self.cmp.dist_abs('a', ''), 1.0) self.assertEqual(self.cmp.dist_abs('', 'a'), 1.0) self.assertEqual(self.cmp.dist_abs('abc', ''), 2.845793595028118) self.assertEqual(self.cmp.dist_abs('', 'abc'), 2.845793595028118) self.assertEqual(self.cmp.dist_abs('abc', 'abc'), 0.0) self.assertEqual(self.cmp.dist_abs('abcd', 'efgh'), 3.594032108779918) self.assertAlmostEqual( self.cmp.dist_abs('Nigel', 'Niall'), 1.5940321087799176 ) self.assertAlmostEqual( self.cmp.dist_abs('Niall', 'Nigel'), 1.5940321087799176 ) self.assertAlmostEqual( self.cmp.dist_abs('Colin', 'Coiln'), 1.5940321087799176 ) self.assertAlmostEqual( self.cmp.dist_abs('Coiln', 'Colin'), 1.5940321087799176 ) self.assertAlmostEqual( self.cmp.dist_abs('ATCAACGAGT', 'AACGATTAG'), 3.480037325627888 ) def test_discounted_levenshtein_dist(self): """Test abydos.distance.DiscountedLevenshtein.dist.""" # Base cases self.assertEqual(self.cmp.dist('', ''), 0.0) self.assertEqual(self.cmp.dist('a', ''), 1.0) self.assertEqual(self.cmp.dist('', 'a'), 1.0) self.assertEqual(self.cmp.dist('abc', ''), 1.0) self.assertEqual(self.cmp.dist('', 'abc'), 1.0) self.assertEqual(self.cmp.dist('abc', 'abc'), 0.0) self.assertEqual(self.cmp.dist('abcd', 'efgh'), 1.0) self.assertAlmostEqual( self.cmp.dist('Nigel', 'Niall'), 0.3729338516783247 ) self.assertAlmostEqual( self.cmp.dist('Niall', 'Nigel'), 0.3729338516783247 ) self.assertAlmostEqual( self.cmp.dist('Colin', 'Coiln'), 0.3729338516783247 ) self.assertAlmostEqual( self.cmp.dist('Coiln', 'Colin'), 0.3729338516783247 ) self.assertAlmostEqual( self.cmp.dist('ATCAACGAGT', 'AACGATTAG'), 0.49025364879218414 ) self.assertEqual(self.cmp_coda.dist('', ''), 0.0) self.assertEqual(self.cmp_coda.dist('a', ''), 1.0) self.assertEqual(self.cmp_coda.dist('', 'a'), 1.0) self.assertEqual(self.cmp_coda.dist('abc', ''), 1.0) self.assertEqual(self.cmp_coda.dist('', 'abc'), 1.0) self.assertEqual(self.cmp_coda.dist('abc', 'abc'), 0.0) self.assertEqual(self.cmp_coda.dist('abcd', 'efgh'), 1.0) self.assertAlmostEqual( self.cmp_coda.dist('Nigel', 'Niall'), 0.43183503707707127 ) self.assertAlmostEqual( self.cmp_coda.dist('Niall', 'Nigel'), 0.43183503707707127 ) self.assertAlmostEqual( self.cmp_coda.dist('Colin', 'Coiln'), 0.43183503707707127 ) self.assertAlmostEqual( self.cmp_coda.dist('Coiln', 'Colin'), 0.43183503707707127 ) self.assertAlmostEqual( self.cmp_coda.dist('ATCAACGAGT', 'AACGATTAG'), 0.49025364879218414 ) # coverage additions self.assertAlmostEqual( DiscountedLevenshtein(discount_func='exp').dist('Nigel', 'Niall'), 0.3776202500185377, ) self.assertAlmostEqual( DiscountedLevenshtein( discount_func=lambda x: 1 / (x + 2) ** 0.3 ).dist('Nigel', 'Niall'), 0.3808786265263343, ) self.assertAlmostEqual( self.cmp_coda.dist('da', 'dde'), 0.7027916583599728 ) self.assertAlmostEqual( self.cmp_coda.dist('d', 'dd'), 0.42289679751405895 ) self.assertAlmostEqual( DiscountedLevenshtein(discount_from='invalid value').dist( 'Nigel', 'Niall' ), 0.3729338516783247, ) self.assertAlmostEqual( DiscountedLevenshtein(mode='osa').dist('Nigel', 'Ngiall'), 0.4534644632194963, )
def test_discounted_levenshtein_dist(self): """Test abydos.distance.DiscountedLevenshtein.dist.""" # Base cases self.assertEqual(self.cmp.dist('', ''), 0.0) self.assertEqual(self.cmp.dist('a', ''), 1.0) self.assertEqual(self.cmp.dist('', 'a'), 1.0) self.assertEqual(self.cmp.dist('abc', ''), 1.0) self.assertEqual(self.cmp.dist('', 'abc'), 1.0) self.assertEqual(self.cmp.dist('abc', 'abc'), 0.0) self.assertEqual(self.cmp.dist('abcd', 'efgh'), 1.0) self.assertAlmostEqual( self.cmp.dist('Nigel', 'Niall'), 0.3729338516783247 ) self.assertAlmostEqual( self.cmp.dist('Niall', 'Nigel'), 0.3729338516783247 ) self.assertAlmostEqual( self.cmp.dist('Colin', 'Coiln'), 0.3729338516783247 ) self.assertAlmostEqual( self.cmp.dist('Coiln', 'Colin'), 0.3729338516783247 ) self.assertAlmostEqual( self.cmp.dist('ATCAACGAGT', 'AACGATTAG'), 0.49025364879218414 ) self.assertEqual(self.cmp_coda.dist('', ''), 0.0) self.assertEqual(self.cmp_coda.dist('a', ''), 1.0) self.assertEqual(self.cmp_coda.dist('', 'a'), 1.0) self.assertEqual(self.cmp_coda.dist('abc', ''), 1.0) self.assertEqual(self.cmp_coda.dist('', 'abc'), 1.0) self.assertEqual(self.cmp_coda.dist('abc', 'abc'), 0.0) self.assertEqual(self.cmp_coda.dist('abcd', 'efgh'), 1.0) self.assertAlmostEqual( self.cmp_coda.dist('Nigel', 'Niall'), 0.43183503707707127 ) self.assertAlmostEqual( self.cmp_coda.dist('Niall', 'Nigel'), 0.43183503707707127 ) self.assertAlmostEqual( self.cmp_coda.dist('Colin', 'Coiln'), 0.43183503707707127 ) self.assertAlmostEqual( self.cmp_coda.dist('Coiln', 'Colin'), 0.43183503707707127 ) self.assertAlmostEqual( self.cmp_coda.dist('ATCAACGAGT', 'AACGATTAG'), 0.49025364879218414 ) # coverage additions self.assertAlmostEqual( DiscountedLevenshtein(discount_func='exp').dist('Nigel', 'Niall'), 0.3776202500185377, ) self.assertAlmostEqual( DiscountedLevenshtein( discount_func=lambda x: 1 / (x + 2) ** 0.3 ).dist('Nigel', 'Niall'), 0.3808786265263343, ) self.assertAlmostEqual( self.cmp_coda.dist('da', 'dde'), 0.7027916583599728 ) self.assertAlmostEqual( self.cmp_coda.dist('d', 'dd'), 0.42289679751405895 ) self.assertAlmostEqual( DiscountedLevenshtein(discount_from='invalid value').dist( 'Nigel', 'Niall' ), 0.3729338516783247, ) self.assertAlmostEqual( DiscountedLevenshtein(mode='osa').dist('Nigel', 'Ngiall'), 0.4534644632194963, )