def test_fit_predict_regularized(self): incorrect = [ 'helloooo', 'freshh', 'ffb', 'h0me', 'wonderin', 'relaionship', 'hubby', 'krazii', 'mite', 'tropic' ] correct = [ 'hello', 'fresh', 'facebook', 'home', 'wondering', 'relationship', 'husband', 'crazy', 'might', 'topic' ] training = zip(incorrect, correct) fe = StringPairFeatureExtractor(match=True, numeric=True) xf = fe.fit_transform(training) model = Hacrf(l2_regularization=10.0) model.fit(xf, [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) print(model.parameters) expected_parameters = np.array([[-0.0569188, 0.07413339, 0.], [0.00187709, -0.06377866, 0.], [-0.01908823, 0.00586189, 0.], [0.01721114, -0.00636556, 0.], [0.01578279, 0.0078614, 0.], [-0.0139057, -0.00862948, 0.], [-0.00623241, 0.02937325, 0.], [0.00810951, -0.01774676, 0.]]) assert_array_almost_equal(model.parameters, expected_parameters, decimal=TEST_PRECISION) expected_probas = np.array([[0.5227226, 0.4772774], [0.52568993, 0.47431007], [0.4547091, 0.5452909], [0.51179222, 0.48820778], [0.46347576, 0.53652424], [0.45710098, 0.54289902], [0.46159657, 0.53840343], [0.42997978, 0.57002022], [0.47419724, 0.52580276], [0.50797852, 0.49202148]]) actual_predict_probas = model.predict_proba(xf) print(actual_predict_probas) assert_array_almost_equal(actual_predict_probas, expected_probas, decimal=TEST_PRECISION) expected_predictions = np.array([0, 0, 1, 0, 1, 1, 1, 1, 1, 0]) actual_predictions = model.predict(xf) assert_array_almost_equal(actual_predictions, expected_predictions, decimal=TEST_PRECISION)
def test_fit_predict(self): incorrect = [ 'helloooo', 'freshh', 'ffb', 'h0me', 'wonderin', 'relaionship', 'hubby', 'krazii', 'mite', 'tropic' ] correct = [ 'hello', 'fresh', 'facebook', 'home', 'wondering', 'relationship', 'husband', 'crazy', 'might', 'topic' ] training = zip(incorrect, correct) fe = StringPairFeatureExtractor(match=True, numeric=True) xf = fe.fit_transform(training) model = Hacrf() model.fit(xf, [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) expected_parameters = np.array([[-10.76945326, 144.03414923, 0.], [31.84369748, -106.41885651, 0.], [-52.08919467, 4.56943665, 0.], [31.01495044, -13.0593297, 0.], [49.77302218, -6.42566204, 0.], [-28.69877796, 24.47127009, 0.], [-85.34524911, 21.87370646, 0.], [106.41949333, 6.18587125, 0.]]) print(model.parameters) assert_array_almost_equal(model.parameters, expected_parameters, decimal=TEST_PRECISION) expected_probas = np.array([[1.00000000e+000, 3.51235685e-039], [1.00000000e+000, 4.79716208e-039], [1.00000000e+000, 2.82744641e-139], [1.00000000e+000, 6.49580729e-012], [9.99933798e-001, 6.62022561e-005], [8.78935957e-005, 9.99912106e-001], [4.84538335e-009, 9.99999995e-001], [1.25170233e-250, 1.00000000e+000], [2.46673086e-010, 1.00000000e+000], [1.03521293e-033, 1.00000000e+000]]) actual_predict_probas = model.predict_proba(xf) print(actual_predict_probas) assert_array_almost_equal(actual_predict_probas, expected_probas, decimal=TEST_PRECISION) expected_predictions = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) actual_predictions = model.predict(xf) assert_array_almost_equal(actual_predictions, expected_predictions, decimal=TEST_PRECISION)
def test_fit_predict_regularized_viterbi(self): incorrect = [ 'helloooo', 'freshh', 'ffb', 'h0me', 'wonderin', 'relaionship', 'hubby', 'krazii', 'mite', 'tropic' ] correct = [ 'hello', 'fresh', 'facebook', 'home', 'wondering', 'relationship', 'husband', 'crazy', 'might', 'topic' ] training = zip(incorrect, correct) fe = StringPairFeatureExtractor(match=True, numeric=True) xf = fe.fit_transform(training) model = Hacrf(l2_regularization=10.0, viterbi=True) model.fit(xf, [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) print(model.parameters) expected_parameters = np.array([[-0.0569188, 0.07413339, 0.], [0.00187709, -0.06377866, 0.], [-0.01908823, 0.00586189, 0.], [0.01721114, -0.00636556, 0.], [0.01578279, 0.0078614, 0.], [-0.0139057, -0.00862948, 0.], [-0.00623241, 0.02937325, 0.], [0.00810951, -0.01774676, 0.]]) assert_array_almost_equal(model.parameters, expected_parameters, decimal=TEST_PRECISION) expected_probas = np.array([[0.56394611, 0.43605389], [0.52977205, 0.47022795], [0.4751729, 0.5248271], [0.51183761, 0.48816239], [0.48608081, 0.51391919], [0.4986367, 0.5013633], [0.46947222, 0.53052778], [0.43233544, 0.56766456], [0.47463002, 0.52536998], [0.51265109, 0.48734891]]) actual_predict_probas = model.predict_proba(xf) print(actual_predict_probas) assert_array_almost_equal(actual_predict_probas, expected_probas, decimal=TEST_PRECISION) expected_predictions = np.array([0, 0, 1, 0, 1, 1, 1, 1, 1, 0]) actual_predictions = model.predict(xf) assert_array_almost_equal(actual_predictions, expected_predictions, decimal=TEST_PRECISION)
def train(self): # Training self.fe = StringPairFeatureExtractor(match=True, numeric=True, transition=True) if self.needTraining: lines = open(self.infile, 'r').readlines() # Generate Positive Correction Pair ppairs = [] ppairs = [ line.split('\t')[1].strip().split(' | ') for line in lines ] ppairs = [(pair[0], pair[i]) for pair in ppairs for i in xrange(1, len(pair))] # Generate Positive Training Correction Pairs and Testing Correction Pairs ppairs_train, ppairs_test = train_test_split(ppairs, test_size=200, random_state=1) self.ppairs_train = [ tuple(ppair_train) for ppair_train in ppairs_train ] self.ppairs_test = [ tuple(ppair_test) for ppair_test in ppairs_test ] # Generate Negative Training Correction Pairs incorrect = list(zip(*ppairs_train)[0]) shuffle(incorrect) correct = list(zip(*ppairs_train)[1]) npairs_train = zip(incorrect, correct) # Raw training set x_raw = ppairs_train + npairs_train # Label of the training set self.y_train = [0] * len(ppairs_train) + [1] * len(npairs_train) # Extract Features from the raw training set self.x_train = x_orig = self.fe.fit_transform(x_raw) #x_train, x_test, y_train, y_test = train_test_split(x_orig, y_orig, test_size=0.2, random_state=42) self.m = Hacrf(l2_regularization=10.0, optimizer=fmin_l_bfgs_b, optimizer_kwargs={'maxfun': 45}, state_machine=None) self.m.fit(self.x_train, self.y_train, verbosity=20) cPickle.dump(self.m, open('Corrector.pkl', 'wb')) else: print "start training" self.m = cPickle.load(open('Corrector.pkl', 'rb')) print "finish training"
def __init__(self): classes = ['match', 'non-match'] self.model = Hacrf(l2_regularization=100.0, state_machine=DefaultStateMachine(classes)) self.model.parameters = np.array( [[-0.22937526, 0.51326066], [0.01038001, -0.13348901], [-0.03062821, 0.13769178], [0.02024813, -0.01835538], [0.09208272, 0.15466022], [-0.08170265, -0.02484392], [-0.01762858, 0.17504624], [0.02800866, -0.04442708]], order='F') self.parameters = self.model.parameters.T self.model.classes = ['match', 'non-match'] self.feature_extractor = StringPairFeatureExtractor(match=True, numeric=False)
def test_transform_transition(self): s1 = "ba" s2 = "ca" # a . . # b . . # c a chars = StringPairFeatureExtractor.CHARACTERS nchars = len(chars) print(nchars) expected_x = np.zeros((2, 2, len(chars)**2 + 1)) expected_x[:, :, 0] = 1.0 expected_x[0, 0, 2 + nchars * 1 + 1] = 1.0 # b->c expected_x[0, 1, 0 + nchars * 1 + 1] = 1.0 # b->a expected_x[1, 0, 2 + nchars * 0 + 1] = 1.0 # a->c expected_x[1, 1, 0 + nchars * 0 + 1] = 1.0 # a->a test_extractor = StringPairFeatureExtractor(transition=True) actual_X = test_extractor.fit_transform([(s1, s2)]) assert_array_almost_equal(expected_x, actual_X[0])
def test_transform_binary(self): s1 = "kat1" s2 = "cat2" # 1 . . . n # t . . m . # a . m . . # k . . . . # c a t 2 expected_x = np.zeros((4, 4, 4)) expected_x[:, :, 0] = 2.0 expected_x[:, 0, 1] = 1.0 expected_x[0, :, 1] = 1.0 expected_x[1, 1, 2] = 1.0 expected_x[2, 2, 2] = 1.0 expected_x[3, 3, 3] = 1.0 test_extractor = StringPairFeatureExtractor(bias=2.0, start=True, match=True, numeric=True) actual_X = test_extractor.fit_transform([(s1, s2)]) assert_array_almost_equal(expected_x, actual_X[0])
import numpy, pyhacrf, nltk from pyhacrf import StringPairFeatureExtractor, Hacrf training_X = [] iv = open("iv.txt", "r").read() ovv = open("ovv.txt", "r").read() for p in iv.split("\n"): for q in ovv.split("\n"): training_X.append((p, q)) #print(training_X) #training_X.append(('p', 'q')) #print(training_X) training_y = ['match', 'match', 'match', 'non-match', 'non-match'] # # Extract features feature_extractor = StringPairFeatureExtractor(match=True, numeric=True) training_X_extracted = feature_extractor.fit_transform(training_X) # # # Train model # model = Hacrf(l2_regularization=1.0) # model.fit(training_X_extracted, training_y) # # # Evaluate # from sklearn.metrics import confusion_matrix # predictions = model.predict(training_X_extracted) # # print(confusion_matrix(training_y, predictions))