def test_fit_predict_regularized(self):
incorrect = [
'helloooo', 'freshh', 'ffb', 'h0me', 'wonderin', 'relaionship',
'hubby', 'krazii', 'mite', 'tropic'
]
correct = [
'hello', 'fresh', 'facebook', 'home', 'wondering', 'relationship',
'husband', 'crazy', 'might', 'topic'
]
training = zip(incorrect, correct)
fe = StringPairFeatureExtractor(match=True, numeric=True)
xf = fe.fit_transform(training)
model = Hacrf(l2_regularization=10.0)
model.fit(xf, [0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
print(model.parameters)
expected_parameters = np.array([[-0.0569188, 0.07413339, 0.],
[0.00187709, -0.06377866, 0.],
[-0.01908823, 0.00586189, 0.],
[0.01721114, -0.00636556, 0.],
[0.01578279, 0.0078614, 0.],
[-0.0139057, -0.00862948, 0.],
[-0.00623241, 0.02937325, 0.],
[0.00810951, -0.01774676, 0.]])
assert_array_almost_equal(model.parameters,
expected_parameters,
decimal=TEST_PRECISION)
expected_probas = np.array([[0.5227226, 0.4772774],
[0.52568993, 0.47431007],
[0.4547091, 0.5452909],
[0.51179222, 0.48820778],
[0.46347576, 0.53652424],
[0.45710098, 0.54289902],
[0.46159657, 0.53840343],
[0.42997978, 0.57002022],
[0.47419724, 0.52580276],
[0.50797852, 0.49202148]])
actual_predict_probas = model.predict_proba(xf)
print(actual_predict_probas)
assert_array_almost_equal(actual_predict_probas,
expected_probas,
decimal=TEST_PRECISION)
expected_predictions = np.array([0, 0, 1, 0, 1, 1, 1, 1, 1, 0])
actual_predictions = model.predict(xf)
assert_array_almost_equal(actual_predictions,
expected_predictions,
decimal=TEST_PRECISION)
def test_fit_predict(self):
incorrect = [
'helloooo', 'freshh', 'ffb', 'h0me', 'wonderin', 'relaionship',
'hubby', 'krazii', 'mite', 'tropic'
]
correct = [
'hello', 'fresh', 'facebook', 'home', 'wondering', 'relationship',
'husband', 'crazy', 'might', 'topic'
]
training = zip(incorrect, correct)
fe = StringPairFeatureExtractor(match=True, numeric=True)
xf = fe.fit_transform(training)
model = Hacrf()
model.fit(xf, [0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
expected_parameters = np.array([[-10.76945326, 144.03414923, 0.],
[31.84369748, -106.41885651, 0.],
[-52.08919467, 4.56943665, 0.],
[31.01495044, -13.0593297, 0.],
[49.77302218, -6.42566204, 0.],
[-28.69877796, 24.47127009, 0.],
[-85.34524911, 21.87370646, 0.],
[106.41949333, 6.18587125, 0.]])
print(model.parameters)
assert_array_almost_equal(model.parameters,
expected_parameters,
decimal=TEST_PRECISION)
expected_probas = np.array([[1.00000000e+000, 3.51235685e-039],
[1.00000000e+000, 4.79716208e-039],
[1.00000000e+000, 2.82744641e-139],
[1.00000000e+000, 6.49580729e-012],
[9.99933798e-001, 6.62022561e-005],
[8.78935957e-005, 9.99912106e-001],
[4.84538335e-009, 9.99999995e-001],
[1.25170233e-250, 1.00000000e+000],
[2.46673086e-010, 1.00000000e+000],
[1.03521293e-033, 1.00000000e+000]])
actual_predict_probas = model.predict_proba(xf)
print(actual_predict_probas)
assert_array_almost_equal(actual_predict_probas,
expected_probas,
decimal=TEST_PRECISION)
expected_predictions = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
actual_predictions = model.predict(xf)
assert_array_almost_equal(actual_predictions,
expected_predictions,
decimal=TEST_PRECISION)
def test_fit_predict_regularized_viterbi(self):
incorrect = [
'helloooo', 'freshh', 'ffb', 'h0me', 'wonderin', 'relaionship',
'hubby', 'krazii', 'mite', 'tropic'
]
correct = [
'hello', 'fresh', 'facebook', 'home', 'wondering', 'relationship',
'husband', 'crazy', 'might', 'topic'
]
training = zip(incorrect, correct)
fe = StringPairFeatureExtractor(match=True, numeric=True)
xf = fe.fit_transform(training)
model = Hacrf(l2_regularization=10.0, viterbi=True)
model.fit(xf, [0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
print(model.parameters)
expected_parameters = np.array([[-0.0569188, 0.07413339, 0.],
[0.00187709, -0.06377866, 0.],
[-0.01908823, 0.00586189, 0.],
[0.01721114, -0.00636556, 0.],
[0.01578279, 0.0078614, 0.],
[-0.0139057, -0.00862948, 0.],
[-0.00623241, 0.02937325, 0.],
[0.00810951, -0.01774676, 0.]])
assert_array_almost_equal(model.parameters,
expected_parameters,
decimal=TEST_PRECISION)
expected_probas = np.array([[0.56394611, 0.43605389],
[0.52977205, 0.47022795],
[0.4751729, 0.5248271],
[0.51183761, 0.48816239],
[0.48608081, 0.51391919],
[0.4986367, 0.5013633],
[0.46947222, 0.53052778],
[0.43233544, 0.56766456],
[0.47463002, 0.52536998],
[0.51265109, 0.48734891]])
actual_predict_probas = model.predict_proba(xf)
print(actual_predict_probas)
assert_array_almost_equal(actual_predict_probas,
expected_probas,
decimal=TEST_PRECISION)
expected_predictions = np.array([0, 0, 1, 0, 1, 1, 1, 1, 1, 0])
actual_predictions = model.predict(xf)
assert_array_almost_equal(actual_predictions,
expected_predictions,
decimal=TEST_PRECISION)
def train(self):
# Training
self.fe = StringPairFeatureExtractor(match=True,
numeric=True,
transition=True)
if self.needTraining:
lines = open(self.infile, 'r').readlines()
# Generate Positive Correction Pair
ppairs = []
ppairs = [
line.split('\t')[1].strip().split(' | ') for line in lines
]
ppairs = [(pair[0], pair[i]) for pair in ppairs
for i in xrange(1, len(pair))]
# Generate Positive Training Correction Pairs and Testing Correction Pairs
ppairs_train, ppairs_test = train_test_split(ppairs,
test_size=200,
random_state=1)
self.ppairs_train = [
tuple(ppair_train) for ppair_train in ppairs_train
]
self.ppairs_test = [
tuple(ppair_test) for ppair_test in ppairs_test
]
# Generate Negative Training Correction Pairs
incorrect = list(zip(*ppairs_train)[0])
shuffle(incorrect)
correct = list(zip(*ppairs_train)[1])
npairs_train = zip(incorrect, correct)
# Raw training set
x_raw = ppairs_train + npairs_train
# Label of the training set
self.y_train = [0] * len(ppairs_train) + [1] * len(npairs_train)
# Extract Features from the raw training set
self.x_train = x_orig = self.fe.fit_transform(x_raw)
#x_train, x_test, y_train, y_test = train_test_split(x_orig, y_orig, test_size=0.2, random_state=42)
self.m = Hacrf(l2_regularization=10.0,
optimizer=fmin_l_bfgs_b,
optimizer_kwargs={'maxfun': 45},
state_machine=None)
self.m.fit(self.x_train, self.y_train, verbosity=20)
cPickle.dump(self.m, open('Corrector.pkl', 'wb'))
else:
print "start training"
self.m = cPickle.load(open('Corrector.pkl', 'rb'))
print "finish training"
def __init__(self):
classes = ['match', 'non-match']
self.model = Hacrf(l2_regularization=100.0,
state_machine=DefaultStateMachine(classes))
self.model.parameters = np.array(
[[-0.22937526, 0.51326066], [0.01038001, -0.13348901],
[-0.03062821, 0.13769178], [0.02024813, -0.01835538],
[0.09208272, 0.15466022], [-0.08170265, -0.02484392],
[-0.01762858, 0.17504624], [0.02800866, -0.04442708]],
order='F')
self.parameters = self.model.parameters.T
self.model.classes = ['match', 'non-match']
self.feature_extractor = StringPairFeatureExtractor(match=True,
numeric=False)
def test_transform_transition(self):
s1 = "ba"
s2 = "ca"
# a . .
# b . .
# c a
chars = StringPairFeatureExtractor.CHARACTERS
nchars = len(chars)
print(nchars)
expected_x = np.zeros((2, 2, len(chars)**2 + 1))
expected_x[:, :, 0] = 1.0
expected_x[0, 0, 2 + nchars * 1 + 1] = 1.0 # b->c
expected_x[0, 1, 0 + nchars * 1 + 1] = 1.0 # b->a
expected_x[1, 0, 2 + nchars * 0 + 1] = 1.0 # a->c
expected_x[1, 1, 0 + nchars * 0 + 1] = 1.0 # a->a
test_extractor = StringPairFeatureExtractor(transition=True)
actual_X = test_extractor.fit_transform([(s1, s2)])
assert_array_almost_equal(expected_x, actual_X[0])
def test_transform_binary(self):
s1 = "kat1"
s2 = "cat2"
# 1 . . . n
# t . . m .
# a . m . .
# k . . . .
# c a t 2
expected_x = np.zeros((4, 4, 4))
expected_x[:, :, 0] = 2.0
expected_x[:, 0, 1] = 1.0
expected_x[0, :, 1] = 1.0
expected_x[1, 1, 2] = 1.0
expected_x[2, 2, 2] = 1.0
expected_x[3, 3, 3] = 1.0
test_extractor = StringPairFeatureExtractor(bias=2.0,
start=True,
match=True,
numeric=True)
actual_X = test_extractor.fit_transform([(s1, s2)])
assert_array_almost_equal(expected_x, actual_X[0])
import numpy, pyhacrf, nltk
from pyhacrf import StringPairFeatureExtractor, Hacrf
training_X = []
iv = open("iv.txt", "r").read()
ovv = open("ovv.txt", "r").read()
for p in iv.split("\n"):
for q in ovv.split("\n"):
training_X.append((p, q))
#print(training_X)
#training_X.append(('p', 'q'))
#print(training_X)
training_y = ['match', 'match', 'match', 'non-match', 'non-match']
# # Extract features
feature_extractor = StringPairFeatureExtractor(match=True, numeric=True)
training_X_extracted = feature_extractor.fit_transform(training_X)
#
# # Train model
# model = Hacrf(l2_regularization=1.0)
# model.fit(training_X_extracted, training_y)
#
# # Evaluate
# from sklearn.metrics import confusion_matrix
# predictions = model.predict(training_X_extracted)
#
# print(confusion_matrix(training_y, predictions))
