def __init__(self,
n_states=None,
n_features=None,
inference_method=None,
class_weight=None,
directed=True):
ChainCRF.__init__(self, n_states, n_features, inference_method,
class_weight, directed)
def test_directed_chain():
# check that a directed model actually works differntly in the two
# directions. chain of length three, three states 0, 1, 2 which want to be
# in this order, evidence only in the middle
x = np.array([[0, 0, 0], [0, 1, 0], [0, 0, 0]])
w = np.array([1, 0, 0, # unary
0, 1, 0,
0, 0, 1,
0, 1, 0, # pairwise
0, 0, 1,
0, 0, 0])
crf = ChainCRF(n_states=3, n_features=3)
y = crf.inference(x, w)
assert_array_equal([0, 1, 2], y)
def train_SSVM(X_train, y_train):
#print X_train.shape, X_train[0].shape
# splitting the 8 sub-arrays into further:
#X_train = np.concatenate([np.array_split(x, 100) for x in X_train])
#y_train = np.concatenate([np.array_split(y, 100) for y in y_train])
#X_test = np.concatenate([np.array_split(x, 30) for x in X_test])
#y_test = np.concatenate([np.array_split(y, 30) for y in y_test])
#print X_train.shape
#print X_train[0].shape
#print y_train[0].shape
#exit()
#Train using linear chain CRF
#https://groups.google.com/forum/#!topic/pystruct/KIkF7fzCyDI
model = ChainCRF()
#ssvm = NSlackSSVM(model=model, C=.1, max_iter=11) # almost similar to FrankWolfeSSVM
ssvm = FrankWolfeSSVM(model=model, C=0.001, max_iter=11)
# c=0.2 -> 62.86 % accuracy <==> c=0.1
#ssvm = OneSlackSSVM(model=model) #doesn't work as well
ssvm.fit(X_train, y_train)
print "Learning complete..."
return ssvm
def pick_best_C_value(train_sentences, sentence_labels, test_SF,
test_sentences, test_sentence_labels):
i = 0.10
best_C = i
f_old = 0
for z in range(1, 20):
print "----------------- Training on C-value %f" % i
modelCRF = ChainCRF()
ssvm = FrankWolfeSSVM(model=modelCRF, C=i, max_iter=20, random_state=5)
ssvm.fit(train_sentences, sentence_labels)
print "\n"
print "-------- Training complete --------"
predictions = ssvm.predict(test_sentences)
test_SF['predicted_labels'] = predictions
#Saving model
print "Saving model...."
pickle.dump(ssvm, open('models/ote/otemodel.sav', 'wb'))
#Evaluating Trained CRF model
p, r, f1, common, retrieved, relevant = evaluating_ote(test_SF)
if (f1 >= f_old):
#save value of 'C'
f_old = f1
best_C = i
i = i + 0.05
return best_C
def __init__(self):
self.classifierMNB = Pipeline([ #Multinomial Naive Bayes
('extract', ExtractFeatures()),
#('encoding', MultiColumnLabelEncoder()),
('clf', MultinomialNB(alpha=0.5))
])
# self.classifierMaxEnt = Pipeline([
# ('extract', ExtractFeatures()),
# #('encoding', MultiColumnLabelEncoder()),
# ('clf', nltk.maxent.MaxentClassifier.train(x, algorithm = 'gis', trace = 0, max_iter = 10))
# ])
self.classifierMaxEnt_LogReg = Pipeline([ #Maximum Entropy
('extract', ExtractFeatures()),
('clf', linear_model.LogisticRegression())
])
self.classifierCRF = Pipeline([ #CRF
('extract', ExtractFeaturesToArray()),
('clf', FrankWolfeSSVM(model=ChainCRF(),
C=2,
max_iter=10,
tol=0.01))
])
self.classifierSVM = Pipeline([ #Support Vector Machine
('extract', ExtractFeatures()), ('clf', svm.LinearSVC())
])
pass
def create_crf(self):
"""
:return:
"""
# to load nltk tagger, a time consuming, one time needed operation
self.nltk_tagger = nltk.tag._get_tagger()
self.crf = FrankWolfeSSVM(model=ChainCRF(), C=1.0, max_iter=50)
self.X, self.y, self.label_code, self.folds, generate_fold = self.load_training_data(
)
score = 0
# only need to iterate through if fold was generated
num_tries = 10 if generate_fold else 1
while (score <= 0.90) and (num_tries > 0):
try:
X_train, y_train = self.get_train_data()
self.train(X_train, y_train)
X_test, y_test = self.get_test_data()
score = self.evaluate(X_test, y_test)
except Exception as e:
current_app.logger.error('Exception: %s' % (str(e)))
current_app.logger.error(traceback.format_exc())
pass
num_tries -= 1
return (score > 0)
def chain_crf():
letters = load_letters()
x, y, folds = letters['data'], letters['labels'], letters['folds']
print "Letters : "
print letters
# print "Data : "
# print letters['data']
# print "Labels : "
# print letters['labels']
x, y = np.array(x), np.array(y)
x_train, x_test = x[folds == 1], x[folds != 1]
y_train, y_test = y[folds == 1], y[folds != 1]
print len(x_train)
print len(x_test)
print "Done"
print x_train[0].shape
print y_train[0].shape
print x_train[10].shape
print y_train[10].shape
model = ChainCRF()
ssvm = FrankWolfeSSVM(model=model, C=.1, max_iter=10)
print ssvm.fit(x_train, y_train)
print ssvm.score(x_test, y_test)
def test_initialize():
rnd = np.random.RandomState(0)
x = rnd.normal(size=(13, 5))
y = rnd.randint(3, size=13)
crf = ChainCRF(n_states=3, n_features=5)
# no-op
crf.initialize([x], [y])
#test initialization works
crf = ChainCRF()
crf.initialize([x], [y])
assert_equal(crf.n_states, 3)
assert_equal(crf.n_features, 5)
crf = ChainCRF(n_states=2)
assert_raises(ValueError, crf.initialize, X=[x], Y=[y])
pass
def scope_trainer(sentence_dicts):
scope_instances, scope_labels, sentence_splits = extract_features_scope(sentence_dicts, 'training')
scope_vec = DictVectorizer()
fvs = scope_vec.fit_transform(scope_instances).toarray()
X_train, y_train = split_data(fvs, scope_labels, sentence_splits)
scope_ssvm = FrankWolfeSSVM(model=ChainCRF(), C=0.20, max_iter=10)
scope_ssvm.fit(X_train, y_train)
return scope_ssvm, scope_vec
def CRF_pred(eeg1,
eeg2,
emg,
y,
eeg1test,
eeg2test,
emgtest,
C=0.9,
weight_shift=0,
max_iter=1000,
fs=128):
# For the ith iteration, select as trainin the sub_indices other than those at index i for train_index
eeg1_train = eeg1.values
eeg2_train = eeg2.values
emg_train = emg.values
y_train = y.values
# The test subject is the one at index i
eeg1_test = eeg1test.values
eeg2_test = eeg2test.values
emg_test = emgtest.values
# CRF Model Preprocessing
eeg1_ = process_EEG(eeg1_train)
eeg2_ = process_EEG(eeg2_train)
emg_ = process_EMG(emg_train)
xtrain_ = np.concatenate((eeg1_, eeg2_, emg_), axis=1)
ytrain_classes = np.reshape(y_train, (y_train.shape[0], ))
ytrain_ = y_train
eeg1_ = process_EEG(eeg1_test)
eeg2_ = process_EEG(eeg2_test)
emg_ = process_EMG(emg_test)
xtest_ = np.concatenate((eeg1_, eeg2_, emg_), axis=1)
xtrain_crf = np.reshape(
xtrain_,
(3, -1, xtrain_.shape[1])) # Reshape so that it works with CRF
ytrain_crf = np.reshape(ytrain_,
(3, -1)) - 1 # Reshape so that it works with CRF
# CRF Model fitting:
classes = np.unique(ytrain_)
weights_crf = compute_class_weight("balanced", list(classes),
list(ytrain_classes))
weights_crf[0] = weights_crf[0] + (2.5 * weight_shift)
weights_crf[1] = weights_crf[1] + (1.5 * weight_shift)
model = ChainCRF(class_weight=weights_crf)
ssvm = OneSlackSSVM(model=model, C=C, max_iter=max_iter)
ssvm.fit(xtrain_crf, ytrain_crf)
# Test on the third guy
xtest_crf = np.reshape(xtest_, (2, -1, xtest_.shape[1]))
y_pred_crf = ssvm.predict(xtest_crf)
y_pred_crf = np.asarray(y_pred_crf).reshape(-1) + 1
return y_pred_crf
def losocv_CRF_prepro(xtrain, y, C=0.5, weight_shift=0, max_iter=1000, fs=128):
"""Leave one subject out cross validation for the CRF model becasuse it requires
special datahandling. Input should be a Pandas Dataframe."""
epochs = 21600
num_sub = 3
# Indices of the subjects
sub_indices = [
np.arange(0, epochs),
np.arange(epochs, epochs * 2),
np.arange(epochs * 2, epochs * 3)
]
res = []
for i in range(len(sub_indices)):
# For the ith iteration, select as trainin the sub_indices other than those at index i for train_index
train_index = np.concatenate(
[sub_indices[(i + 1) % num_sub], sub_indices[(i + 2) % num_sub]])
xtrain_ = xtrain[train_index]
y_train = y.values[train_index]
ytrain_ = y_train
# The test subject is the one at index i
test_index = sub_indices[i]
xtest_ = xtrain[test_index]
y_test = y.values[test_index]
ytest_ = y_test
# CRF Model Preprocessing
ytrain_classes = np.reshape(y_train, (y_train.shape[0], ))
xtrain_crf = np.reshape(
xtrain_,
(2, -1, xtrain_.shape[1])) # Reshape so that it works with CRF
ytrain_crf = np.reshape(
ytrain_, (2, -1)) - 1 # Reshape so that it works with CRF
# CRF Model fitting:
classes = np.unique(ytrain_)
weights_crf = compute_class_weight("balanced", list(classes),
list(ytrain_classes))
weights_crf[0] = weights_crf[0] + (2.5 * weight_shift)
weights_crf[1] = weights_crf[1] + (1.5 * weight_shift)
model = ChainCRF(class_weight=weights_crf)
ssvm = OneSlackSSVM(model=model, C=C, max_iter=max_iter)
ssvm.fit(xtrain_crf, ytrain_crf)
# Test on the third guy
xtest_crf = np.reshape(xtest_, (1, -1, xtest_.shape[1]))
ytest_crf = np.reshape(ytest_, (1, -1)) - 1
y_pred_crf = ssvm.predict(xtest_crf)
y_pred_crf = np.asarray(y_pred_crf).reshape(-1) + 1
resy = sklearn.metrics.balanced_accuracy_score(ytest_, y_pred_crf)
print("Iteration, result:", i, resy)
res.append(resy)
return res
def __init__(self):
# the model
self.model = ChainCRF(directed=True)
# the learner
self.learner = OneSlackSSVM(model=self.model,
C=.1,
inference_cache=50,
tol=0.1,
n_jobs=1)
def __init__(self, c_value, classifier_name='ChainCRF'):
self.c_value = c_value
self.classifier_name = classifier_name
if self.classifier_name == 'ChainCRF':
model = ChainCRF()
self.clf = FrankWolfeSSVM(model=model, C=self.c_value, max_iter=50)
else:
raise TypeError('Invalid classifier type')
def main():
parser = argparse.ArgumentParser(
description="learn to segment and tokenize (really, any labeling)",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--untokfile",
"-u",
nargs='?',
type=argparse.FileType('r'),
default=sys.stdin,
help="untok file")
parser.add_argument(
"--biofile",
"-b",
nargs='?',
type=argparse.FileType('r'),
default=sys.stdin,
help="bio file. must match untok file and be space separated")
parser.add_argument("--outfile",
"-o",
nargs='?',
type=argparse.FileType('wb'),
default=None,
help="output file")
parser.add_argument("--debug",
"-d",
action='store_true',
default=False,
help="debug mode")
try:
args = parser.parse_args()
except IOError as msg:
parser.error(str(msg))
untokfile = prepfile(args.untokfile, 'r')
biofile = prepfile(args.biofile, 'r')
data, labels, datamap, labelmap = prepdata(untokfile, biofile, args.debug)
# print(data)
# print(labels)
model = ChainCRF()
#ssvm = SubgradientSSVM(model=model, C=.1)#, show_loss_every=5)
ssvm = FrankWolfeSSVM(model=model, max_iter=100,
C=.1) #, show_loss_every=5)
ssvm.fit(data, labels)
# curve = ssvm.loss_curve_
# TONT
# print("TONT score with chain CRF: %f" % ssvm.score(data, labels))
ret = {}
ret['model'] = ssvm
ret['feats'] = datamap
ret['labels'] = labelmap
if args.outfile is not None:
pickle.dump(ret, args.outfile)
def train_scope_learner(sentence_dicts, C_value):
scope_sentence_dicts, scope_instances, scope_labels, sentence_splits = extract_features_scope(
sentence_dicts, 'training')
vectorizer = DictVectorizer()
fvs = vectorizer.fit_transform(scope_instances).toarray()
X_train, y_train = make_splits(fvs, scope_labels, sentence_splits)
model = ChainCRF()
scope_ssvm = FrankWolfeSSVM(model=model, C=C_value, max_iter=10)
scope_ssvm.fit(X_train, y_train)
return scope_ssvm, vectorizer
def fresh_train(self, x, y, iterations=10):
self.model = ChainCRF(inference_method="max-product")
self.learner = SubgradientSSVM(
model=self.model,
max_iter=iterations,
logger=SaveLogger(
MODEL_PATH_TEMPLATE.format(self.userId + "-learner")),
show_loss_every=50)
self.learner.fit(x, y, warm_start=False)
self.save()
def train_seq(X, y, crf_params):
X_ = [X[k] for k in sorted(X.keys())]
y_ = [y[k] for k in sorted(y.keys())]
class_sizes = np.bincount(np.hstack(y_))
cw = 1. / class_sizes
cw = cw / cw.sum()
return OneSlackSSVM(model=ChainCRF(inference_method='max-product',
class_weight=cw),
max_iter=100000,
verbose=False,
**crf_params).fit(X_, y_)
def __init__(self, c_value, classifier_name='ChainCRF'):
self.c_value = c_value
self.classifier_name = classifier_name
#using chain crf to analyze the data, so add an error check for this:
if self.classifier_name == 'ChainCRF':
model = ChainCRF()
#define the classifier to use with CRF model.
self.clf = FrankWolfeSSVM(model=model,
C=self.c_value,
max_iter=100)
else:
raise TypeError('Invalid classifier type')
def cross_val(self, X_train, y_train):
'''
method to conduct 5-fold cross validation
'''
kf = KFold(len(X_train), n_folds=5, random_state=None, shuffle=False)
for train_idx, test_idx in kf:
xtrain, xval = X_train[train_idx], X_train[test_idx]
ytrain, yval = y_train[train_idx], y_train[test_idx]
model = ChainCRF()
ssvm = FrankWolfeSSVM(model=model, C=0.5, max_iter=15)
ssvm.fit(xtrain, ytrain)
print ssvm.score(xval, yval)
def createModel(data, labels, num_classes=2):
model = ChainCRF(n_states=num_classes,
n_features=int(len(columns) - 5),
directed=True)
clf = StructuredPerceptron(model=model,
max_iter=200,
verbose=False,
batch=False,
average=True)
print("Structured Perceptron + Chain CRF")
train_start = time()
clf.fit(X=data, Y=labels)
train_end = time()
print("Training took " + str((train_end - train_start) / 60) +
" minutes to complete\n")
return clf
def train_one_C(X_train, y_train, X_val, y_val, Cs):
crf = ChainCRF(n_states=10, inference_method="max-product", directed=True)
best_model = None
best_C = None
smallest_error = None
for C in Cs:
print("C =", C, ", training...")
ssvm = OneSlackSSVM(crf, max_iter=200, C=C)
ssvm.fit(X_train, y_train)
error = 1 - ssvm.score(X_val, y_val) # Note: score 1 - error
if not smallest_error or error < smallest_error:
best_model = ssvm
best_C = C
smallest_error = error
print("Completed.")
return best_model, best_C, smallest_error
def MLfitCRF(data_train, data_test, records, folds):
fvector = np.array([data_train[0]])
labels = np.array([data_train[1]])
#create CRF model
CRFmodel = ChainCRF()
#create ML classifier
ssvm = FrankWolfeSSVM(model = CRFmodel, C = 0.1)
#training
ssvm.fit(fvector, labels)
#model testing
fvector_test = np.array(data_test[0])
labels_test = np.array(data_test[1])
score = ssvm.score(fvector_train, labels_test)
print score
return
def chaincrf_test():
num_pics = 3000
X, Y = load_pictures(num_pics)
X = np.array(X)
Y = np.array(Y)
print X.shape
print Y.shape
# 0: pixel, 1: row, 2: picture
mode = 0
outstr = "Test score with data arranged by "
if mode == 0:
X, Y = arrange_by_pixel(X, Y)
outstr += "pixel:"
elif mode == 1:
X, Y = arrange_by_row(X, Y)
outstr += "row:"
elif mode == 2:
X, Y = arrange_by_picture(X, Y)
outstr += "picture:"
print X.shape
print Y.shape
#print X.shape, Y.shape
train_pct = 0.66
test_pct = 1 - train_pct
X_train = X[0:math.floor(train_pct * num_pics)]
X_test = X[math.floor(test_pct * num_pics):]
Y_train = Y[0:math.floor(train_pct * num_pics)]
Y_test = Y[math.floor(test_pct * num_pics):]
model = ChainCRF()
ssvm = FrankWolfeSSVM(model=model, C=.1, max_iter=10)
# #print X_train.shape, Y_train.shape
ssvm.fit(X_train, Y_train)
results = ssvm.score(X_test, Y_test)
print outstr
print results
def Chain_CRF(x, y, x_test, model_args):
# Reshape for CRF
#svc = SVC(class_weight='balanced', kernel='rbf', decision_function_shape='ovr')
#svc.fit(x, y)
#x = svc.decision_function(x)
#x_test = svc.decision_function(x_test)
#scaler = StandardScaler().fit(x)
#x = scaler.transform(x)
#x_test = scaler.transform(x_test)
x = x[:, :11]
x_test = x_test[:, :11]
x = x.reshape(-1, 21600, x.shape[-1])
x_test = x_test.reshape(-1, 21600, x.shape[-1])
y = y.reshape(-1, 21600)
crf = ChainCRF(directed=False)
ssvm = FrankWolfeSSVM(model=crf,
C=model_args['C'],
max_iter=model_args['max_iter'])
ssvm.fit(x, y)
y_pred = np.array(ssvm.predict(x_test))
return y_pred.flatten()
def __init__(self, do_train=False, trained_model_name="passage_crf_model", algorithm="crf"):
self.trained_model_name = trained_model_name
self.fp = FeatureProcessing()
self.do_train = do_train
self.algorithm = algorithm
if algorithm == "crf":
if do_train:
self.trainer = Trainer()
else:
self.tagger = Tagger()
else:
if do_train:
model = ChainCRF()
self.trainer = FrankWolfeSSVM(model=model)
self.feat_index = {}
self.label_index = {}
else:
self.tagger = pickle.load(open(self.trained_model_name, "rb"))
self.feat_index = pickle.load(open("ssvm_feat_index.pkl", "rb"))
label_index = pickle.load(open("ssvm_label_index.pkl", "rb"))
self.rev_label_index = {i: x for x, i in label_index.items()}
def build_models(X_train, y_train):
'''
PURPOSE: ouput model objects which have been fitted with training data
INPUT: X_train (np.array) - features matrix
y_train (np.array) - label matrix
OUTPUT: nmb (MultinomialNB obj) - model trained on X_train, y_train
svm (LinearSVC obj) - model trained on X_train, y_train
ssvm (PyStruct chainCRF object) - trained Chain CRF model
'''
# Multinomial Naive Bayes Classifier:
nmb = MultinomialNB()
nmb.fit(np.vstack(X_train), np.hstack(y_train))
# Support Vector Machine Classifier
svm = LinearSVC(dual=False, C=.1)
svm.fit(np.vstack(X_train), np.hstack(y_train))
# Chain Conditional Random Field Classifier
model = ChainCRF()
ssvm = FrankWolfeSSVM(model=model, C=0.5, max_iter=15)
ssvm.fit(X_train, y_train)
return nmb, svm, ssvm
def CRF_pred_prepro(xtrain,
y,
xtest,
C=0.9,
weight_shift=0,
max_iter=1000,
fs=128):
y_train = y.values
# CRF Model Preprocessing
xtrain_ = xtrain
ytrain_classes = np.reshape(y_train, (y_train.shape[0], ))
ytrain_ = y_train
xtest_ = xtest
xtrain_crf = np.reshape(
xtrain_,
(3, -1, xtrain_.shape[1])) # Reshape so that it works with CRF
ytrain_crf = np.reshape(ytrain_,
(3, -1)) - 1 # Reshape so that it works with CRF
# CRF Model fitting:
classes = np.unique(ytrain_)
weights_crf = compute_class_weight("balanced", list(classes),
list(ytrain_classes))
weights_crf[0] = weights_crf[0] + (2.5 * weight_shift)
weights_crf[1] = weights_crf[1] + (1.5 * weight_shift)
model = ChainCRF(class_weight=weights_crf)
ssvm = OneSlackSSVM(model=model, C=C, max_iter=max_iter)
ssvm.fit(xtrain_crf, ytrain_crf)
# Test on the third guy
xtest_crf = np.reshape(xtest_, (2, -1, xtest_.shape[1]))
y_pred_crf = ssvm.predict(xtest_crf)
y_pred_crf = np.asarray(y_pred_crf).reshape(-1) + 1
return y_pred_crf
letters = load_letters()
X, y, folds = letters['data'], letters['labels'], letters['folds']
# we convert the lists to object arrays, as that makes slicing much more
# convenient
X, y = np.array(X), np.array(y)
X_train, X_test = X[folds == 1], X[folds != 1]
y_train, y_test = y[folds == 1], y[folds != 1]
# Train linear SVM
svm = LinearSVC(dual=False, C=.1)
# flatten input
svm.fit(np.vstack(X_train), np.hstack(y_train))
# Train linear chain CRF
model = ChainCRF()
ssvm = FrankWolfeSSVM(model=model, C=.1, max_iter=11)
ssvm.fit(X_train, y_train)
print("Test score with chain CRF: %f" % ssvm.score(X_test, y_test))
print("Test score with linear SVM: %f" %
svm.score(np.vstack(X_test), np.hstack(y_test)))
# plot some word sequenced
n_words = 4
rnd = np.random.RandomState(1)
selected = rnd.randint(len(y_test), size=n_words)
max_word_len = max([len(y_) for y_ in y_test[selected]])
fig, axes = plt.subplots(n_words, max_word_len, figsize=(10, 10))
fig.subplots_adjust(wspace=0)
def train_all_C(x_Train, y_Train):
model_tuples = []
#let's define the range of Cs
l1 = [10**i for i in range(-4, 3, 1)]
l1.extend([5 * l for l in l1])
Cs = sorted(l1)
model, C, train_error = train_one_C(x_Train[:4500], y_Train[:4500],
x_Train[-500:], y_Train[-500:], Cs)
model_tuples.append((model, C, train_error))
return model_tuples
X_train, Y_train, TrainSent = ReadData("train")
X_test, Y_test, TrainSent = ReadData("test")
model, best_C, train_error = train_all_C(X_train, Y_train)[0]
test_error = 1 - model.score(X_test, Y_test)
print("Best C: ", best_C, ", training error: ", train_error,
", validation error: ", test_error)
list = [best_C, train_error]
crf = ChainCRF(n_states=10, inference_method="max-product", directed=True)
ssvm = OneSlackSSVM(crf, max_iter=200, C=best_C)
ssvm.fit(X_train[:4500], Y_train[:4500])
list.append(1. - ssvm.score(X_test, Y_test))
print(list)
ssvm.fit(X_train, Y_train)
error = 1. - ssvm.score(X_test, Y_test)
print('test error: ', error)
def __init__(self, c_val=1.0):
self.clf = FrankWolfeSSVM(model=ChainCRF(), C=c_val, max_iter=50)
def __init__(self, n_states=2, n_features=None, inference_method='qpbo'):
ChainCRF.__init__(self, n_states, n_features,
inference_method=inference_method)
self.n_edge_types = 4
self.size_psi = (n_states * self.n_features
+ self.n_edge_types * n_states ** 2)
