def train(self, trainfile_name):
print >>sys.stderr, "Reading data.."
train_data = [tuple(x.strip().split("\t")) for x in codecs.open(trainfile_name, "r", "utf-8")]
shuffle(train_data)
filter_feature = get_filter()
train_labels, train_clauses = zip(*train_data)
train_labels = [tl.lower() for tl in train_labels]
print >>sys.stderr, "Indexing features.."
self.fp.index_data(train_clauses, filter_feature)
X = numpy.asarray([self.fp.featurize(clause, filter_feature) for clause in train_clauses])
tagset = list(set(train_labels))
tag_index = {l:i for (i, l) in enumerate(tagset)}
Y = numpy.asarray([[tag_index[label]] for label in train_labels])
classifier = OneVsRestClassifier(SVC(kernel='linear'))
if self.cv:
print >>sys.stderr, "Starting Cross-validation for %d folds.."%(self.folds)
y = [l[0] for l in Y]
scores = cross_validation.cross_val_score(classifier, X, y, cv=self.folds, scoring='f1_weighted')
print >>sys.stderr, "Scores:", scores
print >>sys.stderr, "Average: %0.4f (+/- %0.4f)"%(scores.mean(), scores.std() * 2)
print >>sys.stderr, "Starting training.."
classifier.fit(X, Y)
pickle.dump(classifier, open(self.trained_model_name, "wb"))
pickle.dump(self.fp.feat_index, open(self.feat_index_name, "wb"))
pickle.dump(tagset, open(self.stored_tagset, "wb"))
print >>sys.stderr, "Done"
def predict(self, testfile_name):
test_data = [x.strip() for x in codecs.open(testfile_name, "r", "utf-8")]
filter_feature = get_filter()
if len(test_data) == 0:
return []
X = numpy.asarray([self.fp.featurize(clause, filter_feature) for clause in test_data])
predictions = [self.tagset[ind] for ind in self.classifier.predict(X)]
return zip(predictions, test_data)
