def draw_roc_curve_saved_model(self):
self.logger.println("drawing roc curve from saved model")
start_time = timeit.default_timer()
cs = CrfSuite()
crf = cs.load_model("current_crf_model.pkl")
dataset = Dataset()
data = dataset.read(nr_of_files=1000)
nr_of_filled_lines, data1 = dataset.filter_for_filled_tags(data)
data2 = dataset.obtain_default_tags(nr_of_filled_lines * 3, data)
data = data1 + data2
data = dataset.shuffle_data(data)
train_set, test_set = dataset.split_dataset(data)
we_model = WeModel()
w2v_model = we_model.read()
we_model = None
word2count, word2idx = dataset.encode_dataset(train_set)
f_generator = FeatureGenerator(w2v_model, word2count, word2idx)
w2v_model = None
train_features = f_generator.generate_features_docs(train_set)
y_train = f_generator.generate_true_outcome(train_set)
test_features = f_generator.generate_features_docs(test_set)
y_test = f_generator.generate_true_outcome(test_set)
f_generator = None
evaluator = Evaluator()
evaluator.draw_roc_proba(crf, test_features, y_test)
def train_model(self, nr_of_files=-1):
self.logger.println("train model called")
start_time = timeit.default_timer()
cs = CrfSuite()
dataset = Dataset()
data = dataset.read(nr_of_files=nr_of_files)
nr_of_filled_lines, data1 = dataset.filter_for_filled_tags(data)
data2 = dataset.obtain_default_tags(nr_of_filled_lines * 3, data)
data = data1 + data2
data = dataset.shuffle_data(data)
train_set, test_set = dataset.split_dataset(data)
we_model = WeModel()
w2v_model = we_model.train(
data) # optionally load a pretrained model here
we_model.save(w2v_model)
we_model = None
word2count, word2idx = dataset.encode_dataset(train_set)
f_generator = FeatureGenerator(w2v_model, word2count, word2idx)
w2v_model = None
train_features = f_generator.generate_features_docs(train_set)
y_train = f_generator.generate_true_outcome(train_set)
test_features = f_generator.generate_features_docs(test_set)
y_test = f_generator.generate_true_outcome(test_set)
f_generator = None
model = cs.train_model(train_features, y_train)
cs.save_model(model)
y_train_pred = cs.test_model(model, train_features)
y_test_pred = cs.test_model(model, test_features)
print("printing training results")
cs.print_classification_report(dataset.docs2lines(y_train),
y_train_pred)
score_train = cs.score_model(dataset.docs2lines(y_train), y_train_pred)
print("training f1 score: %s" % score_train)
print("printing test results")
cs.print_classification_report(dataset.docs2lines(y_test), y_test_pred)
score_test = cs.score_model(dataset.docs2lines(y_test), y_test_pred)
print("test f1 score: %s" % score_test)
elapsed_seconds = timeit.default_timer() - start_time
self.logger.print_time_taken("train model operation took",
elapsed_seconds)
evaluator = Evaluator()
evaluator.perform_roc_analysis(dataset.docs2lines(y_train),
y_train_pred)
evaluator.perform_roc_analysis(dataset.docs2lines(y_test), y_test_pred)
def optimise_model(self, argv):
self.logger.println("optimise model called")
start_time = timeit.default_timer()
cs = CrfSuite()
dataset = Dataset()
data = dataset.read(nr_of_files=argv)
we_model = WeModel()
w2v_model = we_model.train(
data) # optionally load a pretrained model here
we_model.save(w2v_model)
word2count, word2idx = dataset.encode_dataset(data)
f_generator = FeatureGenerator(w2v_model, word2count, word2idx)
train_features = f_generator.generate_features_docs(data)
y_train = f_generator.generate_true_outcome(data)
cs.optimise_model(train_features, y_train)
elapsed_seconds = timeit.default_timer() - start_time
self.logger.print_time_taken("optimise model operation took",
elapsed_seconds)
def train_model_learning_curve(self, arg):
self.logger.println("train model called")
start_time = timeit.default_timer()
cs = CrfSuite()
dataset = Dataset()
data = dataset.read(nr_of_files=arg)
nr_of_filled_lines, data1 = dataset.filter_for_filled_tags(data)
data2 = dataset.obtain_default_tags(nr_of_filled_lines * 3, data)
data = data1 + data2
data = dataset.shuffle_data(data)
train_set, test_set = dataset.split_dataset(data)
we_model = WeModel()
w2v_model = we_model.train(
data) # optionally load a pretrained model here
#w2v_model = we_model.load_pretrained_model() # optionally load a pretrained model here
word2count, word2idx = dataset.encode_dataset(train_set)
f_generator = FeatureGenerator(w2v_model, word2count, word2idx)
train_features = f_generator.generate_features_docs(train_set)
y_train = f_generator.generate_true_outcome(train_set)
cs.plot_learning_curve(train_features, y_train)
plt.show()
elapsed_seconds = timeit.default_timer() - start_time
self.logger.print_time_taken("train model operation took",
elapsed_seconds)
def get_feature_generator_results(self):
input = [[("this", "", "", ""), ("is", "", "", ""), ("a", "", "", ""),
("test", "", "", "")],
[("this", "", "", ""), ("is", "", "", ""), ("a", "", "", ""),
("test", "", "", "")]]
dataset = Dataset()
word2count, word2idx = dataset.encode_dataset([input])
f_generator = FeatureGenerator(self.w2v_model, word2count, word2idx)
X = f_generator.generate_features_docs([input])
y = f_generator.generate_true_outcome([input])
# run tests on X and y
return input, X, y
def perform_bootstrapping(self, dataset, sample_size, iterations):
"""
bootstraps a sample n times. Averages the precision, recall, f1, tpr and
fpr for each of the entities. Prints results of precision, recall and
f1. Plots roc curves for tpr and fpr of each entity.
"""
training_scores = []
test_scores = []
emp_pos_scores = np.empty(shape=(0, 3), dtype='float64')
emp_comp_scores = np.empty(shape=(0, 3), dtype='float64')
edu_major_scores = np.empty(shape=(0, 3), dtype='float64')
edu_inst_scores = np.empty(shape=(0, 3), dtype='float64')
mean_fpr = np.linspace(0, 1, 100)
lb = LabelBinarizer()
emp_pos_tpr = np.empty(shape=(0, 3), dtype='float64')
emp_pos_fpr = np.empty(shape=(0, 3), dtype='float64')
emp_comp_tpr = np.empty(shape=(0, 3), dtype='float64')
emp_comp_fpr = np.empty(shape=(0, 3), dtype='float64')
edu_major_tpr = np.empty(shape=(0, 3), dtype='float64')
edu_major_fpr = np.empty(shape=(0, 3), dtype='float64')
edu_inst_tpr = np.empty(shape=(0, 3), dtype='float64')
edu_inst_fpr = np.empty(shape=(0, 3), dtype='float64')
for x in range(0, iterations):
print("iteration nr %s" % x)
sampled_train_set, oob_test_set = self.resample_data(
dataset, sample_size, return_leftovers=True)
cs = CrfSuite()
ds = Dataset()
we_model = WeModel()
w2v_model = we_model.train(
dataset) # optionally load a pretrained model here
word2count, word2idx = ds.encode_dataset(sampled_train_set)
f_generator = FeatureGenerator(w2v_model, word2count, word2idx)
train_features = f_generator.generate_features_docs(
sampled_train_set)
y_train = f_generator.generate_true_outcome(sampled_train_set)
test_features = f_generator.generate_features_docs(oob_test_set)
y_test = f_generator.generate_true_outcome(oob_test_set)
trainer = cs.train_model(train_features, y_train)
y_train_pred = cs.test_model(trainer, train_features)
y_test_pred = cs.test_model(trainer, test_features)
score_train = cs.score_model(ds.docs2lines(y_train), y_train_pred)
score_test = cs.score_model(ds.docs2lines(y_test), y_test_pred)
y_true_combined = lb.fit_transform(
list(chain.from_iterable(ds.docs2lines(y_test))))
y_pred_combined = lb.transform(
list(chain.from_iterable(y_test_pred)))
class_indices = {cls: idx for idx, cls in enumerate(lb.classes_)}
"""
# fpr and tpr for one class
temp_fpr, temp_tpr, _ = roc_curve(y_true_combined[:, class_indices["B-EMP-POS"]], y_pred_combined[:, class_indices["B-EMP-POS"]], pos_label=1)
temp_fpr1, temp_tpr1, _ = roc_curve(y_true_combined[:, class_indices["I-EMP-POS"]], y_pred_combined[:, class_indices["I-EMP-POS"]], pos_label=1)
temp_fpr = np.vstack([temp_fpr, temp_fpr1])
temp_tpr = np.vstack([temp_tpr, temp_tpr1])
emp_pos_tpr = np.vstack([emp_pos_tpr, temp_tpr.mean(axis=0)])
emp_pos_fpr = np.vstack([emp_pos_fpr, temp_fpr.mean(axis=0)])
temp_fpr = temp_tpr = temp_fpr1 = temp_tpr1 = np.empty(shape=(0,3),dtype='float64')
temp_fpr, temp_tpr, _ = roc_curve(y_true_combined[:, class_indices["B-EMP-COMP"]], y_pred_combined[:, class_indices["B-EMP-COMP"]], pos_label=1)
temp_fpr1, temp_tpr1, _ = roc_curve(y_true_combined[:, class_indices["I-EMP-COMP"]], y_pred_combined[:, class_indices["I-EMP-COMP"]], pos_label=1)
temp_fpr = np.vstack([temp_fpr, temp_fpr1])
temp_tpr = np.vstack([temp_tpr, temp_tpr1])
emp_comp_tpr = np.vstack([emp_comp_tpr, temp_tpr.mean(axis=0)])
emp_comp_fpr = np.vstack([emp_comp_fpr, temp_fpr.mean(axis=0)])
temp_fpr = temp_tpr = temp_fpr1 = temp_tpr1 = np.empty(shape=(0,3),dtype='float64')
temp_fpr, temp_tpr, _ = roc_curve(y_true_combined[:, class_indices["B-EDU-MAJOR"]], y_pred_combined[:, class_indices["B-EDU-MAJOR"]], pos_label=1)
temp_fpr1, temp_tpr1, _ = roc_curve(y_true_combined[:, class_indices["I-EDU-MAJOR"]], y_pred_combined[:, class_indices["I-EDU-MAJOR"]], pos_label=1)
temp_fpr = np.vstack([temp_fpr, temp_fpr1])
temp_tpr = np.vstack([temp_tpr, temp_tpr1])
edu_major_tpr = np.vstack([edu_major_tpr, temp_tpr.mean(axis=0)])
edu_major_fpr = np.vstack([edu_major_fpr, temp_fpr.mean(axis=0)])
temp_fpr = temp_tpr = temp_fpr1 = temp_tpr1 = np.empty(shape=(0,3),dtype='float64')
temp_fpr, temp_tpr, _ = roc_curve(y_true_combined[:, class_indices["B-EDU-INST"]], y_pred_combined[:, class_indices["B-EDU-INST"]], pos_label=1)
temp_fpr1, temp_tpr1, _ = roc_curve(y_true_combined[:, class_indices["I-EDU-INST"]], y_pred_combined[:, class_indices["I-EDU-INST"]], pos_label=1)
temp_fpr = np.vstack([temp_fpr, temp_fpr1])
temp_tpr = np.vstack([temp_tpr, temp_tpr1])
edu_inst_tpr = np.vstack([edu_inst_tpr, temp_tpr.mean(axis=0)])
edu_inst_fpr = np.vstack([edu_inst_fpr, temp_fpr.mean(axis=0)])
"""
emp_pos_scores = np.vstack([
emp_pos_scores,
self.entity_scorer(ds.docs2lines(y_test), y_test_pred,
"EMP-POS")
])
emp_comp_scores = np.vstack([
emp_comp_scores,
self.entity_scorer(ds.docs2lines(y_test), y_test_pred,
"EMP-COMP")
])
edu_major_scores = np.vstack([
edu_major_scores,
self.entity_scorer(ds.docs2lines(y_test), y_test_pred,
"EDU-MAJOR")
])
edu_inst_scores = np.vstack([
edu_inst_scores,
self.entity_scorer(ds.docs2lines(y_test), y_test_pred,
"EDU-INST")
])
w2v_model = None
train_features = test_features = None
"""
print("EMP-POS")
print("precision %s" % np.mean(emp_pos_scores[:,0]))
print("recall %s" % np.mean(emp_pos_scores[:,1]))
print("f1 %s" % np.mean(emp_pos_scores[:,2]))
print("EMP-COMP")
print("precision %s" % np.mean(emp_comp_scores[:,0]))
print("recall %s" % np.mean(emp_comp_scores[:,1]))
print("f1 %s" % np.mean(emp_comp_scores[:,2]))
print("EDU-MAJOR")
print("precision %s" % np.mean(edu_major_scores[:,0]))
print("recall %s" % np.mean(edu_major_scores[:,1]))
print("f1 %s" % np.mean(edu_major_scores[:,2]))
print("EDU-INST")
print("precision %s" % np.mean(edu_inst_scores[:,0]))
print("recall %s" % np.mean(edu_inst_scores[:,1]))
print("f1 %s" % np.mean(edu_inst_scores[:,2]))
emp_pos_tpr = emp_pos_tpr.mean(axis=0)
emp_pos_fpr = emp_pos_fpr.mean(axis=0)
emp_comp_tpr = emp_comp_tpr.mean(axis=0)
emp_comp_fpr = emp_comp_fpr.mean(axis=0)
edu_major_tpr = edu_major_tpr.mean(axis=0)
edu_major_fpr = edu_major_fpr.mean(axis=0)
edu_inst_tpr = edu_inst_tpr.mean(axis=0)
edu_inst_fpr = edu_inst_fpr.mean(axis=0)
lw=2
plt.subplot(221)
plt.plot(emp_pos_fpr, emp_pos_tpr, color='g', linestyle='--', label='EMP-POS', lw=lw)
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC')
plt.legend(loc="lower right")
plt.subplot(222)
plt.plot(emp_comp_fpr, emp_comp_tpr, color='g', linestyle='--', label='EMP-COMP', lw=lw)
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC')
plt.legend(loc="lower right")
plt.subplot(223)
plt.plot(edu_major_fpr, edu_major_tpr, color='g', linestyle='--', label='EDU-MAJOR', lw=lw)
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC')
plt.legend(loc="lower right")
plt.subplot(224)
plt.plot(edu_inst_fpr, edu_inst_tpr, color='g', linestyle='--', label='EDU-INST', lw=lw)
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC')
plt.legend(loc="lower right")
plt.show()
"""
return emp_pos_scores, emp_comp_scores, edu_inst_scores, edu_major_scores
class CrfSuite(Tags):
__seperator = "/"
__crf_model_name = "current_crf_model.pkl"
def __init__(self):
self.logger = Logger()
self.logger.println("CrfSuite created")
def train_model(self, X, y):
self.logger.println("transforming data to train model")
X_combined = list(chain.from_iterable(X))
y_combined = list(chain.from_iterable(y))
self.logger.println("crf trainer init")
crf = sklearn_crfsuite.CRF(algorithm='lbfgs',
c1=0.35,
c2=0.35,
max_iterations=125,
all_possible_transitions=True,
verbose=False)
crf.fit(X_combined, y_combined)
return crf
def save_model(self, model, name=__crf_model_name):
joblib.dump(model, name)
def load_model(self, name=__crf_model_name):
return joblib.load(name)
def score_model(self, y_true, y_pred):
lb = LabelBinarizer()
y_true_combined = lb.fit_transform(list(chain.from_iterable(y_true)))
y_pred_combined = lb.transform(list(chain.from_iterable(y_pred)))
tagset = set(lb.classes_) - {'O'}
tagset = sorted(tagset, key=lambda tag: tag.split('-', 1)[::-1])
class_indices = {cls: idx for idx, cls in enumerate(lb.classes_)}
return f1_score(y_true_combined,
y_pred_combined,
average="weighted",
labels=[class_indices[cls] for cls in tagset])
def print_classification_report(self, y_true, y_pred):
lb = LabelBinarizer()
y_true_combined = lb.fit_transform(list(chain.from_iterable(y_true)))
y_pred_combined = lb.transform(list(chain.from_iterable(y_pred)))
tagset = set(lb.classes_) - {'O'}
tagset = sorted(tagset, key=lambda tag: tag.split('-', 1)[::-1])
class_indices = {cls: idx for idx, cls in enumerate(lb.classes_)}
# TODO return f1 score or other wise here: http://scikit-learn.org/stable/modules/classes.html#module-sklearn.metrics
print(
classification_report(
y_true_combined,
y_pred_combined,
labels=[class_indices[cls] for cls in tagset],
target_names=tagset))
def load_tagger(self):
self.__trained_tagger = pycrfsuite.Tagger()
self.__trained_tagger.open('test_NER.crfsuite')
we_model = WeModel()
self.w2v_model = we_model.read()
dataset = Dataset()
data = dataset.read(nr_of_files=-1)
word2count, word2idx = dataset.encode_dataset(data)
self.f_generator = FeatureGenerator(self.w2v_model, word2count,
word2idx)
# doc: in format of tagged tuples
def tag_doc(self, doc):
feature_input = self.f_generator.generate_features_docs([doc])
model = self.load_model()
"""
xseq = []
for line_idx, line in enumerate(feature_input[0]):
for token_idx, token in enumerate(line):
xseq.append(token)
"""
predicted_tags = model.predict(feature_input[0])
# TODO change this to take in doc and not xseq (convert predicted tags
# to the structure of doc)
return self.interpret_predicted_tags(doc, predicted_tags)
def interpret_predicted_tags(self, doc, tags):
dataset = Dataset()
identified_entities = []
doc = dataset.docs2lines(doc)
tags = dataset.docs2lines(tags)
for tag_idx, tag in enumerate(tags):
if tag in Tags.start_tagset:
entity_found = ""
tag_idx_forward = tag_idx
while True:
if tag_idx_forward >= len(tags) or tags[
tag_idx_forward] == self._Tags__outside_tag:
break
entity_found = entity_found + " " + doc[tag_idx_forward][0]
#entity_found = entity_found + " " + doc[line_idx]['word']
tag_idx_forward += 1
identified_entities.append((entity_found, tags[tag_idx]))
return identified_entities
# use an existing model to tag data
def test_model(self, model, features):
X_features = list(chain.from_iterable(features))
y_pred = model.predict(X_features)
return y_pred
# hyperparamter optimisation
def optimise_model(self, X, y):
# prepare data structure
xseq = []
yseq = []
# transform data structure to group tokens by lines
for doc_x, doc_y in zip(X, y):
for line_idx, line in enumerate(doc_x):
xseq.append(line)
yseq.append(doc_y[line_idx])
# define fixed parameters and parameters to search
crf = sklearn_crfsuite.CRF(algorithm='lbfgs',
max_iterations=200,
all_possible_transitions=True,
verbose=True)
params_space = {
'c1': scipy.stats.expon(scale=0.03),
'c2': scipy.stats.expon(scale=0.03),
}
labels = Tags.tag_list
labels.remove('O')
# use the same metric for evaluation
f1_scorer = make_scorer(metrics.flat_f1_score,
average='weighted',
labels=labels)
# search
rs = RandomizedSearchCV(crf,
params_space,
cv=5,
verbose=1,
n_jobs=2,
n_iter=50,
scoring=f1_scorer)
rs.fit(xseq, yseq)
print('best params:', rs.best_params_)
print('best CV score:', rs.best_score_)
print('model size: {:0.2f}M'.format(rs.best_estimator_.size_ /
1000000))
_x = [s.parameters['c1'] for s in rs.grid_scores_]
_y = [s.parameters['c2'] for s in rs.grid_scores_]
_c = [s.mean_validation_score for s in rs.grid_scores_]
fig = plt.figure()
fig.set_size_inches(12, 12)
ax = plt.gca()
ax.set_yscale('log')
ax.set_xscale('log')
ax.set_xlabel('C1')
ax.set_ylabel('C2')
ax.set_title(
"Randomized Hyperparameter Search CV Results (min={:0.3}, max={:0.3})"
.format(min(_c), max(_c)))
ax.scatter(_x, _y, c=_c, s=60, alpha=0.9, edgecolors=[0, 0, 0])
print("Dark blue => {:0.4}, dark red => {:0.4}".format(
min(_c), max(_c)))
plt.show()
def plot_learning_curve(self, X, y):
train_sizes = np.linspace(.1, 1.0, 5)
n_jobs = 8
title = "Learning Curves"
cv = ShuffleSplit(n_splits=5, test_size=0.2, random_state=0)
plt.figure()
plt.title(title)
ylim = (0.01, 1.01)
if ylim is not None:
plt.ylim(*ylim)
plt.xlabel("Training examples")
plt.ylabel("Score")
X_lines = []
y_lines = []
for doc_x, doc_y in zip(X, y):
for line_idx, line in enumerate(doc_x):
X_lines.append(line)
y_lines.append(doc_y[line_idx])
estimator = sklearn_crfsuite.CRF(algorithm='lbfgs',
c1=0.001,
c2=0.001,
max_iterations=110,
all_possible_transitions=True,
verbose=True)
custom_scorer = make_scorer(self.score_model, greater_is_better=True)
#train_sizes, train_scores, test_scores = learning_curve(estimator, X_lines, y_lines, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
train_sizes, train_scores, test_scores = learning_curve(
estimator,
X_lines,
y_lines,
cv=cv,
scoring=custom_scorer,
n_jobs=n_jobs,
train_sizes=train_sizes)
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
plt.grid()
plt.fill_between(train_sizes,
train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std,
alpha=0.1,
color="r")
plt.fill_between(train_sizes,
test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std,
alpha=0.1,
color="g")
plt.plot(train_sizes,
train_scores_mean,
'o-',
color="r",
label="Training score")
plt.plot(train_sizes,
test_scores_mean,
'o-',
color="g",
label="Cross-validation score")
plt.legend(loc="best")
return plt