def test(self, multilabel_threshold=0.5):
if not self.include_test:
raise ValueError('Please do not call .test() if no test data is available.')
score_dict = {}
# get test predictions:
test_in = {}
if self.include_token:
test_in['focus_in'] = self.test_X_focus
if self.include_context:
test_in['context_in'] = self.test_contexts
test_preds = self.model.predict(test_in,
batch_size=self.batch_size)
if isinstance(test_preds, np.ndarray):
test_preds = [test_preds]
if self.include_lemma:
print('::: Test scores (lemmas) :::')
pred_lemmas = self.preprocessor.inverse_transform_lemmas(predictions=test_preds[self.lemma_out_idx])
if self.postcorrect:
for i in range(len(pred_lemmas)):
if pred_lemmas[i] not in self.known_lemmas:
pred_lemmas[i] = min(self.known_lemmas,
key=lambda x: editdistance.eval(x, pred_lemmas[i]))
score_dict['test_lemma'] = evaluation.single_label_accuracies(gold=self.test_lemmas,
silver=pred_lemmas,
test_tokens=self.test_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_pos:
print('::: Test scores (pos) :::')
pred_pos = self.preprocessor.inverse_transform_pos(predictions=test_preds[self.pos_out_idx])
score_dict['test_pos'] = evaluation.single_label_accuracies(gold=self.test_pos,
silver=pred_pos,
test_tokens=self.test_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_morph:
print('::: Test scores (morph) :::')
pred_morph = self.preprocessor.inverse_transform_morph(predictions=test_preds[self.morph_out_idx],
threshold=multilabel_threshold)
if self.include_morph == 'label':
score_dict['test_morph'] = evaluation.single_label_accuracies(gold=self.test_morph,
silver=pred_morph,
test_tokens=self.test_tokens,
known_tokens=self.preprocessor.known_tokens)
elif self.include_morph == 'multilabel':
score_dict['test_morph'] = evaluation.multilabel_accuracies(gold=self.test_morph,
silver=pred_morph,
test_tokens=self.test_tokens,
known_tokens=self.preprocessor.known_tokens)
return score_dict
def epoch(self, autosave=True):
if not self.setup:
raise ValueError('Not set up yet... Call Tagger.setup_() first.')
# update nb of epochs ran so far:
self.curr_nb_epochs += 1
print("-> epoch ", self.curr_nb_epochs, "...")
if self.curr_nb_epochs and self.halve_lr_at:
# update learning rate at specific points:
if self.curr_nb_epochs % self.halve_lr_at == 0:
old_lr = self.model.optimizer.lr.get_value()
new_lr = np.float32(old_lr * 0.5)
self.model.optimizer.lr.set_value(new_lr)
print('\t- Lowering learning rate > was:', old_lr, ', now:', new_lr)
# get inputs and outputs straight:
train_in, train_out = {}, {}
if self.include_token:
train_in['focus_in'] = self.train_X_focus
if self.include_context:
train_in['context_in'] = self.train_contexts
if self.include_lemma:
train_out['lemma_out'] = self.train_X_lemma
if self.include_pos:
train_out['pos_out'] = self.train_X_pos
if self.include_morph:
train_out['morph_out'] = self.train_X_morph
self.model.fit(train_in, train_out,
nb_epoch = 1,
shuffle = True,
batch_size = self.batch_size)
# get train preds:
train_preds = self.model.predict(train_in,
batch_size=self.batch_size)
if isinstance(train_preds, np.ndarray):
train_preds = [train_preds]
if self.include_dev:
dev_in = {}
if self.include_token:
dev_in['focus_in'] = self.dev_X_focus
if self.include_context:
dev_in['context_in'] = self.dev_contexts
dev_preds = self.model.predict(dev_in,
batch_size=self.batch_size)
if isinstance(dev_preds, np.ndarray):
dev_preds = [dev_preds]
score_dict = {}
if self.include_lemma:
print('::: Train scores (lemmas) :::')
pred_lemmas = self.preprocessor.inverse_transform_lemmas(predictions=train_preds[self.lemma_out_idx])
score_dict['train_lemma'] = evaluation.single_label_accuracies(gold=self.train_lemmas,
silver=pred_lemmas,
test_tokens=self.train_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_dev:
print('::: Dev scores (lemmas) :::')
pred_lemmas = self.preprocessor.inverse_transform_lemmas(predictions=dev_preds[self.lemma_out_idx])
score_dict['dev_lemma'] = evaluation.single_label_accuracies(gold=self.dev_lemmas,
silver=pred_lemmas,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.postcorrect:
print('::: Dev scores (lemmas) -> postcorrected :::')
for i in range(len(pred_lemmas)):
if pred_lemmas[i] not in self.known_lemmas:
pred_lemmas[i] = min(self.known_lemmas,
key=lambda x: editdistance.eval(x, pred_lemmas[i]))
score_dict['dev_lemma_postcorrect'] = evaluation.single_label_accuracies(gold=self.dev_lemmas,
silver=pred_lemmas,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_pos:
print('::: Train scores (pos) :::')
pred_pos = self.preprocessor.inverse_transform_pos(predictions=train_preds[self.pos_out_idx])
score_dict['train_pos'] = evaluation.single_label_accuracies(gold=self.train_pos,
silver=pred_pos,
test_tokens=self.train_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_dev:
print('::: Dev scores (pos) :::')
pred_pos = self.preprocessor.inverse_transform_pos(predictions=dev_preds[self.pos_out_idx])
score_dict['dev_pos'] = evaluation.single_label_accuracies(gold=self.dev_pos,
silver=pred_pos,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_morph:
print('::: Train scores (morph) :::')
pred_morph = self.preprocessor.inverse_transform_morph(predictions=train_preds[self.morph_out_idx])
if self.include_morph == 'label':
score_dict['train_morph'] = evaluation.single_label_accuracies(gold=self.train_morph,
silver=pred_morph,
test_tokens=self.train_tokens,
known_tokens=self.preprocessor.known_tokens)
elif self.include_morph == 'multilabel':
score_dict['train_morph'] = evaluation.multilabel_accuracies(gold=self.train_morph,
silver=pred_morph,
test_tokens=self.train_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_dev:
print('::: Dev scores (morph) :::')
pred_morph = self.preprocessor.inverse_transform_morph(predictions=dev_preds[self.morph_out_idx])
if self.include_morph == 'label':
score_dict['dev_morph'] = evaluation.single_label_accuracies(gold=self.train_morph,
silver=pred_morph,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
elif self.include_morph == 'multilabel':
score_dict['dev_morph'] = evaluation.multilabel_accuracies(gold=self.train_morph,
silver=pred_morph,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
if autosave:
self.save()
return score_dict
def epoch(self, autosave=True):
if not self.setup:
raise ValueError('Not set up yet... Call Tagger.setup_() first.')
# update nb of epochs ran so far:
self.curr_nb_epochs += 1
print("-> epoch ", self.curr_nb_epochs, "...")
if self.curr_nb_epochs and self.halve_lr_at:
# update learning rate at specific points:
if self.curr_nb_epochs % self.halve_lr_at == 0:
old_lr = self.model.optimizer.lr.get_value()
new_lr = np.float32(old_lr * 0.5)
self.model.optimizer.lr.set_value(new_lr)
print('\t- Lowering learning rate > was:', old_lr, ', now:',
new_lr)
# get inputs and outputs straight:
train_in, train_out = {}, {}
if self.include_token:
train_in['focus_in'] = self.train_X_focus
if self.include_context:
train_in['context_in'] = self.train_contexts
if self.include_lemma:
train_out['lemma_out'] = self.train_X_lemma
if self.include_pos:
train_out['pos_out'] = self.train_X_pos
if self.include_morph:
train_out['morph_out'] = self.train_X_morph
self.model.fit(train_in,
train_out,
nb_epoch=1,
shuffle=True,
batch_size=self.batch_size)
# get train preds:
train_preds = self.model.predict(train_in, batch_size=self.batch_size)
if isinstance(train_preds, np.ndarray):
train_preds = [train_preds]
if self.include_dev:
dev_in = {}
if self.include_token:
dev_in['focus_in'] = self.dev_X_focus
if self.include_context:
dev_in['context_in'] = self.dev_contexts
dev_preds = self.model.predict(dev_in, batch_size=self.batch_size)
if isinstance(dev_preds, np.ndarray):
dev_preds = [dev_preds]
score_dict = {}
if self.include_lemma:
print('::: Train scores (lemmas) :::')
pred_lemmas = self.preprocessor.inverse_transform_lemmas(
predictions=train_preds[self.lemma_out_idx])
score_dict['train_lemma'] = evaluation.single_label_accuracies(
gold=self.train_lemmas,
silver=pred_lemmas,
test_tokens=self.train_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_dev:
print('::: Dev scores (lemmas) :::')
pred_lemmas = self.preprocessor.inverse_transform_lemmas(
predictions=dev_preds[self.lemma_out_idx])
score_dict['dev_lemma'] = evaluation.single_label_accuracies(
gold=self.dev_lemmas,
silver=pred_lemmas,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.postcorrect:
print('::: Dev scores (lemmas) -> postcorrected :::')
for i in range(len(pred_lemmas)):
if pred_lemmas[i] not in self.known_lemmas:
pred_lemmas[i] = min(self.known_lemmas,
key=lambda x: editdistance.
eval(x, pred_lemmas[i]))
score_dict[
'dev_lemma_postcorrect'] = evaluation.single_label_accuracies(
gold=self.dev_lemmas,
silver=pred_lemmas,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_pos:
print('::: Train scores (pos) :::')
pred_pos = self.preprocessor.inverse_transform_pos(
predictions=train_preds[self.pos_out_idx])
score_dict['train_pos'] = evaluation.single_label_accuracies(
gold=self.train_pos,
silver=pred_pos,
test_tokens=self.train_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_dev:
print('::: Dev scores (pos) :::')
pred_pos = self.preprocessor.inverse_transform_pos(
predictions=dev_preds[self.pos_out_idx])
score_dict['dev_pos'] = evaluation.single_label_accuracies(
gold=self.dev_pos,
silver=pred_pos,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_morph:
print('::: Train scores (morph) :::')
pred_morph = self.preprocessor.inverse_transform_morph(
predictions=train_preds[self.morph_out_idx])
if self.include_morph == 'label':
score_dict['train_morph'] = evaluation.single_label_accuracies(
gold=self.train_morph,
silver=pred_morph,
test_tokens=self.train_tokens,
known_tokens=self.preprocessor.known_tokens)
elif self.include_morph == 'multilabel':
score_dict['train_morph'] = evaluation.multilabel_accuracies(
gold=self.train_morph,
silver=pred_morph,
test_tokens=self.train_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_dev:
print('::: Dev scores (morph) :::')
pred_morph = self.preprocessor.inverse_transform_morph(
predictions=dev_preds[self.morph_out_idx])
if self.include_morph == 'label':
score_dict[
'dev_morph'] = evaluation.single_label_accuracies(
gold=self.train_morph,
silver=pred_morph,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
elif self.include_morph == 'multilabel':
score_dict['dev_morph'] = evaluation.multilabel_accuracies(
gold=self.train_morph,
silver=pred_morph,
test_tokens=self.dev_tokens,
known_tokens=self.preprocessor.known_tokens)
if autosave:
self.save()
return score_dict
def test(self, multilabel_threshold=0.5):
if not self.include_test:
raise ValueError(
'Please do not call .test() if no test data is available.')
score_dict = {}
# get test predictions:
test_in = {}
if self.include_token:
test_in['focus_in'] = self.test_X_focus
if self.include_context:
test_in['context_in'] = self.test_contexts
test_preds = self.model.predict(test_in, batch_size=self.batch_size)
if isinstance(test_preds, np.ndarray):
test_preds = [test_preds]
if self.include_lemma:
print('::: Test scores (lemmas) :::')
pred_lemmas = self.preprocessor.inverse_transform_lemmas(
predictions=test_preds[self.lemma_out_idx])
if self.postcorrect:
for i in range(len(pred_lemmas)):
if pred_lemmas[i] not in self.known_lemmas:
pred_lemmas[i] = min(
self.known_lemmas,
key=lambda x: editdistance.eval(x, pred_lemmas[i]))
score_dict['test_lemma'] = evaluation.single_label_accuracies(
gold=self.test_lemmas,
silver=pred_lemmas,
test_tokens=self.test_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_pos:
print('::: Test scores (pos) :::')
pred_pos = self.preprocessor.inverse_transform_pos(
predictions=test_preds[self.pos_out_idx])
score_dict['test_pos'] = evaluation.single_label_accuracies(
gold=self.test_pos,
silver=pred_pos,
test_tokens=self.test_tokens,
known_tokens=self.preprocessor.known_tokens)
if self.include_morph:
print('::: Test scores (morph) :::')
pred_morph = self.preprocessor.inverse_transform_morph(
predictions=test_preds[self.morph_out_idx],
threshold=multilabel_threshold)
if self.include_morph == 'label':
score_dict['test_morph'] = evaluation.single_label_accuracies(
gold=self.test_morph,
silver=pred_morph,
test_tokens=self.test_tokens,
known_tokens=self.preprocessor.known_tokens)
elif self.include_morph == 'multilabel':
score_dict['test_morph'] = evaluation.multilabel_accuracies(
gold=self.test_morph,
silver=pred_morph,
test_tokens=self.test_tokens,
known_tokens=self.preprocessor.known_tokens)
return score_dict