def test_update(self): tagger = TaggingPerceptron(vocabulary(self.dataset), tags(self.dataset)) for ss, pred ,tt, expected_w in zip(self.sents, self.predicted_tags, self.tags, self.expected_ws): w = tagger.update(ss, pred, tt) self.assertSequenceEqual(list(w), list(expected_w))
def test_update(self): tagger = TaggingPerceptron(vocabulary(self.dataset), tags(self.dataset)) for ss, pred ,tt, expected_w in zip(self.sents, self.predicted_tags, self.tags, self.expected_ws): w = tagger.update(ss, pred, tt) self.assertSequenceEqual(list(w), list(expected_w))
def test_feature_vector(self): tagger = TaggingPerceptron(vocabulary(self.dataset), tags(self.dataset)) for sample_ix, (sent, tag) in enumerate(zip(self.sents, \ self.tags)): ftr_vectr = tagger.feature_vector(sent,tag) self.assertSequenceEqual(list(ftr_vectr), list(self.expected_train_ftr_vectos[sample_ix]))
def test_feature_vector(self): tagger = TaggingPerceptron(vocabulary(self.dataset), tags(self.dataset)) for sample_ix, (sent, tag) in enumerate(zip(self.sents, \ self.tags)): ftr_vectr = tagger.feature_vector(sent,tag) self.assertSequenceEqual(list(ftr_vectr), list(self.expected_train_ftr_vectos[sample_ix]))
def test_decode(self): tagger = TaggingPerceptron(vocabulary(self.dataset), tags(self.dataset)) # one iteration of updates tagger.train(2, self.sents, self.tags, self.sents, self.tags) for sent, expected_output in zip(self.sents, self.expected_tags_two_itr): tag_seq = tagger.decode(sent) self.assertSequenceEqual(tag_seq, expected_output)
def test_decode(self): tagger = TaggingPerceptron(vocabulary(self.dataset), tags(self.dataset)) # one iteration of updates tagger.train(2, self.sents, self.tags, self.sents, self.tags) for sent, expected_output in zip(self.sents, self.expected_tags_two_itr): tag_seq = tagger.decode(sent) self.assertSequenceEqual(tag_seq, expected_output)
if self._n % report_every == 1: test_accuracy, confusion, word_errors = \ self.accuracy(test_sents, test_tags) print("\t".join(ss)) print("\t".join(tt)) print("\t".join(pred)) print("After %i sents, accuracy is %f, nonzero feats %i" % (self._n, test_accuracy, sum(1 for x in self._w if x != 0.0))) self.finalize() test_accuracy, confusion, word_errors = \ self.accuracy(test_sents, test_tags) print("---------------") print("Final accuracy: %f" % test_accuracy) if __name__ == "__main__": #CoNLL conll_train = CoNLL2003_Train() conll_valid = CoNLL2003_Valid() train_sents, train_tags = dataset_to_sents_and_tags(conll_train) valid_sents, valid_tags = dataset_to_sents_and_tags(conll_valid) tp = TaggingPerceptron(vocabulary(conll_train), tags(conll_train)) itrs = 5 tp.train(itrs, train_sents, train_tags, valid_sents, valid_tags)