def train(self, essay_feats, save_loc=None, nr_iter=5, verbose=True): '''Train a model from sentences, and save it at ``save_loc``. ``nr_iter`` controls the number of Perceptron training iterations. :param sentences: A list of (words, tags) tuples. :param save_loc: If not ``None``, saves a pickled model in this location. :param nr_iter: Number of training iterations. ''' cp_essay_feats = list(essay_feats) # Copy as we do an inplace shuffle below tag_freq = defaultdict(int) for essay in cp_essay_feats: for taggged_sentence in essay.sentences: for wd in taggged_sentence: fs_tags = self.__get_tags_(wd.tags) tag_freq[fs_tags] +=1 self.classes = set([ fs for fs, cnt in tag_freq.items() if cnt >= self.combo_freq_threshold]) self.model = AveragedPerceptron(self.classes) for iter_ in range(nr_iter): class2predictions = defaultdict(list) class2tags = defaultdict(list) for essay_ix, essay in enumerate(cp_essay_feats): for sent_ix, taggged_sentence in enumerate(essay.sentences): """ Start Sentence """ prev = list(self.START) for i, (wd) in enumerate(taggged_sentence): # Don't mutate the feat dictionary shared_features = dict(wd.features.items()) # get all tagger predictions for previous 2 tags self._add_secondary_tag_features(shared_features, prev) tagger_feats = dict(shared_features.items()) # add more in depth features for this tag actual = self.__get_tags_(wd.tags) if self.use_tag_features: self._add_tag_features(tagger_feats, wd.word, prev[-1], prev[-2]) guess = self.model.predict(tagger_feats) self.model.update(actual, guess, tagger_feats) prev.append(guess) for cls in self.individual_tags: class2predictions[cls].append( 1 if cls in guess else 0 ) class2tags[cls].append( 1 if cls in actual else 0) random.shuffle(cp_essay_feats) class2metrics = ResultsProcessor.compute_metrics(class2tags, class2predictions) micro_metrics = micro_rpfa(class2metrics.values()) if verbose: logging.info("Iter {0}: Micro Avg Metrics: {1}".format(iter_, str(micro_metrics))) self.model.average_weights() return None
def train(self, tagged_essays, max_epochs): ys_by_sent = self.get_label_data(tagged_essays) for i in range(0, max_epochs): self.epoch += 1 self.log("Epoch: {epoch}".format(epoch=self.epoch)) # TODO - provide option for different model types here? parse_examples = WeightedExamples(labels=PARSE_ACTIONS, positive_value=self.positive_val) crel_examples = WeightedExamples(labels=None, positive_value=self.positive_val) pred_ys_by_sent = defaultdict(list) for essay_ix, essay in enumerate(tagged_essays): for sent_ix, taggged_sentence in enumerate(essay.sentences): predicted_tags = essay.pred_tagged_sentences[sent_ix] pred_relations = self.generate_training_data( taggged_sentence, predicted_tags, parse_examples, crel_examples) # Store predictions for evaluation self.add_cr_labels(pred_relations, pred_ys_by_sent) class2metrics = ResultsProcessor.compute_metrics( ys_by_sent, pred_ys_by_sent) micro_metrics = micro_rpfa(class2metrics.values()) # type: rpfa self.log( "Training Metrics: {metrics}".format(metrics=micro_metrics)) self.train_parse_models(parse_examples) self.train_crel_models(crel_examples) self.training_datasets_parsing[self.epoch] = parse_examples self.training_datasets_crel[self.epoch] = crel_examples
def train(self, essay_feats, save_loc=None, nr_iter=5, verbose=True): '''Train a model from sentences, and save it at ``save_loc``. ``nr_iter`` controls the number of Perceptron training iterations. :param sentences: A list of (words, tags) tuples. :param save_loc: If not ``None``, saves a pickled model in this location. :param nr_iter: Number of training iterations. ''' # Copy as we do an inplace shuffle below cp_essay_feats = list(essay_feats) for iter_ in range(nr_iter): class2predictions = defaultdict(list) class2tags = defaultdict(list) for essay_ix, essay in enumerate(cp_essay_feats): for sent_ix, taggged_sentence in enumerate(essay.sentences): """ Start Sentence """ class2prev = defaultdict(list) for cls in self.classes: class2prev[cls] = list(self.START) for wd in taggged_sentence: # Don't mutate the feat dictionary shared_features = dict(wd.features.items()) # get all tagger predictions for previous 2 tags for cls in self.classes: self._add_secondary_tag_features( shared_features, wd.word, cls, class2prev[cls]) # train each binary tagger for cls in self.classes: tagger_feats = dict(shared_features.items()) # add more in depth features for this tag self._add_tag_features(tagger_feats, wd.word, class2prev[cls][-1], class2prev[cls][-2]) actual = self.__get_yal_(wd, cls) model = self.class2model[cls] guess = model.predict(tagger_feats) model.update(actual, guess, tagger_feats) class2prev[cls].append(guess) class2predictions[cls].append(guess) class2tags[cls].append(actual) random.shuffle(cp_essay_feats) class2metrics = ResultsProcessor.compute_metrics( class2tags, class2predictions) wtd_mean = weighted_mean_rpfa(class2metrics.values()) if verbose: logging.info("Iter {0}: Wtd Mean: {1}".format( iter_, str(wtd_mean))) for cls in self.classes: self.class2model[cls].average_weights() return None
def train(self, essay_feats, save_loc=None, nr_iter=5, verbose=True): '''Train a model from sentences, and save it at ``save_loc``. ``nr_iter`` controls the number of Perceptron training iterations. :param sentences: A list of (words, tags) tuples. :param save_loc: If not ``None``, saves a pickled model in this location. :param nr_iter: Number of training iterations. ''' # Copy as we do an inplace shuffle below cp_essay_feats = list(essay_feats) for iter_ in range(nr_iter): class2predictions = defaultdict(list) class2tags = defaultdict(list) for essay_ix, essay in enumerate(cp_essay_feats): for sent_ix, taggged_sentence in enumerate(essay.sentences): """ Start Sentence """ class2prev = defaultdict(list) for cls in self.classes: class2prev[cls] = list(self.START) for wd in taggged_sentence: # Don't mutate the feat dictionary shared_features = dict(wd.features.items()) # get all tagger predictions for previous 2 tags for cls in self.classes: self._add_secondary_tag_features(shared_features, wd.word, cls, class2prev[cls]) # train each binary tagger for cls in self.classes: tagger_feats = dict(shared_features.items()) # add more in depth features for this tag self._add_tag_features(tagger_feats, wd.word, class2prev[cls][-1], class2prev[cls][-2]) actual = self.__get_yal_(wd, cls) model = self.class2model[cls] guess = model.predict(tagger_feats) model.update(actual, guess, tagger_feats) class2prev[cls].append(guess) class2predictions[cls].append(guess) class2tags[cls].append(actual) random.shuffle(cp_essay_feats) class2metrics = ResultsProcessor.compute_metrics(class2tags, class2predictions) wtd_mean = weighted_mean_rpfa(class2metrics.values()) if verbose: logging.info("Iter {0}: Wtd Mean: {1}".format(iter_, str(wtd_mean))) for cls in self.classes: self.class2model[cls].average_weights() return None
def train(self, tagged_essays, max_epochs): trained_with_beta0 = False ys_by_sent = self.get_label_data(tagged_essays) for i in range(0, max_epochs): if self.beta < 0: trained_with_beta0 = True self.epoch += 1 print("Epoch: {epoch}".format(epoch=self.epoch)) print("Beta: {beta}".format(beta=self.beta)) # TODO - provide option for different model types here? parse_examples = WeightedExamples(labels=PARSE_ACTIONS, positive_value=self.positive_val) crel_examples = WeightedExamples(labels=None, positive_value=self.positive_val) pred_ys_by_sent = defaultdict(list) for essay_ix, essay in enumerate(tagged_essays): for sent_ix, taggged_sentence in enumerate(essay.sentences): predicted_tags = essay.pred_tagged_sentences[sent_ix] pred_relations = self.generate_training_data(taggged_sentence, predicted_tags, parse_examples, crel_examples) # Store predictions for evaluation self.add_cr_labels(pred_relations, pred_ys_by_sent) class2metrics = ResultsProcessor.compute_metrics(ys_by_sent, pred_ys_by_sent) micro_metrics = micro_rpfa(class2metrics.values()) # type: rpfa print("Training Metrics: {metrics}".format(metrics=micro_metrics)) # TODO, dictionary vectorize examples, train a weighted binary classifier for each separate parsing action self.train_parse_models(parse_examples) self.train_crel_models(crel_examples) self.training_datasets_parsing[self.epoch] = parse_examples self.training_datasets_crel[self.epoch] = crel_examples # Decay beta self.beta = self.beta_decay_fn(self.beta) if self.beta < 0 and trained_with_beta0: print("beta decayed below 0 - beta:'{beta}', stopping".format(beta=self.beta)) break # end [for each epoch] if not trained_with_beta0: print("Algorithm hit max epochs without training with beta <= 0 - final_beta:{beta}".format(beta=self.beta))
def train(self, tagged_essays, max_epochs): trained_with_beta0 = False ys_by_sent = self.get_label_data(tagged_essays) for i in range(0, max_epochs): if self.beta < 0: trained_with_beta0 = True self.epoch += 1 print("Epoch: {epoch}".format(epoch=self.epoch)) print("Beta: {beta}".format(beta=self.beta)) # TODO - provide option for different model types here? parse_examples = WeightedExamples(labels=PARSE_ACTIONS, positive_value=self.positive_val) crel_examples = WeightedExamples(labels=None, positive_value=self.positive_val) pred_ys_by_sent = defaultdict(list) for essay_ix, essay in enumerate(tagged_essays): for sent_ix, taggged_sentence in enumerate(essay.sentences): predicted_tags = essay.pred_tagged_sentences[sent_ix] pred_relations = self.generate_training_data(taggged_sentence, predicted_tags, parse_examples, crel_examples) # Store predictions for evaluation self.add_cr_labels(pred_relations, pred_ys_by_sent) class2metrics = ResultsProcessor.compute_metrics(ys_by_sent, pred_ys_by_sent) micro_metrics = micro_rpfa(class2metrics.values()) # type: rpfa print("Training Metrics: {metrics}".format(metrics=micro_metrics)) # TODO, dictionary vectorize examples, train a weighted binary classifier for each separate parsing action self.train_parse_models(parse_examples) self.train_crel_models(crel_examples) self.training_datasets_parsing[self.epoch] = parse_examples self.training_datasets_crel[self.epoch] = crel_examples # Decay beta self.beta = self.beta_decay_fn(self.beta) if self.beta < 0 and trained_with_beta0: print("beta decayed below 0 - beta:'{beta}', stopping".format(beta=self.beta)) break # end [for each epoch] if not trained_with_beta0: print("Algorithm hit max epochs without training with beta <= 0 - final_beta:{beta}".format(beta=self.beta))
def evaluate_tagger_on_fold(kfold, wd_train_tags, tag_history, tag_plus_word, tag_ngram, avg_weights=True, split=0.2): # logger.info("Loading data for fold %i" % kfold) k_fold_data = k_fold_2data[kfold] essays_TD, essays_VD, essays_TD_most_freq, wd_td_ys_bytag, wd_vd_ys_bytag = k_fold_data """ TRAINING """ tagger = PerceptronTaggerLabelPowerset(wd_train_tags, combo_freq_threshold=1, tag_history=tag_history, tag_plus_word=tag_plus_word, tag_ngram_size=tag_ngram) # Split into train and test set np_essays = np.asarray(essays_TD_most_freq) ixs = np.arange(len(essays_TD_most_freq)) np.random.shuffle(ixs) split_size = int(split * len(essays_TD_most_freq)) test, train = np_essays[ixs[:split_size]], np_essays[ixs[split_size:]] _, test_tags = flatten_to_wordlevel_feat_tags(test) class2ys = get_wordlevel_ys_by_code(test_tags, wd_train_tags) optimal_num_iterations = -1 last_f1 = -1 """ EARLY STOPPING USING TEST SET """ for i in range(30): tagger.train(train, nr_iter=1, verbose=False, average_weights=False) wts_copy = dict(tagger.model.weights.items()) if avg_weights: tagger.model.average_weights() class2predictions = tagger.predict(test) #Compute F1 score, stop early if worse than previous class2metrics = ResultsProcessor.compute_metrics(class2ys, class2predictions) micro_metrics = micro_rpfa(class2metrics.values()) current_f1 = micro_metrics.f1_score if current_f1 <= last_f1: optimal_num_iterations = i # i.e. this number minus 1, but 0 based break # Reset weights (as we are averaging weights) tagger.model.weights = wts_copy last_f1 = current_f1 # print("fold %i - Optimal F1 obtained at iteration %i " % (kfold, optimal_num_iterations)) """ Re-train model using stopping criterion on full training set """ final_tagger = PerceptronTaggerLabelPowerset(wd_train_tags, combo_freq_threshold=1, tag_history=tag_history, tag_plus_word=tag_plus_word, tag_ngram_size=tag_ngram) final_tagger.train(essays_TD_most_freq, nr_iter=optimal_num_iterations, verbose=False, average_weights=avg_weights) """ PREDICT """ td_wd_predictions_by_code = final_tagger.predict(essays_TD) vd_wd_predictions_by_code = final_tagger.predict(essays_VD) # logger.info("Fold %i finished" % kfold) """ Aggregate results """ return kfold, td_wd_predictions_by_code, vd_wd_predictions_by_code, optimal_num_iterations
def score_predictions(model, xs, ys_by_tag, seq_len): preds = model.predict_classes(xs, batch_size=batch_size, verbose=0) pred_ys_by_tag = collapse_results(seq_len, preds) class2metrics = ResultsProcessor.compute_metrics(ys_by_tag, pred_ys_by_tag) micro_metrics = micro_rpfa(class2metrics.values()) return micro_metrics, pred_ys_by_tag
def evaluate_tagger_on_fold(kfold, wd_train_tags, tag_history, tag_plus_word, tag_ngram, avg_weights=True, split=0.2): # logger.info("Loading data for fold %i" % kfold) k_fold_data = k_fold_2data[kfold] essays_TD, essays_VD, essays_TD_most_freq, wd_td_ys_bytag, wd_vd_ys_bytag = k_fold_data """ TRAINING """ tagger = PerceptronTaggerLabelPowerset(wd_train_tags, combo_freq_threshold=1, tag_history=tag_history, tag_plus_word=tag_plus_word, tag_ngram_size=tag_ngram) # Split into train and test set np_essays = np.asarray(essays_TD_most_freq) ixs = np.arange(len(essays_TD_most_freq)) np.random.shuffle(ixs) split_size = int(split * len(essays_TD_most_freq)) test, train = np_essays[ixs[:split_size]], np_essays[ixs[split_size:]] _, test_tags = flatten_to_wordlevel_feat_tags(test) class2ys = get_wordlevel_ys_by_code(test_tags, wd_train_tags) optimal_num_iterations = -1 last_f1 = -1 """ EARLY STOPPING USING TEST SET """ for i in range(30): tagger.train(train, nr_iter=1, verbose=False, average_weights=False) wts_copy = dict(tagger.model.weights.items()) if avg_weights: tagger.model.average_weights() class2predictions = tagger.predict(test) #Compute F1 score, stop early if worse than previous class2metrics = ResultsProcessor.compute_metrics( class2ys, class2predictions) micro_metrics = micro_rpfa(class2metrics.values()) current_f1 = micro_metrics.f1_score if current_f1 <= last_f1: optimal_num_iterations = i # i.e. this number minus 1, but 0 based break # Reset weights (as we are averaging weights) tagger.model.weights = wts_copy last_f1 = current_f1 # print("fold %i - Optimal F1 obtained at iteration %i " % (kfold, optimal_num_iterations)) """ Re-train model using stopping criterion on full training set """ final_tagger = PerceptronTaggerLabelPowerset(wd_train_tags, combo_freq_threshold=1, tag_history=tag_history, tag_plus_word=tag_plus_word, tag_ngram_size=tag_ngram) final_tagger.train(essays_TD_most_freq, nr_iter=optimal_num_iterations, verbose=False, average_weights=avg_weights) """ PREDICT """ td_wd_predictions_by_code = final_tagger.predict(essays_TD) vd_wd_predictions_by_code = final_tagger.predict(essays_VD) # logger.info("Fold %i finished" % kfold) """ Aggregate results """ return kfold, td_wd_predictions_by_code, vd_wd_predictions_by_code, optimal_num_iterations
def score_predictions(model, xs, ys_by_tag, seq_len): preds = model.predict_classes(xs, batch_size=batch_size, verbose=0) pred_ys_by_tag = collapse_results(seq_len, preds) class2metrics = ResultsProcessor.compute_metrics(ys_by_tag, pred_ys_by_tag) micro_metrics = micro_rpfa(class2metrics.values()) return micro_metrics, pred_ys_by_tag
def train(self, essay_feats, save_loc=None, nr_iter=5, verbose=True): '''Train a model from sentences, and save it at ``save_loc``. ``nr_iter`` controls the number of Perceptron training iterations. :param sentences: A list of (words, tags) tuples. :param save_loc: If not ``None``, saves a pickled model in this location. :param nr_iter: Number of training iterations. ''' cp_essay_feats = list(essay_feats) # Copy as we do an inplace shuffle below tag_freq = defaultdict(int) for essay in cp_essay_feats: for taggged_sentence in essay.sentences: for wd in taggged_sentence: fs_tags = self.__get_tags_(wd.tags) tag_freq[fs_tags] += 1 self.classes = set([ fs for fs, cnt in tag_freq.items() if cnt >= self.combo_freq_threshold ]) self.model = AveragedPerceptron(self.classes) for iter_ in range(nr_iter): class2predictions = defaultdict(list) class2tags = defaultdict(list) for essay_ix, essay in enumerate(cp_essay_feats): for sent_ix, taggged_sentence in enumerate(essay.sentences): """ Start Sentence """ prev = list(self.START) for i, (wd) in enumerate(taggged_sentence): # Don't mutate the feat dictionary shared_features = dict(wd.features.items()) # get all tagger predictions for previous 2 tags self._add_secondary_tag_features(shared_features, prev) tagger_feats = dict(shared_features.items()) # add more in depth features for this tag actual = self.__get_tags_(wd.tags) if self.use_tag_features: self._add_tag_features(tagger_feats, wd.word, prev[-1], prev[-2]) guess = self.model.predict(tagger_feats) self.model.update(actual, guess, tagger_feats) prev.append(guess) for cls in self.individual_tags: class2predictions[cls].append(1 if cls in guess else 0) class2tags[cls].append(1 if cls in actual else 0) random.shuffle(cp_essay_feats) class2metrics = ResultsProcessor.compute_metrics( class2tags, class2predictions) micro_metrics = micro_rpfa(class2metrics.values()) if verbose: logging.info("Iter {0}: Micro Avg Metrics: {1}".format( iter_, str(micro_metrics))) self.model.average_weights() return None