def gold_evaluation(sentences, extractor, gazetteer, model_cls, model_args): logger.info("Evaluating on the gold sentences") for each in sentences: if not each.get("gold_fes"): extractor.process_sentence(each["sentence"], each["lu"], each["fes"], add_unknown=True, gazetteer=gazetteer) x_tr, y_tr = extractor.get_features(refit=True) extractor.start() tagged_gold = [] for each in sentences: if each.get("gold_fes"): tagged_gold.append( ( each["gold_fes"], extractor.process_sentence( each["sentence"], each["lu"], each["fes"], add_unknown=False, gazetteer=gazetteer ), ) ) if not tagged_gold: logger.warn("asked to evaluate gold, but no gold sentences found") return x_gold, _ = extractor.get_features(refit=False) y_gold = [] for gold_fes, tagged in tagged_gold: for chunk, is_sample in tagged: if is_sample: y_gold.append([extractor.label_index[fe or "O"] for fe in gold_fes.get(chunk, [])]) assert len(y_gold) == x_gold.shape[0] model = FeatureSelectedClassifier(model_cls, extractor.lu_column(), model_args) model.fit(x_tr, y_tr) y_pred = model.predict(x_gold) correct = len([1 for actual, predicted in zip(y_gold, y_pred) if predicted in actual]) logger.info("Gold accuracy: %f (%d / %d roles)", float(correct) / len(y_gold), correct, len(y_gold))
def gold_evaluation(sentences, extractor, gazetteer, model_cls, model_args): logger.info('Evaluating on the gold sentences') for each in sentences: if not each.get('gold_fes'): extractor.process_sentence(each['sentence'], each['lu'], each['fes'], add_unknown=True, gazetteer=gazetteer) x_tr, y_tr = extractor.get_features(refit=True) extractor.start() tagged_gold = [] for each in sentences: if each.get('gold_fes'): tagged_gold.append((each['gold_fes'], extractor.process_sentence( each['sentence'], each['lu'], each['fes'], add_unknown=False, gazetteer=gazetteer ))) if not tagged_gold: logger.warn('asked to evaluate gold, but no gold sentences found') return x_gold, _ = extractor.get_features(refit=False) y_gold = [] for gold_fes, tagged in tagged_gold: for chunk, is_sample in tagged: if is_sample: y_gold.append([extractor.label_index[fe or 'O'] for fe in gold_fes.get(chunk, [])]) assert len(y_gold) == x_gold.shape[0] model = FeatureSelectedClassifier(model_cls, extractor.lu_column(), model_args) model.fit(x_tr, y_tr) y_pred = model.predict(x_gold) correct = len([1 for actual, predicted in zip(y_gold, y_pred) if predicted in actual]) logger.info('Gold accuracy: %f (%d / %d roles)', float(correct) / len(y_gold), correct, len(y_gold))