def eval_exp_train(preds, part='train', postproc=None, zip_fname=None):
"""
Evaluate predictions from experiment
Converts IOB tags predicted by CRF to Brat format and then calls the official scoring function.
"""
part_dir = join(LOCAL_DIR, part)
true_iob_dir = join(part_dir, 'iob')
labels_fname = join(part_dir, part + '_labels.pkl')
labels = read_labels(labels_fname)
filenames = labels['__filenames__']
# Convert CRF prediction to IOB tags
pred_iob_dir = '_' + part + '/iob'
pred_to_iob(preds, filenames, true_iob_dir, pred_iob_dir)
if postproc:
postproc_dir = '_' + part + '/iob_pp'
postproc(pred_iob_dir, postproc_dir)
pred_iob_dir = postproc_dir
# Convert predicted IOB tags to predicted Brat annotations
txt_dir = join(DATA_DIR, part)
brat_dir = '_' + part + '/brat'
iob_to_brat(pred_iob_dir, txt_dir, brat_dir)
# Evaluate
calculateMeasures(txt_dir, brat_dir, 'rel')
if zip_fname:
package(brat_dir, part, zip_fname)
return brat_dir
splits = list(
group_k_fold.split(data['feats'], data['Material'], data['filenames']))
# Step 4: Run CRF classifier
crf = CRF(c1=0.1, c2=0.1, all_possible_transitions=True)
pred = {}
for ent in ENTITIES:
pred[ent] = cross_val_predict(crf, data['feats'], data[ent], cv=splits)
# Report scores directly on I and B tags,
# disregard 'O' because it is by far the most frequent class
print('\n' + ent + ':\n')
print(flat_classification_report(data[ent], pred[ent], digits=3,
labels=('B', 'I')))
# Step 5: Convert CRF prediction to IOB tags
pred_iob_dir = '_train/iob'
pred_to_iob(pred, data['filenames'], true_iob_dir, pred_iob_dir)
# Step 6: Convert predicted IOB tags to predicted Brat annotations
txt_dir = join(DATA_DIR, 'train')
brat_dir = '_train/brat'
iob_to_brat(pred_iob_dir, txt_dir, brat_dir)
# Step 7: Evaluate
calculateMeasures(txt_dir, brat_dir, 'rel')