def eval_exp_train(preds, part='train', postproc=None, zip_fname=None):
"""
Evaluate predictions from experiment
Converts IOB tags predicted by CRF to Brat format and then calls the official scoring function.
"""
part_dir = join(LOCAL_DIR, part)
true_iob_dir = join(part_dir, 'iob')
labels_fname = join(part_dir, part + '_labels.pkl')
labels = read_labels(labels_fname)
filenames = labels['__filenames__']
# Convert CRF prediction to IOB tags
pred_iob_dir = '_' + part + '/iob'
pred_to_iob(preds, filenames, true_iob_dir, pred_iob_dir)
if postproc:
postproc_dir = '_' + part + '/iob_pp'
postproc(pred_iob_dir, postproc_dir)
pred_iob_dir = postproc_dir
# Convert predicted IOB tags to predicted Brat annotations
txt_dir = join(DATA_DIR, part)
brat_dir = '_' + part + '/brat'
iob_to_brat(pred_iob_dir, txt_dir, brat_dir)
# Evaluate
calculateMeasures(txt_dir, brat_dir, 'rel')
if zip_fname:
package(brat_dir, part, zip_fname)
return brat_dir
splits = list(
group_k_fold.split(data['feats'], data['Material'], data['filenames']))
# Step 4: Run CRF classifier
crf = CRF(c1=0.1, c2=0.1, all_possible_transitions=True)
pred = {}
for ent in ENTITIES:
pred[ent] = cross_val_predict(crf, data['feats'], data[ent], cv=splits)
# Report scores directly on I and B tags,
# disregard 'O' because it is by far the most frequent class
print('\n' + ent + ':\n')
print(flat_classification_report(data[ent], pred[ent], digits=3,
labels=('B', 'I')))
# Step 5: Convert CRF prediction to IOB tags
pred_iob_dir = '_train/iob'
pred_to_iob(pred, data['filenames'], true_iob_dir, pred_iob_dir)
# Step 6: Convert predicted IOB tags to predicted Brat annotations
txt_dir = join(DATA_DIR, 'train')
brat_dir = '_train/brat'
iob_to_brat(pred_iob_dir, txt_dir, brat_dir)
# Step 7: Evaluate
calculateMeasures(txt_dir, brat_dir, 'rel')
labeled as Process. Unless the token string is a substring of a larger token string already labeled as Process (e.g. "chemical reaction enhancement"). We take the majority label. If there is a draw, then we skip it. Reads IOB files and writes new IOB files. """ from os.path import join from eval import calculateMeasures from sie import EXPS_DIR, DATA_DIR from sie.brat import iob_to_brat from sie.postproc import postproc_labels in_iob_dir = join(EXPS_DIR, 'best/_train/iob') #in_iob_dir = join(EXPS_DIR, 'prune/_train/iob') out_iob_dir = '_train/iob' postproc_labels(in_iob_dir, out_iob_dir) # Step 6: Convert predicted IOB tags to predicted Brat annotations txt_dir = join(DATA_DIR, 'train') brat_dir = '_train/brat' iob_to_brat(out_iob_dir, txt_dir, brat_dir) # Step 7: Evaluate calculateMeasures(txt_dir, brat_dir, 'rel')
parser.add_argument('material_dir',
help='directory containing tab-delimited files with predicted IOB tags for label "Material" in 3rd column')
parser.add_argument('process_dir',
help='directory containing tab-delimited files with predicted IOB tags for label "Process" in 3rd column')
parser.add_argument('task_dir',
help='directory containing tab-delimited files with predicted IOB tags for label "Task" in 3rd column')
parser.add_argument('pred_iob_dir',
help='directory for writing json files with predicted IOB tags')
parser.add_argument('pred_brat_dir',
help='directory for writing predicted Brat annotation files')
args = parser.parse_args()
# Step 1: Convert CFR++ output to IOB tags in Json format
crfplus_dirs = {
'Material': args.material_dir,
'Process': args.process_dir,
'Task': args.task_dir
}
convert(crfplus_dirs, args.true_iob_dir, args.pred_iob_dir)
# Step 2: Convert predicted IOB tags to predicted Brat annotations
iob_to_brat(args.pred_iob_dir, args.true_brat_dir, args.pred_brat_dir)
# Step 3: Evaluate
calculateMeasures(args.true_brat_dir, args.pred_brat_dir, 'rel')
"""
Test of performance loss due to use of IOB scheme
Converts the derived IOB tags for train/dev data back to Brat annotation format
(.ann files) and then use the evaluation script to compare it to the original
Brat annotation files.
If the conversion was perfect, scores would be perfect.
However, scores are lower because some annotation spans cannot be aligned
to the tokens resulting from Spacy.
Also, entities embedded in entities of the same type (e.g. a Material text span
containing another Material text span) can not be represented in an IOB schema.
"""
from os.path import join
from sie import DATA_DIR, LOCAL_DIR
from sie.brat import iob_to_brat
from eval import calculateMeasures
for part in 'train', 'dev':
iob_dir = join(LOCAL_DIR, part, 'iob')
txt_dir = join(DATA_DIR, part)
brat_dir = join('_brat', part)
iob_to_brat(iob_dir, txt_dir, brat_dir)
print('\nScores for {} part:\n'.format(part))
calculateMeasures(txt_dir, brat_dir, 'rel')
indent=4,
sort_keys=True,
ensure_ascii=False)
except Exception as err:
print('*** ERRROR **', err)
print(crfplus_fname)
print(line)
print()
# Step 1: Convert CFR++ output to IOB tags in Json format
true_iob_dir = join(LOCAL_DIR, 'train/iob')
pred_iob_dir = '_entityOp_Utpal/iob'
crfplus_dirs = {
'Material': '_entityOp_Utpal/materialOp',
'Process': '_entityOp_Utpal/processOp',
'Task': '_entityOp_Utpal/taskOp'
}
convert(crfplus_dirs, true_iob_dir, pred_iob_dir)
# Step 2: Convert predicted IOB tags to predicted Brat annotations
true_brat_dir = join(DATA_DIR, 'train')
pred_brat_dir = '_entityOp_Utpal/brat'
iob_to_brat(pred_iob_dir, true_brat_dir, pred_brat_dir)
# Step 3: Evaluate
calculateMeasures(true_brat_dir, pred_brat_dir, 'rel')