def model_stats(dataset, spacy_model, label=None, isPrf=False):
"""
Evaluate model accuracy of model based on dataset with no training
inspired from https://support.prodi.gy/t/evaluating-precision-and-recall-of-ner/193/2
found on https://support.prodi.gy/t/evaluating-precision-and-recall-of-ner/193
got basic model evaluation by looking at the batch-train recipe
"""
log("RECIPE: Starting recipe ner.stats", locals())
DB = connect()
nlp = spacy.load(spacy_model)
isPrf = 'True'
if (isPrf):
examples = gold_to_spacy(dataset, spacy_model)
score = evaluate_prf(nlp, examples)
print("Precision {:0.4f}\tRecall {:0.4f}\tF-score {:0.4f}".format(
score['ents_p'], score['ents_r'], score['ents_f']))
else:
# ripped this from ner.batch-train recipe
model = EntityRecognizer(nlp, label=label)
evaldoc = merge_spans(DB.get_dataset(dataset))
evals = list(split_sentences(model.orig_nlp, evaldoc))
scores = model.evaluate(evals)
print(
"Accuracy {:0.4f}\tRight {:0.0f}\tWrong {:0.0f}\tUnknown {:0.0f}\tEntities {:0.0f}"
.format(scores['acc'], scores['right'], scores['wrong'],
scores['unk'], scores['ents']))
else:
ner = nlp.get_pipe("ner")
for label in all_labels:
ner.add_label(label)
random.shuffle(examples)
train_examples, evals, eval_split = split_evals(
merged_examples, eval_split)
st.success(
f"✅ Using **{len(train_examples)}** training examples "
f"and **{len(evals)}** evaluation examples with "
f"**{len(all_labels)}** label(s)")
annot_model = EntityRecognizer(nlp,
label=all_labels,
no_missing=no_missing)
batch_size = guess_batch_size(len(train_examples))
baseline = annot_model.evaluate(evals)
st.info(
f"ℹ️ **Baseline**\n**{baseline['right']:.0f}** right "
f"entities, **{baseline['wrong']:.0f}** wrong entities, "
f"**{baseline['unk']:.0f}** unkown entities, "
f"**{baseline['ents']:.0f}** total predicted, "
f"**{baseline['acc']:.2f}** accuracy")
progress = st.progress(0)
results = []
result_table = st.empty()
best_acc = 0.0
for i in range(n_iter):
random.shuffle(train_examples)
losses = annot_model.batch_train(
train_examples,
batch_size=batch_size,