def train_test_metrics(train_dataset_path,
test_dataset_path,
output_path,
config_path=None,
exclude_slot_metrics=False,
include_errors=False,
verbose=False):
if verbose:
set_nlu_logger(logging.DEBUG)
if config_path is not None:
with Path(config_path).open("r", encoding="utf-8") as f:
config = json.load(f)
engine_cls = make_engine_cls(config)
else:
engine_cls = SnipsNLUEngine
metrics_args = dict(train_dataset=train_dataset_path,
test_dataset=test_dataset_path,
engine_class=engine_cls,
include_slot_metrics=not exclude_slot_metrics)
with Path(train_dataset_path).open("r", encoding="utf8") as f:
load_resources(json.load(f)["language"])
from snips_nlu_metrics import compute_train_test_metrics
metrics = compute_train_test_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with Path(output_path).open(mode="w", encoding="utf8") as f:
f.write(json_string(metrics))
def main_train_test_metrics():
args = vars(parse_train_test_args(sys.argv[1:]))
train_dataset_path = args.pop("train_dataset_path")
test_dataset_path = args.pop("test_dataset_path")
output_path = args.pop("output_path")
exclude_slot_metrics = args.get("exclude_slot_metrics", False)
metrics_args = dict(
train_dataset=train_dataset_path,
test_dataset=test_dataset_path,
engine_class=SnipsNLUEngine,
include_slot_metrics=not exclude_slot_metrics
)
include_errors = args.get("include_errors", False)
with io.open(train_dataset_path, "r", encoding="utf-8") as f:
load_resources(json.load(f)["language"])
from snips_nlu_metrics import compute_train_test_metrics
metrics = compute_train_test_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with io.open(output_path, mode="w") as f:
json_dump = json.dumps(metrics, sort_keys=True, indent=2)
f.write(bytes(json_dump, encoding="utf8").decode("utf8"))
def main_train_test_metrics():
args = vars(parse_train_test_args(sys.argv[1:]))
train_dataset_path = args.pop("train_dataset_path")
test_dataset_path = args.pop("test_dataset_path")
output_path = args.pop("output_path")
exclude_slot_metrics = args.get("exclude_slot_metrics", False)
metrics_args = dict(
train_dataset=train_dataset_path,
test_dataset=test_dataset_path,
engine_class=SnipsNLUEngine,
include_slot_metrics=not exclude_slot_metrics
)
include_errors = args.get("include_errors", False)
with io.open(train_dataset_path, "r", encoding="utf-8") as f:
load_resources(json.load(f)["language"])
metrics = compute_train_test_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with io.open(output_path, mode="w") as f:
f.write(bytes(json.dumps(metrics), encoding="utf8").decode("utf8"))
def train_eval_snips_nlu_model(lang='en', cross=False, save=''):
""" Train snips data from all brat annotation object
:param lang: abbreviate language name
:param save: path where model will be save
:return: None
:rtype: None
"""
from snips_nlu import SnipsNLUEngine
from snips_nlu.default_configs import CONFIG_EN
from snips_nlu_metrics import compute_train_test_metrics, compute_cross_val_metrics
import pickle
import json
if cross:
train_data_obj = BuildSnipsDataTask1(lang, cross=cross, vers=save)
train_data = train_data_obj.build_snips_data_task1()
print("--> Evaluating training data with Snips metrics...")
filename_results = source_result / "snips_semeval_2020_evaluation_task1_{}.pkl".format(save)
if not Path(filename_results).exists():
tt_metrics = compute_train_test_metrics(train_dataset=train_data[0],
test_dataset=train_data[1],
engine_class=SnipsNLUEngine,
include_slot_metrics=False)
#print(tt_metrics)
if not Path(filename_results).exists():
print("--> Writing snips nlu metrics data to file...")
with codecs.open(filename_results, 'wb') as metric:
pickle.dump(tt_metrics, metric)
from datetime import datetime
dmtime = "_{}_{}".format(save, datetime.now().strftime("%Y%m%d-%H%M%S"))
name = "snips_semeval_2020_evaluation_task1{}.json".format(dmtime)
filename_results_json = source_result / name
with codecs.open(filename_results_json, 'w', "utf-8") as m_json:
json.dump(tt_metrics, m_json)
else:
filename_results = source_result / "snips_semeval_2020_model_task1_{}".format(save)
train_data_obj = BuildSnipsDataTask1(lang, cross=cross, vers=save)
train_data = train_data_obj.build_snips_data_task1()
nlu_engine = SnipsNLUEngine(config=CONFIG_EN)
print("--> Training patent data with Snips...")
nlu_engine.fit(train_data)
try:
print("--> Saving model trained with Snips (JOBLIB)...")
filename_joblib = source_result / "snips_semeval_2020_model_task1_{}.pkl".format(save)
with codecs.open(filename_joblib, 'wb') as metric:
pickle.dump(nlu_engine, metric)
except: pass
print("--> Saving model trained with Snips (SNIPS)...")
try: nlu_engine.persist(filename_results)
except: pass
def train_test_metrics(train_dataset_path,
test_dataset_path,
output_path,
exclude_slot_metrics=False,
include_errors=False):
metrics_args = dict(train_dataset=train_dataset_path,
test_dataset=test_dataset_path,
engine_class=SnipsNLUEngine,
include_slot_metrics=not exclude_slot_metrics)
with Path(train_dataset_path).open("r", encoding="utf8") as f:
load_resources(json.load(f)["language"])
from snips_nlu_metrics import compute_train_test_metrics
metrics = compute_train_test_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with Path(output_path).open(mode="w", encoding="utf8") as f:
f.write(json_string(metrics))
def train_test_metrics(train_dataset_path,
test_dataset_path,
output_path,
config_path=None,
exclude_slot_metrics=False,
include_errors=False,
verbosity=0):
import json
import logging
from pathlib import Path
from snips_nlu_metrics import compute_train_test_metrics
from snips_nlu import SnipsNLUEngine
from snips_nlu.cli.utils import set_nlu_logger
from snips_nlu.common.utils import json_string
if verbosity == 1:
set_nlu_logger(logging.INFO)
elif verbosity >= 2:
set_nlu_logger(logging.DEBUG)
if config_path is not None:
with Path(config_path).open("r", encoding="utf-8") as f:
config = json.load(f)
engine_cls = make_engine_cls(config)
else:
engine_cls = SnipsNLUEngine
metrics_args = dict(train_dataset=train_dataset_path,
test_dataset=test_dataset_path,
engine_class=engine_cls,
include_slot_metrics=not exclude_slot_metrics,
slot_matching_lambda=_match_trimmed_values)
metrics = compute_train_test_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with Path(output_path).open(mode="w", encoding="utf8") as f:
f.write(json_string(metrics))
def main_train_test_metrics():
args = vars(parse_train_test_args(sys.argv[1:]))
train_dataset_path = args.pop("train_dataset_path")
test_dataset_path = args.pop("test_dataset_path")
output_path = args.pop("output_path")
metrics_args = dict(
train_dataset=train_dataset_path,
test_dataset=test_dataset_path,
engine_class=SnipsNLUEngine
)
include_errors = args.get("include_errors", False)
with io.open(train_dataset_path, "r", encoding="utf-8") as f:
load_resources(json.load(f)["language"])
metrics = compute_train_test_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with io.open(output_path, mode="w") as f:
f.write(bytes(json.dumps(metrics), encoding="utf8").decode("utf8"))
from automatic_data_generation.utils.conversion import csv2json
datadir = os.path.join(*args.train_path.split('/')[:-1])
csv2json(datadir, datadir, augmented=False)
csv2json(datadir, datadir, augmented=True)
print('Starting benchmarking...')
def my_matching_lambda(lhs_slot, rhs_slot):
return lhs_slot['text'].strip() == rhs_slot["rawValue"].strip()
raw_metrics = compute_train_test_metrics(
train_dataset="data/train.json",
test_dataset="data/validate.json",
engine_class=SnipsNLUEngine,
slot_matching_lambda=my_matching_lambda)
augmented_metrics = compute_train_test_metrics(
train_dataset="data/train_augmented.json",
test_dataset="data/validate.json",
engine_class=SnipsNLUEngine,
slot_matching_lambda=my_matching_lambda)
print('----------METRICS----------')
print('Without augmentation : ')
print(raw_metrics['average_metrics'])
print('With augmentation : ')
print(augmented_metrics['average_metrics'])
intent_improvement = 100 * (
(augmented_metrics['average_metrics']['intent']['f1'] -
def compute_sample_train_test_metrics():
load_resources("en")
return compute_train_test_metrics(train_dataset=TRAIN_DATASET_PATH,
test_dataset=TEST_DATASET_PATH,
engine_class=SnipsNLUEngine)
