def main_cross_val_metrics():
args = vars(parse_cross_val_args(sys.argv[1:]))
dataset_path = args.pop("dataset_path")
output_path = args.pop("output_path")
def progression_handler(progress):
print("%d%%" % int(progress * 100))
metrics_args = dict(dataset=dataset_path,
engine_class=SnipsNLUEngine,
progression_handler=progression_handler)
if args.get("nb_folds") is not None:
nb_folds = args.pop("nb_folds")
metrics_args.update(dict(nb_folds=nb_folds))
if args.get("train_size_ratio") is not None:
train_size_ratio = args.pop("train_size_ratio")
metrics_args.update(dict(train_size_ratio=train_size_ratio))
include_errors = args.get("include_errors", False)
with io.open(dataset_path, "r", encoding="utf-8") as f:
load_resources(json.load(f)["language"])
metrics = compute_cross_val_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with io.open(output_path, mode="w") as f:
f.write(bytes(json.dumps(metrics), encoding="utf8").decode("utf8"))
def cross_val_metrics(dataset_path,
output_path,
nb_folds=5,
train_size_ratio=1.0,
exclude_slot_metrics=False,
include_errors=False):
def progression_handler(progress):
print("%d%%" % int(progress * 100))
metrics_args = dict(
dataset=dataset_path,
engine_class=SnipsNLUEngine,
progression_handler=progression_handler,
nb_folds=nb_folds,
train_size_ratio=train_size_ratio,
include_slot_metrics=not exclude_slot_metrics,
)
with Path(dataset_path).open("r", encoding="utf8") as f:
load_resources(json.load(f)["language"])
from snips_nlu_metrics import compute_cross_val_metrics
metrics = compute_cross_val_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with Path(output_path).open(mode="w", encoding="utf8") as f:
f.write(json_string(metrics))
def main_cross_val_metrics():
args = vars(parse_cross_val_args(sys.argv[1:]))
dataset_path = args.pop("dataset_path")
output_path = args.pop("output_path")
def progression_handler(progress):
print("%d%%" % int(progress * 100))
metrics_args = dict(
dataset=dataset_path,
engine_class=SnipsNLUEngine,
progression_handler=progression_handler
)
if args.get("nb_folds") is not None:
nb_folds = args.pop("nb_folds")
metrics_args.update(dict(nb_folds=nb_folds))
if args.get("train_size_ratio") is not None:
train_size_ratio = args.pop("train_size_ratio")
metrics_args.update(dict(train_size_ratio=train_size_ratio))
include_errors = args.get("include_errors", False)
with io.open(dataset_path, "r", encoding="utf-8") as f:
load_resources(json.load(f)["language"])
metrics = compute_cross_val_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with io.open(output_path, mode="w") as f:
f.write(bytes(json.dumps(metrics), encoding="utf8").decode("utf8"))
def test_pure_python_engine_performance(self):
# Given
dataset_path = PERFORMANCE_DATASET_PATH
# When
results = compute_cross_val_metrics(
dataset_path,
engine_class=TrainingEngine,
nb_folds=5,
train_size_ratio=1.0,
slot_matching_lambda=_slot_matching_lambda,
progression_handler=None)
# Then
self.check_metrics(results)
def cross_val_metrics(dataset_path,
output_path,
config_path=None,
nb_folds=5,
train_size_ratio=1.0,
exclude_slot_metrics=False,
include_errors=False,
verbose=0):
import json
import logging
from pathlib import Path
from snips_nlu_metrics import compute_cross_val_metrics
from snips_nlu import SnipsNLUEngine
from snips_nlu.cli.utils import set_nlu_logger
from snips_nlu.common.utils import json_string
if verbose == 1:
set_nlu_logger(logging.INFO)
elif verbose >= 2:
set_nlu_logger(logging.DEBUG)
def progression_handler(progress):
print("%d%%" % int(progress * 100))
if config_path is not None:
with Path(config_path).open("r", encoding="utf-8") as f:
config = json.load(f)
engine_cls = make_engine_cls(config)
else:
engine_cls = SnipsNLUEngine
metrics_args = dict(dataset=dataset_path,
engine_class=engine_cls,
progression_handler=progression_handler,
nb_folds=nb_folds,
train_size_ratio=train_size_ratio,
include_slot_metrics=not exclude_slot_metrics,
slot_matching_lambda=_match_trimmed_values)
metrics = compute_cross_val_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with Path(output_path).open(mode="w", encoding="utf8") as f:
f.write(json_string(metrics))
def cross_val_metrics(dataset_path,
output_path,
config_path=None,
nb_folds=5,
train_size_ratio=1.0,
exclude_slot_metrics=False,
include_errors=False,
verbose=False):
if verbose:
set_nlu_logger(logging.DEBUG)
def progression_handler(progress):
print("%d%%" % int(progress * 100))
if config_path is not None:
with Path(config_path).open("r", encoding="utf-8") as f:
config = json.load(f)
engine_cls = make_engine_cls(config)
else:
engine_cls = SnipsNLUEngine
metrics_args = dict(
dataset=dataset_path,
engine_class=engine_cls,
progression_handler=progression_handler,
nb_folds=nb_folds,
train_size_ratio=train_size_ratio,
include_slot_metrics=not exclude_slot_metrics,
)
with Path(dataset_path).open("r", encoding="utf8") as f:
load_resources(json.load(f)["language"])
from snips_nlu_metrics import compute_cross_val_metrics
metrics = compute_cross_val_metrics(**metrics_args)
if not include_errors:
metrics.pop("parsing_errors")
with Path(output_path).open(mode="w", encoding="utf8") as f:
f.write(json_string(metrics))
def compute_sample_cross_val_metrics():
load_resources("en")
return compute_cross_val_metrics(dataset=CROSS_VAL_DATASET_PATH,
engine_class=SnipsNLUEngine,
nb_folds=5)