def main_cross_val_metrics(): args = vars(parse_cross_val_args(sys.argv[1:])) dataset_path = args.pop("dataset_path") output_path = args.pop("output_path") def progression_handler(progress): print("%d%%" % int(progress * 100)) metrics_args = dict(dataset=dataset_path, engine_class=SnipsNLUEngine, progression_handler=progression_handler) if args.get("nb_folds") is not None: nb_folds = args.pop("nb_folds") metrics_args.update(dict(nb_folds=nb_folds)) if args.get("train_size_ratio") is not None: train_size_ratio = args.pop("train_size_ratio") metrics_args.update(dict(train_size_ratio=train_size_ratio)) include_errors = args.get("include_errors", False) with io.open(dataset_path, "r", encoding="utf-8") as f: load_resources(json.load(f)["language"]) metrics = compute_cross_val_metrics(**metrics_args) if not include_errors: metrics.pop("parsing_errors") with io.open(output_path, mode="w") as f: f.write(bytes(json.dumps(metrics), encoding="utf8").decode("utf8"))
def cross_val_metrics(dataset_path, output_path, nb_folds=5, train_size_ratio=1.0, exclude_slot_metrics=False, include_errors=False): def progression_handler(progress): print("%d%%" % int(progress * 100)) metrics_args = dict( dataset=dataset_path, engine_class=SnipsNLUEngine, progression_handler=progression_handler, nb_folds=nb_folds, train_size_ratio=train_size_ratio, include_slot_metrics=not exclude_slot_metrics, ) with Path(dataset_path).open("r", encoding="utf8") as f: load_resources(json.load(f)["language"]) from snips_nlu_metrics import compute_cross_val_metrics metrics = compute_cross_val_metrics(**metrics_args) if not include_errors: metrics.pop("parsing_errors") with Path(output_path).open(mode="w", encoding="utf8") as f: f.write(json_string(metrics))
def main_cross_val_metrics(): args = vars(parse_cross_val_args(sys.argv[1:])) dataset_path = args.pop("dataset_path") output_path = args.pop("output_path") def progression_handler(progress): print("%d%%" % int(progress * 100)) metrics_args = dict( dataset=dataset_path, engine_class=SnipsNLUEngine, progression_handler=progression_handler ) if args.get("nb_folds") is not None: nb_folds = args.pop("nb_folds") metrics_args.update(dict(nb_folds=nb_folds)) if args.get("train_size_ratio") is not None: train_size_ratio = args.pop("train_size_ratio") metrics_args.update(dict(train_size_ratio=train_size_ratio)) include_errors = args.get("include_errors", False) with io.open(dataset_path, "r", encoding="utf-8") as f: load_resources(json.load(f)["language"]) metrics = compute_cross_val_metrics(**metrics_args) if not include_errors: metrics.pop("parsing_errors") with io.open(output_path, mode="w") as f: f.write(bytes(json.dumps(metrics), encoding="utf8").decode("utf8"))
def test_pure_python_engine_performance(self): # Given dataset_path = PERFORMANCE_DATASET_PATH # When results = compute_cross_val_metrics( dataset_path, engine_class=TrainingEngine, nb_folds=5, train_size_ratio=1.0, slot_matching_lambda=_slot_matching_lambda, progression_handler=None) # Then self.check_metrics(results)
def cross_val_metrics(dataset_path, output_path, config_path=None, nb_folds=5, train_size_ratio=1.0, exclude_slot_metrics=False, include_errors=False, verbose=0): import json import logging from pathlib import Path from snips_nlu_metrics import compute_cross_val_metrics from snips_nlu import SnipsNLUEngine from snips_nlu.cli.utils import set_nlu_logger from snips_nlu.common.utils import json_string if verbose == 1: set_nlu_logger(logging.INFO) elif verbose >= 2: set_nlu_logger(logging.DEBUG) def progression_handler(progress): print("%d%%" % int(progress * 100)) if config_path is not None: with Path(config_path).open("r", encoding="utf-8") as f: config = json.load(f) engine_cls = make_engine_cls(config) else: engine_cls = SnipsNLUEngine metrics_args = dict(dataset=dataset_path, engine_class=engine_cls, progression_handler=progression_handler, nb_folds=nb_folds, train_size_ratio=train_size_ratio, include_slot_metrics=not exclude_slot_metrics, slot_matching_lambda=_match_trimmed_values) metrics = compute_cross_val_metrics(**metrics_args) if not include_errors: metrics.pop("parsing_errors") with Path(output_path).open(mode="w", encoding="utf8") as f: f.write(json_string(metrics))
def cross_val_metrics(dataset_path, output_path, config_path=None, nb_folds=5, train_size_ratio=1.0, exclude_slot_metrics=False, include_errors=False, verbose=False): if verbose: set_nlu_logger(logging.DEBUG) def progression_handler(progress): print("%d%%" % int(progress * 100)) if config_path is not None: with Path(config_path).open("r", encoding="utf-8") as f: config = json.load(f) engine_cls = make_engine_cls(config) else: engine_cls = SnipsNLUEngine metrics_args = dict( dataset=dataset_path, engine_class=engine_cls, progression_handler=progression_handler, nb_folds=nb_folds, train_size_ratio=train_size_ratio, include_slot_metrics=not exclude_slot_metrics, ) with Path(dataset_path).open("r", encoding="utf8") as f: load_resources(json.load(f)["language"]) from snips_nlu_metrics import compute_cross_val_metrics metrics = compute_cross_val_metrics(**metrics_args) if not include_errors: metrics.pop("parsing_errors") with Path(output_path).open(mode="w", encoding="utf8") as f: f.write(json_string(metrics))
def compute_sample_cross_val_metrics(): load_resources("en") return compute_cross_val_metrics(dataset=CROSS_VAL_DATASET_PATH, engine_class=SnipsNLUEngine, nb_folds=5)