def test_remove_model_invalid(empty_model_dir): test_file = "something.else" test_content = "Some other stuff" test_file_path = os.path.join(empty_model_dir, test_file) write_to_file(test_file_path, test_content) with pytest.raises(ValueError) as e: remove_model(empty_model_dir) os.remove(test_file_path)
def test_remove_model_invalid(empty_model_dir): test_file = "something.else" test_content = "Some other stuff" test_file_path = os.path.join(empty_model_dir, test_file) write_to_file(test_file_path, test_content) with pytest.raises(ValueError) as e: remove_model(empty_model_dir) os.remove(test_file_path)
def run_cv_evaluation(td, n_folds, nlu_config): # type: (TrainingData, int, RasaNLUConfig) -> CVEvaluationResult """Stratified cross validation on data :param td: Training Data :param n_folds: integer, number of cv folds :param nlu_config: nlu config file :return: dictionary with key, list structure, where each entry in list corresponds to the relevant result for one fold """ from collections import defaultdict import tempfile trainer = Trainer(nlu_config) train_results = defaultdict(list) test_results = defaultdict(list) entity_train_results = defaultdict(lambda: defaultdict(list)) entity_test_results = defaultdict(lambda: defaultdict(list)) tmp_dir = tempfile.mkdtemp() for train, test in generate_folds(n_folds, td): trainer.train(train) model_dir = trainer.persist(tmp_dir) interpreter = Interpreter.load(model_dir, nlu_config) # calculate train accuracy train_results = combine_intent_result(train_results, interpreter, train) test_results = combine_intent_result(test_results, interpreter, test) # calculate test accuracy entity_train_results = combine_entity_result(entity_train_results, interpreter, train) entity_test_results = combine_entity_result(entity_test_results, interpreter, test) utils.remove_model(model_dir) os.rmdir(os.path.join(tmp_dir, "default")) os.rmdir(tmp_dir) return (CVEvaluationResult(dict(train_results), dict(test_results)), CVEvaluationResult(dict(entity_train_results), dict(entity_test_results)))
def test_remove_model_with_files(empty_model_dir): metadata_file = "metadata.json" metadata_content = {"pipeline": "spacy_sklearn", "language": "en"} metadata_path = os.path.join(empty_model_dir, metadata_file) write_json_to_file(metadata_path, metadata_content) fake_obj = {"Fake", "model"} fake_obj_path = os.path.join(empty_model_dir, "component.pkl") with io.open(fake_obj_path, "wb") as f: pickle.dump(fake_obj, f) assert remove_model(empty_model_dir)
def test_remove_model_with_files(empty_model_dir): metadata_file = "metadata.json" metadata_content = {"pipeline": "spacy_sklearn", "language": "en"} metadata_path = os.path.join(empty_model_dir, metadata_file) write_json_to_file(metadata_path, metadata_content) fake_obj = {"Fake", "model"} fake_obj_path = os.path.join(empty_model_dir, "component.pkl") with io.open(fake_obj_path, "wb") as f: pickle.dump(fake_obj, f) assert remove_model(empty_model_dir)
def run_cv_evaluation(td, n_folds, nlu_config): # type: (TrainingData, int, RasaNLUConfig) -> CVEvaluationResult """Stratified cross validation on data :param td: Training Data :param n_folds: integer, number of cv folds :param nlu_config: nlu config file :return: dictionary with key, list structure, where each entry in list corresponds to the relevant result for one fold """ from sklearn import metrics from collections import defaultdict import tempfile trainer = Trainer(nlu_config) train_results = defaultdict(list) test_results = defaultdict(list) tmp_dir = tempfile.mkdtemp() for train, test in generate_folds(n_folds, td): trainer.train(TrainingData(training_examples=train, entity_synonyms=td.entity_synonyms, regex_features=td.regex_features)) model_dir = trainer.persist(tmp_dir) interpreter = Interpreter.load(model_dir, nlu_config) # calculate train accuracy compute_metrics(interpreter, train, train_results) # calculate test accuracy compute_metrics(interpreter, test, test_results) utils.remove_model(model_dir) os.rmdir(os.path.join(tmp_dir, "default")) os.rmdir(tmp_dir) return CVEvaluationResult(dict(train_results), dict(test_results))
def test_remove_model_empty(empty_model_dir): assert remove_model(empty_model_dir)
def test_remove_model_empty(empty_model_dir): assert remove_model(empty_model_dir)