def test_remove_model_invalid(empty_model_dir):
test_file = "something.else"
test_content = "Some other stuff"
test_file_path = os.path.join(empty_model_dir, test_file)
write_to_file(test_file_path, test_content)
with pytest.raises(ValueError) as e:
remove_model(empty_model_dir)
os.remove(test_file_path)
def test_remove_model_invalid(empty_model_dir):
test_file = "something.else"
test_content = "Some other stuff"
test_file_path = os.path.join(empty_model_dir, test_file)
write_to_file(test_file_path, test_content)
with pytest.raises(ValueError) as e:
remove_model(empty_model_dir)
os.remove(test_file_path)
def run_cv_evaluation(td, n_folds, nlu_config):
# type: (TrainingData, int, RasaNLUConfig) -> CVEvaluationResult
"""Stratified cross validation on data
:param td: Training Data
:param n_folds: integer, number of cv folds
:param nlu_config: nlu config file
:return: dictionary with key, list structure, where each entry in list
corresponds to the relevant result for one fold
"""
from collections import defaultdict
import tempfile
trainer = Trainer(nlu_config)
train_results = defaultdict(list)
test_results = defaultdict(list)
entity_train_results = defaultdict(lambda: defaultdict(list))
entity_test_results = defaultdict(lambda: defaultdict(list))
tmp_dir = tempfile.mkdtemp()
for train, test in generate_folds(n_folds, td):
trainer.train(train)
model_dir = trainer.persist(tmp_dir)
interpreter = Interpreter.load(model_dir, nlu_config)
# calculate train accuracy
train_results = combine_intent_result(train_results, interpreter, train)
test_results = combine_intent_result(test_results, interpreter, test)
# calculate test accuracy
entity_train_results = combine_entity_result(entity_train_results,
interpreter, train)
entity_test_results = combine_entity_result(entity_test_results,
interpreter, test)
utils.remove_model(model_dir)
os.rmdir(os.path.join(tmp_dir, "default"))
os.rmdir(tmp_dir)
return (CVEvaluationResult(dict(train_results), dict(test_results)),
CVEvaluationResult(dict(entity_train_results),
dict(entity_test_results)))
def test_remove_model_with_files(empty_model_dir):
metadata_file = "metadata.json"
metadata_content = {"pipeline": "spacy_sklearn", "language": "en"}
metadata_path = os.path.join(empty_model_dir, metadata_file)
write_json_to_file(metadata_path, metadata_content)
fake_obj = {"Fake", "model"}
fake_obj_path = os.path.join(empty_model_dir, "component.pkl")
with io.open(fake_obj_path, "wb") as f:
pickle.dump(fake_obj, f)
assert remove_model(empty_model_dir)
def test_remove_model_with_files(empty_model_dir):
metadata_file = "metadata.json"
metadata_content = {"pipeline": "spacy_sklearn", "language": "en"}
metadata_path = os.path.join(empty_model_dir, metadata_file)
write_json_to_file(metadata_path, metadata_content)
fake_obj = {"Fake", "model"}
fake_obj_path = os.path.join(empty_model_dir, "component.pkl")
with io.open(fake_obj_path, "wb") as f:
pickle.dump(fake_obj, f)
assert remove_model(empty_model_dir)
def run_cv_evaluation(td, n_folds, nlu_config):
# type: (TrainingData, int, RasaNLUConfig) -> CVEvaluationResult
"""Stratified cross validation on data
:param td: Training Data
:param n_folds: integer, number of cv folds
:param nlu_config: nlu config file
:return: dictionary with key, list structure, where each entry in list
corresponds to the relevant result for one fold
"""
from sklearn import metrics
from collections import defaultdict
import tempfile
trainer = Trainer(nlu_config)
train_results = defaultdict(list)
test_results = defaultdict(list)
tmp_dir = tempfile.mkdtemp()
for train, test in generate_folds(n_folds, td):
trainer.train(TrainingData(training_examples=train,
entity_synonyms=td.entity_synonyms,
regex_features=td.regex_features))
model_dir = trainer.persist(tmp_dir)
interpreter = Interpreter.load(model_dir, nlu_config)
# calculate train accuracy
compute_metrics(interpreter, train, train_results)
# calculate test accuracy
compute_metrics(interpreter, test, test_results)
utils.remove_model(model_dir)
os.rmdir(os.path.join(tmp_dir, "default"))
os.rmdir(tmp_dir)
return CVEvaluationResult(dict(train_results), dict(test_results))
def test_remove_model_empty(empty_model_dir):
assert remove_model(empty_model_dir)
def test_remove_model_empty(empty_model_dir):
assert remove_model(empty_model_dir)
