def train_intent_recognizer(example_files,
rasa_config,
project_dir,
project_name,
num_threads=4):
import rasa_nlu
from rasa_nlu.train import do_train
# Write training examples out to single file
with tempfile.NamedTemporaryFile(suffix='.md', mode='w+') as train_file:
for example_path in example_files:
if not os.path.exists(example_path):
continue
# Copy contents
with open(example_path, 'r') as example_file:
for line in example_file:
print(line, file=train_file)
# Back to beginining
train_file.seek(0)
# Run the actual training
do_train(cfg=rasa_nlu.config.load(rasa_config),
data=train_file.name,
path=project_dir,
project=project_name,
num_threads=num_threads)
def train_intent_recognizer(config):
import rasa_nlu
from rasa_nlu.train import do_train
training_cfg = config['training']
# Write training examples out to single file
train_path = training_cfg['user_examples']
with open(train_path, 'w+') as train_file:
for example_path in training_cfg['example_files']:
if not os.path.exists(example_path):
continue
# Copy contents
with open(example_path, 'r') as example_file:
for line in example_file:
print(line, file=train_file)
# Back to beginining
train_file.seek(0)
logging.info('Training intent recognizer')
do_train(cfg=rasa_nlu.config.load(training_cfg['intent_config']),
data=train_path,
path=training_cfg['intent_project_path'],
project=training_cfg['intent_project'],
num_threads=4)
return train_path
def test_train_model_empty_pipeline(component_builder):
# Should return an empty pipeline
_config = utilities.base_test_conf(pipeline_template=None)
with pytest.raises(ValueError):
train.do_train(_config,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
def test_train_model_empty_pipeline(component_builder):
# Should return an empty pipeline
_config = utilities.base_test_conf(pipeline_template=None)
with pytest.raises(ValueError):
train.do_train(
_config,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
def test_handles_pipeline_with_non_existing_component(component_builder):
_config = utilities.base_test_conf("spacy_sklearn")
_config.pipeline.append({"name": "my_made_up_component"})
with pytest.raises(Exception) as execinfo:
train.do_train(_config,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert "Failed to find component" in str(execinfo.value)
def test_handles_pipeline_with_non_existing_component(component_builder):
_config = utilities.base_test_conf("spacy_sklearn")
_config.pipeline.append({"name": "my_made_up_component"})
with pytest.raises(Exception) as execinfo:
train.do_train(
_config,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert "Failed to find component" in str(execinfo.value)
def train_intent_recognizer(examples_file, rasa_config,
project_dir, project_name,
num_threads=4):
import rasa_nlu
from rasa_nlu.train import do_train
# Run the actual training
do_train(cfg=rasa_nlu.config.load(rasa_config),
data=examples_file,
path=project_dir,
project=project_name,
num_threads=num_threads)
def interpreter_for(component_builder, data, path, config):
(trained, _, path) = do_train(config,
data,
path,
component_builder=component_builder)
interpreter = Interpreter.load(path, component_builder)
return interpreter
def test_train_featurizer():
(trained, _, _) = train.do_train(
config.load('sample_configs/sample_use_featurizer.yml'),
data='data/examples/dialogflow',
path='models',
project='current',
fixed_model_name='use-featurizer')
assert trained.pipeline
def test_train_model_noents(language, pipeline, component_builder, tmpdir):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
(trained, _, persisted_path) = train.do_train(
_config,
path=tmpdir.strpath,
data="./data/test/demo-rasa-noents.json",
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model(pipeline_template, component_builder, tmpdir):
_config = utilities.base_test_conf(pipeline_template)
(trained, _,
persisted_path) = train.do_train(_config,
path=tmpdir.strpath,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model_noents(language, pipeline, component_builder, tmpdir):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
(trained, _,
persisted_path) = train.do_train(_config,
path=tmpdir.strpath,
data="./data/test/demo-rasa-noents.json",
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_named_model(component_builder, tmpdir):
_config = utilities.base_test_conf("keyword")
(trained, _, persisted_path) = train.do_train(
_config,
path=tmpdir.strpath,
project="my_keyword_model",
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert trained.pipeline
normalized_path = os.path.dirname(os.path.normpath(persisted_path))
# should be saved in a dir named after a project
assert os.path.basename(normalized_path) == "my_keyword_model"
def test_train_named_model(component_builder, tmpdir):
_config = utilities.base_test_conf("keyword")
(trained, _,
persisted_path) = train.do_train(_config,
path=tmpdir.strpath,
project="my_keyword_model",
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert trained.pipeline
normalized_path = os.path.dirname(os.path.normpath(persisted_path))
# should be saved in a dir named after a project
assert os.path.basename(normalized_path) == "my_keyword_model"
def test_train_model(pipeline_template, component_builder, tmpdir):
_config = utilities.base_test_conf(pipeline_template)
(trained, _, persisted_path) = train.do_train(
_config,
path=tmpdir.strpath,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model_noents(component_builder, tmpdir):
_config = utilities.base_test_conf("all_components")
(trained, _, persisted_path) = train.do_train(
_config,
path=tmpdir.strpath,
data="./data/test/demo-rasa-noents.json",
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model_on_test_pipelines(language, pipeline, component_builder,
tmpdir):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
(trained, _,
persisted_path) = train.do_train(_config,
path=tmpdir.strpath,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model_on_test_pipelines(language, pipeline,
component_builder, tmpdir):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
(trained, _, persisted_path) = train.do_train(
_config,
path=tmpdir.strpath,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_random_seed(component_builder, tmpdir):
'''test if train result is the same for two runs of tf embedding'''
_config = utilities.base_test_conf("tensorflow_embedding")
# set fixed random seed to 1
_config.set_component_attr("intent_classifier_tensorflow_embedding", random_seed=1)
# first run
(trained_a, _, persisted_path_a) = train.do_train(
_config,
path=tmpdir.strpath + "_a",
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
# second run
(trained_b, _, persisted_path_b) = train.do_train(
_config,
path=tmpdir.strpath + "_b",
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
loaded_a = Interpreter.load(persisted_path_a, component_builder)
loaded_b = Interpreter.load(persisted_path_b, component_builder)
result_a = loaded_a.parse("hello")["intent"]["confidence"]
result_b = loaded_b.parse("hello")["intent"]["confidence"]
assert result_a == result_b
def interpreter_for(component_builder, data, path, config):
(trained, _, path) = do_train(config, data, path,
component_builder=component_builder)
interpreter = Interpreter.load(path, component_builder)
return interpreter
def run_train(config, component_builder):
(trained, _, path) = do_train(config, component_builder)
return trained, path
def run_train(_config):
config = RasaNLUConfig(cmdline_args=_config)
(trained, path) = do_train(config)
return trained, path
from __future__ import unicode_literals
import logging
from rasa_nlu import config
from rasa_nlu.config import RasaNLUModelConfig
from rasa_nlu.train import do_train
from rasa_nlu.components import ComponentBuilder
if __name__ == '__main__':
logging.basicConfig(level='INFO')
builder = ComponentBuilder(use_cache=False)
do_train(cfg=config.load("./nlu_config.yml"),
data='data/nlu_data.md',
path='models',
fixed_model_name="nlu",
project='current',
component_builder=builder)
# train_dialogue_model(
# domain_file='domain.yml',
# stories_file='data/stories.md',
# output_path='./models/dialogue/',
# max_history=10,
# nlu_model_path='./data/nlu_data.md',
# kwargs={
# "fixed_model_name": "current",
# "epochs":500,
# "max_training_samples":300
# }
#
def run_train(config):
(trained, path) = do_train(config)
return trained, path
def run_train(_config):
config = RasaNLUConfig(cmdline_args=_config)
do_train(config)
def run_train(config, component_builder):
(trained, _, path) = do_train(config, component_builder)
return trained, path