def train_nlu(data=os.path.join(get_data_path(), 'court/court_nlu.md'),
config=os.path.join(get_project_dir(), 'cfg/policy_config.yml'),
model_dir=os.path.join(get_project_dir(), 'models/nlu/court')):
print(data, config, model_dir)
training_data = load_data(data)
trainer = Trainer(nlu_config.load(config))
trainer.train(training_data)
trainer.persist(model_dir)
def main():
parser = create_argument_parser()
cmdline_args = parser.parse_args()
utils.configure_colored_logging(cmdline_args.loglevel)
if cmdline_args.mode == "crossvalidation":
# TODO: move parsing into sub parser
# manual check argument dependency
if cmdline_args.model is not None:
parser.error("Crossvalidation will train a new model "
"- do not specify external model.")
if cmdline_args.config is None:
parser.error("Crossvalidation will train a new model "
"you need to specify a model configuration.")
nlu_config = config.load(cmdline_args.config)
data = training_data.load_data(cmdline_args.data)
data = drop_intents_below_freq(data, cutoff=5)
results, entity_results = run_cv_evaluation(data,
int(cmdline_args.folds),
nlu_config)
logger.info("CV evaluation (n={})".format(cmdline_args.folds))
if any(results):
logger.info("Intent evaluation results")
return_results(results.train, "train")
return_results(results.test, "test")
if any(entity_results):
logger.info("Entity evaluation results")
return_entity_results(entity_results.train, "train")
return_entity_results(entity_results.test, "test")
elif cmdline_args.mode == "evaluation":
run_evaluation(cmdline_args.data, cmdline_args.model,
cmdline_args.errors, cmdline_args.confmat,
cmdline_args.histogram)
logger.info("Finished evaluation")
def _load_default_config(path):
if path:
return config.load(path).as_dict()
else:
return {}
if url is not None:
training_data = load_data_from_url(url, cfg.language)
else:
training_data = load_data(data, cfg.language)
interpreter = trainer.train(training_data, **kwargs)
if path:
persisted_path = trainer.persist(path, persistor, project,
fixed_model_name)
else:
persisted_path = None
return trainer, interpreter, persisted_path
if __name__ == '__main__':
cmdline_args = create_argument_parser().parse_args()
utils.configure_colored_logging(cmdline_args.loglevel)
do_train(config.load(cmdline_args.config),
cmdline_args.data,
cmdline_args.path,
cmdline_args.project,
cmdline_args.fixed_model_name,
cmdline_args.storage,
url=cmdline_args.url,
num_threads=cmdline_args.num_threads)
logger.info("Finished training")
def train_nlu(data, configs, model_dir):
"""Train model."""
training_data = load_data(data)
trainer = Trainer(config.load(configs))
trainer.train(training_data)
trainer.persist(model_dir, fixed_model_name='test')