def run_evaluation(data_path, model_path,
component_builder=None): # pragma: no cover
"""Evaluate intent classification and entity extraction."""
# get the metadata config from the package data
interpreter = Interpreter.load(model_path, component_builder)
test_data = training_data.load_data(data_path,
interpreter.model_metadata.language)
extractors = get_entity_extractors(interpreter)
entity_predictions, tokens = get_entity_predictions(interpreter,
test_data)
if duckling_extractors.intersection(extractors):
entity_predictions = remove_duckling_entities(entity_predictions)
extractors = remove_duckling_extractors(extractors)
if is_intent_classifier_present(interpreter):
intent_targets = get_intent_targets(test_data)
intent_predictions = get_intent_predictions(interpreter, test_data)
logger.info("Intent evaluation results:")
evaluate_intents(intent_targets, intent_predictions)
if extractors:
entity_targets = get_entity_targets(test_data)
logger.info("Entity evaluation results:")
evaluate_entities(entity_targets, entity_predictions, tokens,
extractors)
def run_intent_evaluation(config, model_path, component_builder=None):
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
from sklearn.metrics import precision_score
from sklearn.utils.multiclass import unique_labels
# get the metadata config from the package data
test_data = load_data(config['data'])
metadata = Metadata.load(model_path)
interpreter = Interpreter.load(metadata, config, component_builder)
test_y = [e.get("intent") for e in test_data.training_examples]
preds = []
for e in test_data.training_examples:
res = interpreter.parse(e.text)
if res.get('intent'):
preds.append(res['intent'].get('name'))
else:
preds.append(None)
logger.info("Intent Evaluation Results")
logger.info("F1-Score: {}".format(f1_score(test_y, preds, average='weighted')))
logger.info("Precision: {}".format(precision_score(test_y, preds, average='weighted')))
logger.info("Accuracy: {}".format(accuracy_score(test_y, preds)))
logger.info("Classification report: \n{}".format(classification_report(test_y, preds)))
cnf_matrix = confusion_matrix(test_y, preds)
plot_intent_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, preds),
title='Intent Confusion matrix')
plt.show()
return
def run_intent_evaluation(config, model_path, component_builder=None):
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
# get the metadata config from the package data
test_data = load_data(config['data'])
interpreter = Interpreter.load(model_path, config, component_builder)
test_y = [e.get("intent") for e in test_data.training_examples]
preds = []
for e in test_data.training_examples:
res = interpreter.parse(e.text)
if res.get('intent'):
preds.append(res['intent'].get('name'))
else:
preds.append(None)
log_evaluation_table(test_y, preds)
cnf_matrix = confusion_matrix(test_y, preds)
plot_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, preds),
title='Intent Confusion matrix')
plt.show()
return
def run_evaluation(data_path, model,
report_folder=None,
successes_filename=None,
errors_filename='errors.json',
confmat_filename=None,
intent_hist_filename=None,
component_builder=None): # pragma: no cover
"""Evaluate intent classification and entity extraction."""
# get the metadata config from the package data
if isinstance(model, Interpreter):
interpreter = model
else:
interpreter = Interpreter.load(model, component_builder)
test_data = training_data.load_data(data_path,
interpreter.model_metadata.language)
extractors = get_entity_extractors(interpreter)
entity_predictions, tokens = get_entity_predictions(interpreter,
test_data)
if duckling_extractors.intersection(extractors):
entity_predictions = remove_duckling_entities(entity_predictions)
extractors = remove_duckling_extractors(extractors)
result = {
"intent_evaluation": None,
"entity_evaluation": None
}
if report_folder:
utils.create_dir(report_folder)
if is_intent_classifier_present(interpreter):
intent_targets = get_intent_targets(test_data)
intent_results = get_intent_predictions(
intent_targets, interpreter, test_data)
logger.info("Intent evaluation results:")
result['intent_evaluation'] = evaluate_intents(intent_results,
report_folder,
successes_filename,
errors_filename,
confmat_filename,
intent_hist_filename)
if extractors:
entity_targets = get_entity_targets(test_data)
logger.info("Entity evaluation results:")
result['entity_evaluation'] = evaluate_entities(entity_targets,
entity_predictions,
tokens,
extractors,
report_folder)
return result
def run_cmdline(model_path, component_builder=None):
interpreter = Interpreter.load(model_path, component_builder)
logger.info("NLU model loaded. Type a message and "
"press enter to parse it.")
while True:
text = input().strip()
r = interpreter.parse(text)
print(json.dumps(r, indent=2))
logger.info("Next message:")
def test_train_with_empty_data(language, pipeline, component_builder, tmpdir):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
trainer = Trainer(_config, component_builder)
trainer.train(TrainingData())
persistor = create_persistor(_config)
persisted_path = trainer.persist(tmpdir.strpath, persistor,
project_name="my_project")
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def __init__(self, languages):
utils.check_languages(languages)
self.languages = languages
self.config = RasaNLUConfig(cmdline_args=utils.load_config())
self.query_logger = DataRouter._create_query_logger(self.config['response_log'])
self._metadata = {}
self._interpreter = {}
for lang in languages:
self._metadata[lang] = Metadata.load(utils.get_model_dir(lang))
self._interpreter[lang] = Interpreter.load(self._metadata[lang], self.config)
self.registry = IntentRegistry(self.config['active_apps'])
def run_cv_evaluation(data, n_folds, nlu_config):
from sklearn import metrics
from sklearn.model_selection import StratifiedKFold
from collections import defaultdict
# type: (List[rasa_nlu.training_data.Message], int, RasaNLUConfig) -> Dict[Text, List[float]]
"""Stratified cross validation on data
:param data: list of rasa_nlu.training_data.Message objects
:param n_folds: integer, number of cv folds
:param nlu_config: nlu config file
:return: dictionary with key, list structure, where each entry in list
corresponds to the relevant result for one fold
"""
trainer = Trainer(nlu_config)
results = defaultdict(list)
y_true = [e.get("intent") for e in data]
skf = StratifiedKFold(n_splits=n_folds, random_state=11, shuffle=True)
counter = 1
logger.info("Evaluation started")
for train_index, test_index in skf.split(data, y_true):
train = [data[i] for i in train_index]
test = [data[i] for i in test_index]
logger.debug("Fold: {}".format(counter))
logger.debug("Training ...")
trainer.train(TrainingData(training_examples=train))
model_directory = trainer.persist("projects/") # Returns the directory the model is stored in
logger.debug("Evaluation ...")
interpreter = Interpreter.load(model_directory, nlu_config)
test_y = [e.get("intent") for e in test]
preds = []
for e in test:
res = interpreter.parse(e.text)
if res.get('intent'):
preds.append(res['intent'].get('name'))
else:
preds.append(None)
# compute fold metrics
results["Accuracy"].append(metrics.accuracy_score(test_y, preds))
results["F1-score"].append(metrics.f1_score(test_y, preds, average='weighted'))
results["Precision"] = metrics.precision_score(test_y, preds, average='weighted')
# increase fold counter
counter += 1
return dict(results)
def test_train_model_noents(language, pipeline, component_builder, tmpdir):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
(trained, _, persisted_path) = train.do_train(
_config,
path=tmpdir.strpath,
data="./data/test/demo-rasa-noents.json",
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model(pipeline_template, component_builder, tmpdir):
_config = utilities.base_test_conf(pipeline_template)
(trained, _, persisted_path) = train.do_train(
_config,
path=tmpdir.strpath,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def run_cmdline(model_path, component_builder=None):
interpreter = Interpreter.load(model_path, component_builder)
logger.info("NLU model loaded. Type a message and "
"press enter to parse it.")
while True:
text = input().strip()
if six.PY2:
# in python 2 input doesn't return unicode values
text = text.decode("utf-8")
r = interpreter.parse(text)
print(json.dumps(r, indent=2))
logger.info("Next message:")
def run_cmdline(model_path, component_builder=None):
interpreter = Interpreter.load(model_path, component_builder)
logger.info("NLU model loaded. Type a message and "
"press enter to parse it.")
while True:
text = input().strip()
if six.PY2:
# in python 2 input doesn't return unicode values
text = text.decode("utf-8")
r = interpreter.parse(text)
print(json.dumps(r, indent=2))
logger.info("Next message:")
def load_2nd_confidence():
list4 = []
interpreter = Interpreter.load('models/current/nlu_model')
for i in range(df.shape[0]):
x = interpreter.parse(df.Post[i])
v1 = {keys: values for keys, values in x.items() if keys == "intent_ranking"}
v1 = v1['intent_ranking']
nd_confidence = [d['confidence'] for d in v1 if 'confidence' in d]
nd_confidence = nd_confidence[1]
nd_confidence = str(nd_confidence)
list4.append(nd_confidence)
return (list4)
def test_train_model_multithread(language, pipeline, component_builder, tmpdir):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
(trained, _, persisted_path) = train.do_train(
_config,
path=tmpdir.strpath,
data=DEFAULT_DATA_PATH,
component_builder=component_builder,
num_threads=2)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def __init__(self, model_directory, config_file=None, lazy_init=False):
from rasa_nlu.model import Interpreter
from rasa_nlu.model import Metadata
from rasa_nlu.config import RasaNLUConfig
self.metadata = Metadata.load(model_directory)
self.lazy_init = lazy_init
self.config_file = config_file
if not lazy_init:
self.interpreter = Interpreter.load(self.metadata,
RasaNLUConfig(config_file))
else:
self.interpreter = None
def load_2nd_tag():
list3 = []
interpreter = Interpreter.load('models/current/nlu_model')
for i in range(df.shape[0]):
x = interpreter.parse(df.Post[i])
v1 = {keys: values for keys, values in x.items() if keys == "intent_ranking"}
v1 = v1['intent_ranking']
nd_tag = [d['name'] for d in v1 if 'name' in d]
nd_tag = nd_tag[1]
# nd_confidence=round(nd_confidence,2)
list3.append(nd_tag)
return (list3)
def test_train_model_on_test_pipelines(language, pipeline,
component_builder, tmpdir):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
(trained, _, persisted_path) = train.do_train(
_config,
path=tmpdir.strpath,
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
assert trained.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_random_seed(component_builder, tmpdir):
"""test if train result is the same for two runs of tf embedding"""
_config = utilities.base_test_conf("supervised_embeddings")
# set fixed random seed to 1
_config.set_component_attr(5, random_seed=1)
# first run
(trained_a, _,
persisted_path_a) = train(_config,
path=tmpdir.strpath + "_a",
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
# second run
(trained_b, _,
persisted_path_b) = train(_config,
path=tmpdir.strpath + "_b",
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
loaded_a = Interpreter.load(persisted_path_a, component_builder)
loaded_b = Interpreter.load(persisted_path_b, component_builder)
result_a = loaded_a.parse("hello")["intent"]["confidence"]
result_b = loaded_b.parse("hello")["intent"]["confidence"]
assert result_a == result_b
def get_tags(p):
# where `model_directory points to the folder the model is persisted in`
interpreter = Interpreter.load('models/smallset/nlu_model')
result = interpreter.parse(p)
# print(result['intent']['name'])
# no prediction
if result['intent']['name'] == "android" and result['intent_ranking'][1][
'name'] == "swing":
# print ("no tag")
return "NAN"
else:
print(result)
return result['intent']['name']
def reply_my_friend(msg):
ans = agent.handle_message(msg.text)
print(ans)
interpreter = Interpreter.load('models/current/nlu_model')
parser = interpreter.parse(msg.text)
intent = parser["intent"]["name"]
print(intent)
if intent == "ask_historical_plot":
msg.reply_image("1.jpg")
return ans[0]['text']
def __init__(self, models_path=models_path, reg_rule=reg_rule):
if os.path.exists(models_path):
models_name = os.listdir(models_path)
if models_name:
model_name = models_name[0]
if re.match(reg_rule, model_name):
model_path = models_path + '/' + model_name
self.interpreter = Interpreter.load(model_path)
else:
self.train()
else:
self.train()
else:
self.train()
def run_model(model_directory, text):
interpreter = Interpreter.load(model_directory)
return (interpreter.parse(text))
# if __name__=='__main__':
# dt_path = './data/examples/rasa/demo-rasa.json'
# con_path = './sample_configs/config_spacy.yml'
# model_dir = './projects/'
# text = 'looking for a chinese restaurant'
# model_directory = train_model(dt_path, con_path, model_dir)
# resp = run_model(model_directory, text)
# print(resp)
def train_nlu():
from rasa_nlu.converters import load_data
from rasa_nlu.config import RasaNLUConfig
from rasa_nlu.model import Trainer
training_data = load_data('data/demo_rasa.json')
trainer = Trainer(RasaNLUConfig("nlu_model_config.json"))
trainer.train(training_data)
model_directory = trainer.persist('models/nlu/', fixed_model_name="current")
from rasa_nlu.model import Metadata, Interpreter
interpreter = Interpreter.load(train_nlu())
json = interpreter.parse("hello")
print(json)
def rasa_base(request):
if request.method == "POST":
user = request.POST.get('user')
training_data = load_data('main/Bank_Data.json')
trainer = Trainer(config.load('main/config_spacy.yml'))
trainer.train(training_data)
model_directory = trainer.persist('main/')
interpreter = Interpreter.load(model_directory)
print(user)
output = interpreter.parse(str(user))
else:
return render(request, "CHUG/rasa.html")
return render(request, 'CHUG/rasa.html', {'data': output})
def test_random_seed(component_builder, tmpdir):
"""test if train result is the same for two runs of tf embedding"""
_config = utilities.base_test_conf("supervised_embeddings")
# set fixed random seed to 1
_config.set_component_attr(5, random_seed=1)
# first run
(trained_a, _, persisted_path_a) = train(
_config,
path=tmpdir.strpath + "_a",
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
# second run
(trained_b, _, persisted_path_b) = train(
_config,
path=tmpdir.strpath + "_b",
data=DEFAULT_DATA_PATH,
component_builder=component_builder)
loaded_a = Interpreter.load(persisted_path_a, component_builder)
loaded_b = Interpreter.load(persisted_path_b, component_builder)
result_a = loaded_a.parse("hello")["intent"]["confidence"]
result_b = loaded_b.parse("hello")["intent"]["confidence"]
assert result_a == result_b
def run(self):
# Setup interpreter
self.interpreter = Interpreter.load(self.model_path)
# Setup topics for incoming/outgoing messages
self.pub = rospy.Publisher(self.topic_output,
String,
latch=True,
queue_size=1)
self.sub = rospy.Subscriber(self.topic_input_text, String,
self.process_msg)
# Wait until killed
rospy.spin()
def test_entity():
#load model
interpreter = Interpreter.load(model_location)
duckling_extractors = {"ner_duckling", "ner_duckling_http"}
#create dictionary of entity results
entity_results = defaultdict(lambda: defaultdict(list))
#get extractors of the interpreter
extractors = evaluate.get_entity_extractors(interpreter)
#get entity predictions and tokens
entity_predictions, tokens = evaluate.get_entity_predictions(
interpreter, testing_data)
# Create classification report
if duckling_extractors.intersection(extractors):
entity_predictions = evaluate.remove_duckling_entities(
entity_predictions)
extractors = evaluate.remove_duckling_extractors(extractors)
if not extractors:
return entity_results
#get entity_targets
entity_targets = evaluate.get_entity_targets(testing_data)
#get aligned_prections
aligned_predictions = evaluate.align_all_entity_predictions(
entity_targets, entity_predictions, tokens, extractors)
merged_targets = evaluate.merge_labels(aligned_predictions)
merged_targets = evaluate.substitute_labels(merged_targets, "O",
"no_entity")
for extractor in extractors:
merged_predictions = evaluate.merge_labels(aligned_predictions,
extractor)
merged_predictions = evaluate.substitute_labels(
merged_predictions, "O", "no_entity")
report, precision, f1, accuracy = evaluate.get_evaluation_metrics(
merged_targets, merged_predictions)
entity_results[extractor]["Accuracy"].append(accuracy)
entity_results[extractor]["F1-score"].append(f1)
entity_results[extractor]["Precision"].append(precision)
print("entity_results: {}\n".format(entity_results),
"Classification report: \n{}".format(report))
def run_evaluation(config, model_path, component_builder=None): # pragma: no cover
"""Evaluate intent classification and entity extraction."""
# get the metadata config from the package data
test_data = training_data.load_data(config['data'], config['language'])
interpreter = Interpreter.load(model_path, config, component_builder)
intent_targets, entity_targets = get_targets(test_data)
intent_predictions, entity_predictions, tokens = get_predictions(interpreter, test_data)
extractors = get_entity_extractors(interpreter)
if extractors.intersection(duckling_extractors):
entity_predictions = patch_duckling_entities(entity_predictions)
extractors = patch_duckling_extractors(interpreter, extractors)
evaluate_intents(intent_targets, intent_predictions)
evaluate_entities(entity_targets, entity_predictions, tokens, extractors)
def call_for(message):
# from datetime import datetime
# s = datetime.now()
from rasa_nlu.model import Interpreter
interpreter = Interpreter.load('rasa_nlu_api/default/model')
# from random import choice
# from json import loads
import spacy
nlp = spacy.load('./en')
parsed_sentence = interpreter.parse(message)
return parsed_sentence
# print(datetime.now()-s)
# print(call_for('f**k off')['intent']['name'])
def NER(is_train):
if is_train:
training_data = load_data('./data/ner/bert_ner_train.json')
config_file = './sample_configs/config_bert_ner.yml'
ModelConfig = config.load(config_file)
trainer = Trainer(ModelConfig)
interpreter = trainer.train(training_data)
else:
model_directory = './models/rasa_bert/ner_demo'
interpreter = Interpreter.load(model_directory)
query = "这是中国领导人首次在哈佛大学发表演讲。"
while query != "Stop":
print(interpreter.parse(query))
query = input("input query: (insert Stop to close)\n")
print('Ner close')
def en_spell_checker_model(is_train):
if is_train:
training_data = load_data('./data/examples/rasa/demo-rasa.json')
config_file = './sample_configs/config_bert_spell_checker_en.yml'
ModelConfig = config.load(config_file)
trainer = Trainer(ModelConfig)
interpreter = trainer.train(training_data)
else:
model_directory = './models/spell_checker/rasa_bert_spell_checker_en'
interpreter = Interpreter.load(model_directory)
query = "How old aer you?"
while query != "Stop":
print(interpreter.parse(query))
query = input("input query: (insert Stop to close)\n")
print('spell_checker close')
def sentiment_analyzer(is_train):
if is_train:
training_data = load_data('./data/sentiment_analyzer/trainset.json')
config_file = './sample_configs/config_bert_sentiment.yml'
ModelConfig = config.load(config_file)
trainer = Trainer(ModelConfig)
interpreter = trainer.train(training_data)
else:
model_directory = './models/sentiment/sentiment_demo'
interpreter = Interpreter.load(model_directory)
query = "今天好开心呀"
while query != "Stop":
print(interpreter.parse(query))
query = input("input query: (insert Stop to close)\n")
print('sentiment_analyzer close')
def intent_pooled_model(is_train):
if is_train:
training_data = load_data('./data/nlpcc_intent/rasa_nlpcc_train.json')
config_file = './sample_configs/config_bert_intent_classifier_pooled.yml'
ModelConfig = config.load(config_file)
trainer = Trainer(ModelConfig)
interpreter = trainer.train(training_data)
else:
model_directory = './models/rasa_bert/nlpcc_pooled'
interpreter = Interpreter.load(model_directory)
query = "播放一首歌"
while query != "Stop":
print(interpreter.parse(query))
query = input("input query: (insert Stop to close)\n")
print('intent classifier close')
def __init__(self, save_sm_state=False, **kwargs):
ScenarioStateBase.__init__(self,
'retrieve_information',
save_sm_state=save_sm_state,
outcomes=['succeeded', 'failed'],
input_keys=['command'])
self.number_of_retries = kwargs.get('number_of_retries', 0)
self.timeout = kwargs.get('timeout', 120.)
self.threshold = kwargs.get('threshold', 0.68)
# Load rasa nlu model
model_directory = get_package_path('rasa_nlu_models', 'common',
'mdr_hri_behaviours_models', 'nlu')
self.interpreter = Interpreter.load(model_directory)
def generate_rasa_training_data_and_interpreter(self, bot_directories,
module_id):
training_data_dir = bot_directories[
"training_data_directory"] + module_id + ".json"
config_file_dir = bot_directories[
"training_data_directory"] + 'config_spacy.yaml'
model_dir = bot_directories["model_directory"]
training_data = load_data(training_data_dir)
trainer = Trainer(config.load(config_file_dir))
# train the model and save it to a folder
trainer.train(training_data)
model_directory = trainer.persist(model_dir)
print("trained model for module '" + module_id + "'")
rasa_interpreter = Interpreter.load(model_directory)
return training_data, rasa_interpreter
def run_evaluation(data_path, model,
errors_filename='errors.json',
confmat_filename=None,
intent_hist_filename=None,
component_builder=None): # pragma: no cover
"""Evaluate intent classification and entity extraction."""
# get the metadata config from the package data
if isinstance(model, Interpreter):
interpreter = model
else:
interpreter = Interpreter.load(model, component_builder)
test_data = training_data.load_data(data_path,
interpreter.model_metadata.language)
extractors = get_entity_extractors(interpreter)
entity_predictions, tokens = get_entity_predictions(interpreter,
test_data)
if duckling_extractors.intersection(extractors):
entity_predictions = remove_duckling_entities(entity_predictions)
extractors = remove_duckling_extractors(extractors)
result = {
"intent_evaluation": None,
"entity_evaluation": None
}
if is_intent_classifier_present(interpreter):
intent_targets = get_intent_targets(test_data)
intent_results = get_intent_predictions(
intent_targets, interpreter, test_data)
logger.info("Intent evaluation results:")
result['intent_evaluation'] = evaluate_intents(intent_results,
errors_filename,
confmat_filename,
intent_hist_filename)
if extractors:
entity_targets = get_entity_targets(test_data)
logger.info("Entity evaluation results:")
result['entity_evaluation'] = evaluate_entities(entity_targets,
entity_predictions,
tokens,
extractors)
return result
def run_cv_evaluation(data, n_folds, nlu_config):
# type: (List[rasa_nlu.training_data.Message], int, RasaNLUConfig) -> Dict[Text, List[float]]
"""Stratified cross validation on data
:param data: list of rasa_nlu.training_data.Message objects
:param n_folds: integer, number of cv folds
:param nlu_config: nlu config file
:return: dictionary with key, list structure, where each entry in list
corresponds to the relevant result for one fold
"""
from sklearn.model_selection import StratifiedKFold
from collections import defaultdict
trainer = Trainer(nlu_config)
train_results = defaultdict(list)
test_results = defaultdict(list)
y_true = [e.get("intent") for e in data]
skf = StratifiedKFold(n_splits=n_folds, random_state=11, shuffle=True)
counter = 1
logger.info("Evaluation started")
for train_index, test_index in skf.split(data, y_true):
train = [data[i] for i in train_index]
test = [data[i] for i in test_index]
logger.debug("Fold: {}".format(counter))
logger.debug("Training ...")
trainer.train(TrainingData(training_examples=train))
model_directory = trainer.persist(
"projects/") # Returns the directory the model is stored in
logger.debug("Evaluation ...")
interpreter = Interpreter.load(model_directory, nlu_config)
# calculate train accuracy
compute_metrics(interpreter, train, train_results)
# calculate test accuracy
compute_metrics(interpreter, test, test_results)
# increase fold counter
counter += 1
Results = namedtuple('Results', 'train test')
results = Results(dict(train_results), dict(test_results))
return results
def test_intent():
#load model
interpreter = Interpreter.load(model_location)
# get true target of the testing data
targets = evaluate.get_intent_targets(testing_data)
# get predictions of the testing data
predictions = evaluate.get_intent_predictions(interpreter, testing_data)
#create a confusion matrix and summary statistics for intent predictions
evaluate.evaluate_intents(targets, predictions)
#generate classification report, precision, f1 score and accuary
report, precision, f1, accuracy = evaluate.get_evaluation_metrics(
targets, predictions)
print("F1-Score: {}\n".format(f1), "Precision: {}\n".format(precision),
"Accuracy: {}\n".format(accuracy),
"Classification report: \n{}".format(report))
def __init__(self):
# Dict containing the entity and their function
self.intent_functions = {
'arena_where_located': self.arena_where_located,
'arena_count_object': self.arena_count_object,
'crowd_count': self.crowd_count,
'crowd_person_gesture': self.crowd_person_gesture,
'arena_color_object': self.arena_color_object,
'arena_which_object': self.arena_which_object,
'object_category': self.object_category,
'object_adjective': self.object_adjective
}
self.personType = {
"girl": "female",
"girls": "female",
"women": "female",
"woman": "female",
"females": "female",
"female": "female",
"men": "male",
"male": "male",
"males": "male",
"boys": "male",
"boy": "male",
"man": "male",
"people": ""
}
sys.stderr = open(os.devnull, 'w')
# Load the config files
print("Loading config files...")
rospack = rospkg.RosPack()
training_data = load_data(rospack.get_path('wm_nlu')+"/script/robocup_spr.json")
trainer = Trainer(config.load(rospack.get_path('wm_nlu')+"/script/config_spacy.yml"))
print("Training the model...")
# Train the model based on the robocup_spr.json file
trainer.train(training_data)
# Returns the directory the model is stored in
model_directory = trainer.persist(rospack.get_path('wm_nlu')+'/script/default_spr/')
print("Loading the model...")
self.interpreter = Interpreter.load(model_directory)
print("RasaNLU init done.")
def retrain_using_new_info(answer, intent='OK', synonym={}):
entities = answer['entities']
print intent
# new_answer = check_answer(answer)
if intent == 'OK':
answer['intent'] = answer['intent']['name']
else:
answer['intent'] = intent
answer.pop('intent_ranking')
with open('../Codes and More/question_training_data2.json') as json_train:
train = json.load(json_train)
json_train.close()
st_ind = [
i for i in range(len(entities)) if entities[i]['entity'] == 'state'
]
if len(st_ind) > 0:
curr_state = answer['entities'][st_ind[0]]['value'].lower()
else:
curr_state = 'asdiyvls'
for s in states.keys():
t = deepcopy(answer)
t['text'] = t['text'].lower().replace(curr_state, s.lower())
t['entities'] = [{
'value': j['value'].replace(curr_state, s.lower()),
'entity': j['entity'],
'start': j['start'],
'end': j['end']
} for j in t['entities']]
train['rasa_nlu_data']['common_examples'].append(t)
if len(synonym) > 0:
train['rasa_nlu_data']['entity_synonyms'].append(synonym)
print '*** New Synonym Added'
with open('../Codes and More/question_training_data2.json',
'w') as json_data:
json.dump(train, json_data)
json_data.close()
print '*** Re-Creating the Models'
training_data = load_data('../Codes and More/question_training_data2.json')
trainer = Trainer(RasaNLUConfig("../Codes and More/config_spacy.json"))
trainer.train(training_data)
model_directory = trainer.persist('../Codes and More/',
fixed_model_name='my_model')
print '*** Building Interpreter'
interpreter = Interpreter.load(
model_directory, RasaNLUConfig("../Codes and More/config_spacy.json"))
print '--- DONE ---'
return interpreter
def query(self, usr):
#print("User typed : " + usr)
if self.nlu_model == None:
self.nlu_model = Interpreter.load(self.model_directory)
self.populate_dataframe()
try:
response_json = self.nlu_model.parse(usr)
entities = response_json["entities"]
intent = response_json['intent']['name']
if intent == 'query':
return self.process_query_intent(entities)
else:
return self.process_other_intents(intent)
except Exception as e:
print(e)
return "Could not follow your question [" + usr + "], Try again"
def load_interpreter_for_model(config, persisted_path, component_builder):
def read_model_metadata(model_dir, config):
if model_dir is None:
data = Project._default_model_metadata()
return Metadata(data, model_dir)
else:
if not os.path.isabs(model_dir):
model_dir = os.path.join(config['path'], model_dir)
# download model from S3 if needed
if not os.path.isdir(model_dir):
Project._load_model_from_cloud(model_dir, config)
return Metadata.load(model_dir)
metadata = read_model_metadata(persisted_path, config)
return Interpreter.create(metadata, config, component_builder)
def create_interpreter(self, meta, config):
context = {"model_dir": meta.model_dir}
model_config = dict(config.items())
model_config.update(meta.metadata)
pipeline = []
for component_name in meta.pipeline:
try:
component = self.__get_component(component_name, meta, context, model_config)
rasa_nlu.components.init_component(component, context, model_config)
pipeline.append(component)
except rasa_nlu.components.MissingArgumentError as e:
raise Exception("Failed to initialize component '{}'. {}".format(component_name, e.message))
return Interpreter(pipeline, context, model_config)
def __create_model_store(self):
# Fallback for users that specified the model path as a string and hence only want a single default model.
if type(self.config.server_model_dirs) is Text:
model_dict = {self.DEFAULT_MODEL_NAME: self.config.server_model_dirs}
elif self.config.server_model_dirs is None:
model_dict = self.__search_for_models()
else:
model_dict = self.config.server_model_dirs
model_store = {}
for alias, model_path in list(model_dict.items()):
try:
logger.info("Loading model '{}'...".format(model_path))
model_store[alias] = self.__interpreter_for_model(model_path)
except Exception as e:
logger.exception("Failed to load model '{}'. Error: {}".format(model_path, e))
if not model_store:
meta = Metadata({"pipeline": ["intent_classifier_keyword"]}, "")
interpreter = Interpreter.load(meta, self.config, self.component_builder)
model_store[self.DEFAULT_MODEL_NAME] = interpreter
return model_store
def _interpreter_for_model(self, model_name):
metadata = self._read_model_metadata(model_name)
return Interpreter.create(metadata, self._config,
self._component_builder)
def _fallback_model(self):
meta = Metadata({"pipeline": ["intent_classifier_keyword"]}, "")
return Interpreter.create(meta, self._config, self._component_builder)
def load_interpreter_for_model(config, persisted_path, component_builder):
metadata = DataRouter.read_model_metadata(persisted_path, config)
return Interpreter.load(metadata, config, component_builder)
def interpreter_for(component_builder, data, path, config):
(trained, _, path) = train(config, data, path,
component_builder=component_builder)
interpreter = Interpreter.load(path, component_builder)
return interpreter
def test_model_not_compatible(metadata):
with pytest.raises(rasa_nlu.model.UnsupportedModelError):
Interpreter.ensure_model_compatibility(metadata)
def test_model_is_compatible(metadata):
# should not raise an exception
assert Interpreter.ensure_model_compatibility(metadata) is None
def _load_interpreter(self):
from rasa_nlu.model import Interpreter
self.interpreter = Interpreter.load(self.model_directory)
def _fallback_model(self):
meta = Metadata({"pipeline": [{
"name": "KeywordIntentClassifier",
"class": utils.module_path_from_object(KeywordIntentClassifier())
}]}, "")
return Interpreter.create(meta, self._component_builder)
def __init__(self, model_dir):
from rasa_nlu.model import Metadata, Interpreter
self.interpreter = Interpreter.load(model_dir, RasaNLUConfig("mobile_nlu_model_config.json"))
self._items = {}
def __interpreter_for_model(self, model_path):
metadata = DataRouter.read_model_metadata(model_path, self.config)
return Interpreter.load(metadata, self.config, self.component_builder)
#!/usr/bin/env python
from rasa_nlu.model import Interpreter
import json
interpreter = Interpreter.load("./models/current/nlu")
def print_intent(msg):
result = interpreter.parse(msg)
print(json.dumps(result, indent=2))
print_intent("Can you book me a dental checkup appointment for 10'o clock")
print_intent("I need an appointment for eye checkup at 10'o clock")
