def loadModels(self, force=False):
self.trainModels()
# if file exists import os.path os.path.exists(file_path)
# create an NLU interpreter and dialog agent based on trained models
if self.disable_nlu != "yes":
if force or self.isNluModelModified():
self.interpreter = Interpreter.load(
"{}/".format(self.nlu_model_path),
RasaNLUConfig(self.config_file))
self.interpreter_slots = Interpreter.load(
"{}/".format(self.nlu_model_path_slots),
RasaNLUConfig(self.config_file_slots))
self.nlu_model_modified = self.getNluModelModified()
self.nlu_modified = self.getNluModified()
print('loaded nlu model')
#self.interpreter = RasaNLUInterpreter("models/nlu/default/current")
if self.disable_core != "yes":
if force or self.isCoreModelModified():
self.agentLoaded = self.agent.load(
self.core_model_path, interpreter=self.nlu_model_path)
self.core_model_modified = self.getCoreModelModified()
self.core_modified = self.getCoreModified()
self.core_domain_modified = self.getCoreDomainModified()
print('loaded core model')
def __init__(self,update_model = True):
self.dir_path = os.path.dirname(os.path.realpath(__file__))
print("/".join(self.dir_path.split("/")[0:-1]) + "/config/config.json")
config = json.load(open("/".join(self.dir_path.split("/")[0:-1]) + "/config/config.json"))
self.key = config['aws']['accessKeyId']
self.secret = config['aws']['secretAccessKey']
self.bucketname = config['aws']['bucket_name']
self.s3 = boto3.resource('s3', aws_access_key_id=self.key, aws_secret_access_key=self.secret)
self.config = config
if update_model:
#-----train a new model and save it in mdoel_directory-----------
#load training data
training_data = load_data(self.dir_path +'/data/training_data.json')
# set config and train
trainer = Trainer(RasaNLUConfig(self.dir_path + "/sample_configs/config_spacy.json"))
trainer.train(training_data)
# Returns the directory the model is stored in
self.model_directory = trainer.persist('./projects/')
# return self.model_directory
else:
self.model_directory = self.dir_path + "/projects/default/model_20171110-144019"
# load the model
self.interpreter = Interpreter.load(self.model_directory, RasaNLUConfig(self.dir_path + "/sample_configs/config_spacy.json"))
self.standard_dims = []
with open(self.dir_path + "/data/dimensions.json") as f:
data = json.load(f)
for key, value in data.items():
self.standard_dims = self.standard_dims + value
self.standard_metrics = []
with open(self.dir_path + "/data/metrics.json") as f:
data = json.load(f)
for key, value in data.items():
self.standard_metrics = self.standard_metrics + value
def train_nlu():
training_data = load_data('data/nlu.md')
conf = RasaNLUConfig("nlu_config.json")
trainer = Trainer(RasaNLUConfig("nlu_config.json"))
trainer.train(training_data)
model_directory = trainer.persist(
conf["path"],
project_name=conf["project"],
fixed_model_name=conf["fixed_model_name"])
print("persisted model to ", model_directory)
def __init__(self, model, config, builder=None, percentage=0.65):
self.model = model
self.metadata = Metadata.load(self.model)
self.config = config
self.percentage = percentage
if builder:
self.interpreter = Interpreter.load(self.metadata,
RasaNLUConfig(config), builder)
else:
self.interpreter = Interpreter.load(self.metadata,
RasaNLUConfig(config))
def train_babi_nlu():
training_data = load_data('examples/babi/data/franken_data.json')
trainer = Trainer(RasaNLUConfig("examples/babi/data/config_nlu.json"))
trainer.train(training_data)
model_directory = trainer.persist('examples/babi/models/nlu/',
model_name=model_name)
return model_directory
def train_nlu():
training_data = load_data('nlu/musicplayer.rasa.md')
trainer = Trainer(RasaNLUConfig("nlu/config.json"))
trainer.train(training_data)
model_directory = trainer.persist('nlu/model',
fixed_model_name=model_name)
return model_directory
def train_nlu(self, force=False):
if self.disable_nlu != "yes" and not self.isNluTraining:
#print("TRY NLU TRAIN {} {} {}".format(force,self.isNluModified() , self.isNluModelMissing()))
if (force or self.isNluModified() or self.isNluModelMissing()):
self.isNluTraining = True
print("NLU TRAIN {} {} {}".format(force, self.isNluModified(),
self.isNluModelMissing()))
from rasa_nlu.converters import load_data
from rasa_nlu.config import RasaNLUConfig
from rasa_nlu.model import Trainer
training_data = load_data(self.nlu_training_file)
trainer = Trainer(RasaNLUConfig(self.config_file))
trainer.train(training_data)
#model_directory = trainer.persist('models/nlu/', fixed_model_name="current")
pathParts = self.nlu_model_path.split('/')
modelName = pathParts[-1]
shortPath = "/".join(pathParts[:-2])
print("model {} path {}".format(modelName, shortPath))
model_directory = trainer.persist(shortPath,
fixed_model_name=modelName)
#self.core_model_modified=self.getCoreModelModified()
self.isNluTraining = False
self.nlu_modified = self.getNluModified()
return model_directory
def start_bot(config):
rasa_section = config['rasa']
telegram_section = config['telegram']
logger.info('Loading model')
model = get_model(rasa_section)
metadata = Metadata.load(model)
logger.info('Loading interpreter')
rasa_config = rasa_section['config']
interpreter = Interpreter.load(metadata, RasaNLUConfig(rasa_config))
logger.info('Preparing Bot')
token = telegram_section['token']
bot = telepot.DelegatorBot(token, [
pave_event_space()(per_chat_id(),
create_open,
KiririnHandler,
interpreter,
config,
timeout=20)
])
MessageLoop(bot).run_as_thread()
logger.info('Listening')
while 1:
time.sleep(10)
def run_nlu():
interpreter = Interpreter.load("./models/nlu/default/weathernlu",
RasaNLUConfig("config_spacy.json"))
print(
interpreter.parse(
u"I am planning my holiday to SriLanka. I wonder what the weather is like there"
))
def __init__(self):
self.id_obra = "0"
# CONFIGURA METAFILE
logger.info("[*] CONFIGURANDO CONFIG FILE INTENCAO")
_data_j = json.loads(codecs.open("models/intencao_config.json").read())
_data_j["path"] = real_path("models")
_data_j["mitie_file"] = real_path(
"utils/total_word_feature_extractor.dat")
_data_j["data"] = real_path("models/intencao_data.json")
with codecs.open("models/intencao_config.json", "w") as f:
json.dump(_data_j, f, indent=4)
# CONFIGURA METAFILE
logger.info("[*] CONFIGURANDO METAFILE INTENCAO")
_meta_j = json.loads(codecs.open("models/metadata.json").read())
_meta_j["mitie_file"] = real_path(
"utils/total_word_feature_extractor.dat")
with codecs.open("models/metadata.json", "w") as f:
json.dump(_meta_j, f, indent=4)
# CARREGA MODELO
logger.info("[*] CARREGANDO MODELO -> INTENCOES")
meta_data = Metadata.load(real_path("models"))
self.inteligencia = Interpreter.load(
meta_data, RasaNLUConfig(real_path("models/intencao_config.json")))
def run_nlu():
interpreter = Interpreter.load('./models/nlu/default/weathernlu',
RasaNLUConfig('config_spacy.json'))
print(
interpreter.parse(
"I am planning to go to Warangal. I wonder what is the weather out there."
))
def train():
training_data = load_data('../mom/data/nlu.json')
trainer = Trainer(RasaNLUConfig("../mom/nlu_model_config.json"))
trainer.train(training_data)
model_directory = trainer.persist(
'../models') # Returns the directory the model is stored in
return model_directory
def http_server(port_getter):
def url(port):
return "http://localhost:{0}".format(port)
# basic conf
_, nlu_log_file = tempfile.mkstemp(suffix="_rasa_nlu_logs.json")
_config = {
'write': nlu_log_file,
'port': port_getter(),
"backend": "mitie",
"path": "./",
"data": "./data/demo-restaurants.json",
"emulate": "wit"
}
config = RasaNLUConfig(cmdline_args=_config)
# run server in background
server = RasaNLUServer(config)
p = Process(target=server.start)
p.daemon = True
p.start()
# TODO: implement better way to notify when server is up
time.sleep(2)
yield url(config['port'])
p.terminate()
os.remove(nlu_log_file)
def analyse_message(self, message):
model_directory = settings.TRAINING_MODEL_QUESTION_ORIENTED
interpreter = Interpreter.load(
model_directory,
RasaNLUConfig(settings.TRAINING_CONFIGURATION_FILE))
interpreted_message = interpreter.parse(message)
return interpreted_message
def build_model(self, path):
#training_data = load_data('data/batteryBoss_train.json')
training_data = load_data(path)
trainer = Trainer(RasaNLUConfig("config/config_spacy.json"))
trainer.train(training_data)
model_directory = trainer.persist('./model/')
return model_directory
def train_classificator():
trainer = Trainer(RasaNLUConfig(CONF.get_value('nlu-config-file-path')))
training_data = load_data(CONF.get_value('nlu-training-data-path'))
trainer.train(training_data)
trainer.persist(
CONF.get_value('models-directory'),
fixed_model_name=CONF.get_value('classification-model-name'))
def train_nlu():
training_data = load_data('data/franken_data.json')
trainer = Trainer(RasaNLUConfig("nlu_model_config.json"))
trainer.train(training_data)
model_directory = trainer.persist('models/nlu/', fixed_model_name="current")
return model_directory
def test_model(self, utterance):
model_directory = './model/default/model_20180319-154224'
from rasa_nlu.model import Interpreter
interpreter = Interpreter.load(
model_directory, RasaNLUConfig("train/config_spacy.json"))
interpreter.parse(utterance)
def start_train_process(self, data, config_values):
f = tempfile.NamedTemporaryFile("w+",
suffix="_training_data.json",
delete=False)
f.write(data)
f.close()
# TODO: fix config handling
_config = self.config.as_dict()
for key, val in config_values.items():
_config[key] = val
_config["data"] = f.name
train_config = RasaNLUConfig(cmdline_args=_config)
logger.info("New training queued")
def training_callback(model_path):
self._remove_training_from_queue()
return os.path.basename(os.path.normpath(model_path))
self._add_training_to_queue()
result = self.pool.submit(do_train_in_worker, train_config)
result = deferred_from_future(result)
result.addCallback(training_callback)
return result
def chatbot(appName,trainingData):
model_directory = ""
#获取存储路径图
dataPath = app.config['NLU_SERVER_TRAINING_DATA_PATH']
#将配置生成配置文件
training_data_file = dataPath + appName + "_training.json"
trainingData = json.loads(trainingData)
with open(training_data_file, 'w') as f:
json.dump(trainingData, f)
try:
#训练模型
training_data = load_data(training_data_file)
trainer = Trainer(RasaNLUConfig(app.config['NLU_SERVER_TRAINER_PATH']),skip_validation=True)
trainer.train(training_data)
#设置模型存储路径及名称
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
fixed_model_name = appName + "_" + timestamp
model_directory = trainer.persist(app.config['NLU_MODEL_STORE_PATH'],project_name=appName,fixed_model_name=fixed_model_name)
print(model_directory)
#persist(self, path, persistor=None, project_name=None,
#fixed_model_name=None):
except Exception as e:
print("Error: 模型训练失败"+e.message)
else:
print("Success: 模型训练成功")
return model_directory
def nlu_train_models(self):
training_data = self.get_training_data()
trainer = Trainer(RasaNLUConfig(self.get_config_file()))
trainer.train(training_data)
model_directory = trainer.persist(
os.path.join(self.TRAINING_DIR, 'models'))
def test_file_config_unchanged():
file_config = {"path": "/path/to/dir"}
cmdline_args = {}
env_vars = {}
f = write_file_config(file_config)
final_config = RasaNLUConfig(f.name, env_vars, cmdline_args)
assert final_config['path'] == "/path/to/dir"
def test_cmdline_overrides_envvar():
file_config = {"path": "/path/to/dir"}
cmdline_args = {"path": "/another/path"}
env_vars = {"RASA_PATH": "/alternate/path"}
f = write_file_config(file_config)
final_config = RasaNLUConfig(f.name, env_vars, cmdline_args)
assert final_config['path'] == "/another/path"
def test_pipeline_splits_list():
file_config = {}
cmdline_args = {"pipeline": "nlp_spacy,ner_spacy"}
env_vars = {}
f = write_file_config(file_config)
final_config = RasaNLUConfig(f.name, env_vars, cmdline_args)
assert final_config['pipeline'] == ["nlp_spacy", "ner_spacy"]
def test_blank_config():
file_config = {}
cmdline_args = {}
env_vars = {}
f = write_file_config(file_config)
final_config = RasaNLUConfig(f.name, env_vars, cmdline_args)
assert final_config.as_dict() == defaults
def app(component_builder):
"""
This fixture makes use of the IResource interface of the Klein application to mock Rasa HTTP server.
:param component_builder:
:return:
"""
if "TRAVIS_BUILD_DIR" in os.environ:
root_dir = os.environ["TRAVIS_BUILD_DIR"]
else:
root_dir = os.getcwd()
_, nlu_log_file = tempfile.mkstemp(suffix="_rasa_nlu_logs.json")
_config = {
'write': nlu_log_file,
'port': -1, # unused in test app
"pipeline": "keyword",
"path": os.path.join(root_dir, "test_projects"),
"data": os.path.join(root_dir, "data/demo-restaurants.json"),
"max_training_processes": 1
}
train_models(component_builder)
config = RasaNLUConfig(cmdline_args=_config)
rasa = RasaNLU(config, component_builder, True)
return StubTreq(rasa.app.resource())
def run_nlu():
interpreter = Interpreter.load('./models/nlu/default/weathernlu',
RasaNLUConfig('config_spacy.json'))
print(
interpreter.parse(
u"I am planning my trip to Atlanta. I wonder what is the weather out there"
))
def run_nlu():
interpreter = Interpreter.load('./models/nlu/default/HRQueryNLU',
RasaNLUConfig('config_spacy.json'))
print(
interpreter.parse(
"i cant see my salary slip , i wonder how can i get it"))
print(interpreter.parse("i will get into car"))
def run_nlu():
interpreter = Interpreter.load('./models/nlu/default/weathernlu',
RasaNLUConfig('config_spacy.json'))
print(
interpreter.parse(
u"How are you? I am planning my holiday to Bercelona. I wonder what is the weather out there."
))
def retrain_using_new_info(answer, intent='OK', synonym={}):
entities = answer['entities']
print intent
# new_answer = check_answer(answer)
if intent == 'OK':
answer['intent'] = answer['intent']['name']
else:
answer['intent'] = intent
answer.pop('intent_ranking')
with open('../Codes and More/question_training_data2.json') as json_train:
train = json.load(json_train)
json_train.close()
st_ind = [
i for i in range(len(entities)) if entities[i]['entity'] == 'state'
]
if len(st_ind) > 0:
curr_state = answer['entities'][st_ind[0]]['value'].lower()
else:
curr_state = 'asdiyvls'
for s in states.keys():
t = deepcopy(answer)
t['text'] = t['text'].lower().replace(curr_state, s.lower())
t['entities'] = [{
'value': j['value'].replace(curr_state, s.lower()),
'entity': j['entity'],
'start': j['start'],
'end': j['end']
} for j in t['entities']]
train['rasa_nlu_data']['common_examples'].append(t)
if len(synonym) > 0:
train['rasa_nlu_data']['entity_synonyms'].append(synonym)
print '*** New Synonym Added'
with open('../Codes and More/question_training_data2.json',
'w') as json_data:
json.dump(train, json_data)
json_data.close()
print '*** Re-Creating the Models'
training_data = load_data('../Codes and More/question_training_data2.json')
trainer = Trainer(RasaNLUConfig("../Codes and More/config_spacy.json"))
trainer.train(training_data)
model_directory = trainer.persist('../Codes and More/',
fixed_model_name='my_model')
print '*** Building Interpreter'
interpreter = Interpreter.load(
model_directory, RasaNLUConfig("../Codes and More/config_spacy.json"))
print '--- DONE ---'
return interpreter
