def _create_ensemble(policies):
if policies is None:
return SimplePolicyEnsemble([MemoizationPolicy])
if isinstance(policies, list):
return SimplePolicyEnsemble(policies)
elif isinstance(policies, PolicyEnsemble):
return policies
else:
passed_type = type(policies).__name__
raise ValueError(
"Invalid param `policies`. Passed object is "
"of type '{}', but should be policy, an array of "
"policies, or a policy ensemble".format(passed_type))
def _create_ensemble(policies):
# type: (Union[List[Policy], PolicyEnsemble, None]) -> PolicyEnsemble
if policies is None:
return SimplePolicyEnsemble([])
if isinstance(policies, list):
logger.info("it is a list!")
return SimplePolicyEnsemble(policies)
elif isinstance(policies, PolicyEnsemble):
logger.info("it is a policy ensemble!")
return policies
else:
passed_type = type(policies).__name__
raise ValueError(
"Invalid param `policies`. Passed object is "
"of type '{}', but should be policy, an array of "
"policies, or a policy ensemble".format(passed_type))
def default_processor(default_domain):
ensemble = SimplePolicyEnsemble([ScoringPolicy()])
interpreter = RegexInterpreter()
PolicyTrainer(ensemble, default_domain,
BinaryFeaturizer()).train(DEFAULT_STORIES_FILE,
max_history=3)
tracker_store = InMemoryTrackerStore(default_domain)
return MessageProcessor(interpreter, ensemble, default_domain,
tracker_store)
def _in_training_data_fraction(action_list):
"""Given a list of action items, returns the fraction of actions
that were predicted using one of the Memoization policies."""
in_training_data = [
a["action"] for a in action_list
if not SimplePolicyEnsemble.is_not_memo_policy(a["policy"])
]
return len(in_training_data) / len(action_list)
def default_processor(default_domain, default_nlg):
agent = Agent(default_domain,
SimplePolicyEnsemble([AugmentedMemoizationPolicy()]),
interpreter=RegexInterpreter())
training_data = agent.load_data(DEFAULT_STORIES_FILE)
agent.train(training_data)
tracker_store = InMemoryTrackerStore(default_domain)
return MessageProcessor(agent.interpreter, agent.policy_ensemble,
default_domain, tracker_store, default_nlg)
def test_message_processor(default_domain, capsys):
story_filename = "data/dsl_stories/stories_defaultdomain.md"
ensemble = SimplePolicyEnsemble([ScoringPolicy()])
interpreter = RegexInterpreter()
PolicyTrainer(ensemble, default_domain,
BinaryFeaturizer()).train(story_filename, max_history=3)
tracker_store = InMemoryTrackerStore(default_domain)
processor = MessageProcessor(interpreter, ensemble, default_domain,
tracker_store)
processor.handle_message(UserMessage("_greet", ConsoleOutputChannel()))
out, _ = capsys.readouterr()
assert "hey there!" in out
def test_message_processor(default_domain, capsys):
story_filename = "data/dsl_stories/stories_defaultdomain.md"
ensemble = SimplePolicyEnsemble([ScoringPolicy()])
interpreter = RegexInterpreter()
PolicyTrainer(ensemble, default_domain,
BinaryFeaturizer()).train(story_filename, max_history=3)
tracker_store = InMemoryTrackerStore(default_domain)
processor = MessageProcessor(interpreter, ensemble, default_domain,
tracker_store)
out = CollectingOutputChannel()
processor.handle_message(UserMessage("_greet[name=Core]", out))
assert ("default", "hey there Core!") == out.latest_output()
def _create_ensemble(
policies # type: Union[List[Policy], PolicyEnsemble, None]
):
# type: (...) -> Optional[PolicyEnsemble]
if policies is None:
return None
if isinstance(policies, list):
return SimplePolicyEnsemble(policies)
elif isinstance(policies, PolicyEnsemble):
return policies
else:
passed_type = type(policies).__name__
raise ValueError(
"Invalid param `policies`. Passed object is "
"of type '{}', but should be policy, an array of "
"policies, or a policy ensemble".format(passed_type))
def test_policy_priority():
domain = Domain.load("data/test_domains/default.yml")
tracker = DialogueStateTracker.from_events("test", [UserUttered("hi")], [])
priority_1 = ConstantPolicy(priority=1, predict_index=0)
priority_2 = ConstantPolicy(priority=2, predict_index=1)
policy_ensemble_0 = SimplePolicyEnsemble([priority_1, priority_2])
policy_ensemble_1 = SimplePolicyEnsemble([priority_2, priority_1])
priority_2_result = priority_2.predict_action_probabilities(
tracker, domain)
i = 1 # index of priority_2 in ensemble_0
result, best_policy = policy_ensemble_0.probabilities_using_best_policy(
tracker, domain)
assert best_policy == 'policy_{}_{}'.format(i, type(priority_2).__name__)
assert (result.tolist() == priority_2_result)
i = 0 # index of priority_2 in ensemble_1
result, best_policy = policy_ensemble_1.probabilities_using_best_policy(
tracker, domain)
assert best_policy == 'policy_{}_{}'.format(i, type(priority_2).__name__)
assert (result.tolist() == priority_2_result)
def create_app(env_json_file,
loglevel="INFO", # type: Optional[Text]
logfile="rasa_core.log", # type: Optional[Text]
cors_origins=None, # type: Optional[List[Text]]
action_factory=None, # type: Optional[Text]
auth_token=None, # type: Optional[Text]
tracker_store=None # type: Optional[TrackerStore]
):
"""Class representing a Rasa Core HTTP server."""
app = Flask(__name__)
CORS(app, resources={r"/*": {"origins": "*"}})
# Setting up logfile
utils.configure_file_logging(loglevel, logfile)
if not cors_origins:
cors_origins = []
with open(env_json_file) as f:
env_json = json.load(f)
domain_file = env_json['domain']
interpreter_file = env_json['interpreter_dir']
path_to_glove = env_json['path_to_glove']
tracker_store = tracker_store
action_factory = action_factory
# this needs to be an array, so we can modify it in the nested functions...
domain = OntologyDomain.load(os.path.join(path, domain_file),
None)
from rasa_core.featurizers import FloatSingleStateFeaturizer,MaxHistoryTrackerFeaturizer
feat=MaxHistoryTrackerFeaturizer(FloatSingleStateFeaturizer(),max_history=1)
policy = idare.KerasIDarePolicy(feat)
from rasa_core.policies.ensemble import SimplePolicyEnsemble
ensemble = SimplePolicyEnsemble([policy])
ensemble = ensemble.load(env_json['dst_model_dir'])
# In[5]:
from rasa_core.agent import Agent
from rasa_core.interpreter import NaturalLanguageInterpreter
from rasa_core.tracker_store import InMemoryTrackerStore
_interpreter = Interpreter(suprath_dir = interpreter_file, rasa_dir = interpreter_file,
path_to_glove = path_to_glove)
logger.info("NLU interpreter loaded successfully")
_tracker_store = Agent.create_tracker_store(None,domain,env_json)
_agent = [Agent(domain, ensemble, _interpreter, _tracker_store,env_json)]
global processor
feedback_logger = sql_feedback_logger
processor = _agent[0]._create_processor(feedback_logger=feedback_logger)
usr_channel = None
ha_channel = None
teams_channel = None
def agent():
if _agent and _agent[0]:
return _agent[0]
else:
return None
@app.route("/",
methods=['GET', 'OPTIONS'])
@cross_origin(origins=cors_origins)
def hello():
"""Check if the server is running and responds with the version."""
return "hello from Rasa Core: " + __version__
@app.route("/version",
methods=['GET', 'OPTIONS'])
@cross_origin(origins=cors_origins)
def version():
"""respond with the version number of the installed rasa core."""
return jsonify({'version': __version__})
# <sender_id> can be be 'default' if there's only 1 client
@app.route("/run",
methods=['POST', 'OPTIONS'])
@cross_origin(origins=cors_origins)
@requires_auth(auth_token)
@ensure_loaded_agent(agent)
def continue_predicting():
"""Continue a prediction started with parse.
Caller should have executed the action returned from the parse
endpoint. The events returned from that executed action are
passed to continue which will trigger the next action prediction.
If continue predicts action listen, the caller should wait for the
next user message."""
from rasa_core.run import create_input_channel
from rasa_core.channels.custom_websocket import CustomInput
input_component1=CustomInput(None)
input_component2=CustomInput(None)
from rasa_core.channels.websocket import WebSocketInputChannel
global usr_channel
global ha_channel
try:
usr_channel = WebSocketInputChannel(int(env_json["userchannelport"]),None,input_component1,http_ip='0.0.0.0')
botf_input_channel = BotFrameworkInput(
app_id=env_json['teams_app_id'],
app_password=env_json['teams_app_password']
)
teams_channel = HttpInputChannel(int(env_json['userchannelport2']),'/webhooks/botframework',botf_input_channel)
except OSError as e:
logger.error(str(e))
return str(e)
usr_channel.output_channel = input_component1.output_channel
teams_channel.output_channel = botf_input_channel.output_channel
op_agent = agent()
op_agent.handle_custom_processor([usr_channel,teams_channel],usr_channel,processor)
return "ok"
@app.route("/conversations/<sender_id>/tracker/events",
methods=['POST', 'OPTIONS'])
@cross_origin(origins=cors_origins)
@requires_auth(auth_token)
@ensure_loaded_agent(agent)
def append_events(sender_id):
"""Append a list of events to the state of a conversation"""
request_params = request.get_json(force=True)
evts = events.deserialise_events(request_params)
tracker = agent().tracker_store.get_or_create_tracker(sender_id)
for e in evts:
tracker.update(e)
agent().tracker_store.save(tracker)
return jsonify(tracker.current_state())
@app.route("/conversations",
methods=['GET', 'OPTIONS'])
@cross_origin(origins=cors_origins)
@requires_auth(auth_token)
@ensure_loaded_agent(agent)
def list_trackers():
return jsonify(list(agent().tracker_store.keys()))
@app.route("/conversations/<sender_id>/tracker",
methods=['GET', 'OPTIONS'])
@cross_origin(origins=cors_origins)
@requires_auth(auth_token)
@ensure_loaded_agent(agent)
def retrieve_tracker(sender_id):
"""Get a dump of a conversations tracker including its events."""
# parameters
use_history = bool_arg('ignore_restarts', default=False)
should_include_events = bool_arg('events', default=True)
until_time = request.args.get('until', None)
# retrieve tracker and set to requested state
tracker = agent().tracker_store.get_or_create_tracker(sender_id)
if until_time is not None:
tracker = tracker.travel_back_in_time(float(until_time))
# dump and return tracker
state = tracker.current_state(
should_include_events=should_include_events,
only_events_after_latest_restart=use_history)
return jsonify(state)
@app.route("/conversations/<sender_id>/tracker",
methods=['PUT', 'OPTIONS'])
@cross_origin(origins=cors_origins)
@requires_auth(auth_token)
@ensure_loaded_agent(agent)
def update_tracker(sender_id):
"""Use a list of events to set a conversations tracker to a state."""
request_params = request.get_json(force=True)
tracker = DialogueStateTracker.from_dict(sender_id,
request_params,
agent().domain)
agent().tracker_store.save(tracker)
# will override an existing tracker with the same id!
agent().tracker_store.save(tracker)
return jsonify(tracker.current_state(should_include_events=True))
@app.route("/conversations/<sender_id>/parse",
methods=['GET', 'POST', 'OPTIONS'])
@cross_origin(origins=cors_origins)
@requires_auth(auth_token)
@ensure_loaded_agent(agent)
def parse(sender_id):
request_params = request_parameters()
if 'query' in request_params:
message = request_params.pop('query')
elif 'q' in request_params:
message = request_params.pop('q')
else:
return Response(
jsonify(error="Invalid parse parameter specified."),
status=400,
mimetype="application/json")
try:
# Fetches the predicted action in a json format
response = agent().start_message_handling(message, sender_id)
return jsonify(response)
except Exception as e:
logger.exception("Caught an exception during parse.")
return Response(jsonify(error="Server failure. Error: {}"
"".format(e)),
status=500,
content_type="application/json")
@app.route("/conversations/<sender_id>/respond",
methods=['GET', 'POST', 'OPTIONS'])
@cross_origin(origins=cors_origins)
@requires_auth(auth_token)
@ensure_loaded_agent(agent)
def respond(sender_id):
request_params = request_parameters()
if 'query' in request_params:
message = request_params.pop('query')
elif 'q' in request_params:
message = request_params.pop('q')
else:
return Response(jsonify(error="Invalid respond parameter "
"specified."),
status=400,
mimetype="application/json")
try:
# Set the output channel
out = CollectingOutputChannel()
# Fetches the appropriate bot response in a json format
responses = agent().handle_message(message,
output_channel=out,
sender_id=sender_id)
return jsonify(responses)
except Exception as e:
logger.exception("Caught an exception during respond.")
return Response(jsonify(error="Server failure. Error: {}"
"".format(e)),
status=500,
content_type="application/json")
@app.route("/nlu_parse",
methods=['GET', 'OPTIONS'])
@cross_origin(origins=cors_origins)
@requires_auth(auth_token)
@ensure_loaded_agent(agent)
def nlu_parse():
request_params = request_parameters()
if 'query' in request_params:
message = request_params.get('query')
elif 'q' in request_params:
message = request_params.get('q')
else:
return Response(jsonify(error="Invalid respond parameter "
"specified."),
status=400,
mimetype="application/json")
return jsonify(_interpreter.parse(message))
@app.route("/email/<sender_id>/respond",
methods=['GET', 'POST', 'OPTIONS'])
@cross_origin(origins=cors_origins)
@requires_auth(auth_token)
@ensure_loaded_agent(agent)
def email(sender_id):
request_params = request_parameters()
if 'query' in request_params:
message = request_params.get('query')
elif 'q' in request_params:
message = request_params.get('q')
else:
return Response(jsonify(error="Invalid respond parameter "
"specified."),
status=400,
mimetype="application/json")
global teams_channel
inter_channel_mapper = read_json_file(env_json.get('inter_channel_mapper'))
teams_channel.output_channel.id_map.update(inter_channel_mapper)
#temporary code follows
teams_channel.output_channel.id_map.update({sender_id:
inter_channel_mapper[list(inter_channel_mapper.keys())[0]]})
teams_channel.output_channel.reverse_id_map.update({list(inter_channel_mapper.keys())[0]:sender_id})
#temporary code ends
email_id = request_params.get('email_id')
preprocessor = partial(idare.email_preprocessor,email_id=email_id)
try:
# Set the output channel
out = CollectingOutputChannel()
# Fetches the appropriate bot response in a json format
agent().handle_email(message, email_preprocessor = preprocessor,
output_channel=out,
alternate_channel = teams_channel,
sender_id=sender_id)
response = out.latest_output()
return jsonify(response)
except Exception as e:
logger.exception("Caught an exception during respond.")
return Response(jsonify(error="Server failure. Error: {}"
"".format(e)),
status=500,
content_type="application/json")
@app.route("/load_nlu", methods=['POST', 'OPTIONS'])
@requires_auth(auth_token)
@cross_origin(origins=cors_origins)
def load_nlu_model():
"""Loads a zipped model, replacing the existing one."""
if 'nlu_model' not in request.files:
# model file is missing
abort(400)
model_file = request.files['nlu_model']
logger.info("Received new nlu_model through REST interface.")
zipped_path = tempfile.NamedTemporaryFile(delete=False, suffix=".zip")
zipped_path.close()
model_file.save(zipped_path.name)
logger.debug("Downloaded model to {}".format(zipped_path.name))
zip_ref = zipfile.ZipFile(zipped_path.name, 'r')
zip_ref.extractall(interpreter_file)
zip_ref.close()
logger.debug("Unzipped model to {}".format(
os.path.abspath(interpreter_file)))
global processor
del processor.interpreter
_interpreter.reload()
processor.interpreter = _interpreter
agent().interpreter = _interpreter
logger.debug("Finished loading new interpreter.")
return jsonify({'success': 1})
@app.route("/load_idare", methods=['GET', 'OPTIONS'])
@requires_auth(auth_token)
@cross_origin(origins=cors_origins)
def load_idare():
"""Reload idare."""
from imp import reload
global idare
try:
idare = reload(idare)
except Exception as e:
return str(e)
return jsonify({'success': 1})
return app
# In[4]:
from rasa_core.domain import OntologyDomain
from rasa_core.envconfig import EnvConfig
import os
path = './'
domain = OntologyDomain.load(os.path.join(path, "domain.json"), None)
env_config = EnvConfig(os.path.join(path, "envconfig.json"))
from idare import KerasIDarePolicy
from rasa_core.featurizers import FloatSingleStateFeaturizer, MaxHistoryTrackerFeaturizer
feat = MaxHistoryTrackerFeaturizer(FloatSingleStateFeaturizer(), max_history=1)
policy = KerasIDarePolicy(feat)
from rasa_core.policies.ensemble import SimplePolicyEnsemble
ensemble = SimplePolicyEnsemble([policy])
# In[5]:
from rasa_core.agent import Agent
from rasa_core.interpreter import NaturalLanguageInterpreter
from rasa_core.tracker_store import InMemoryTrackerStore
_interpreter = NaturalLanguageInterpreter.create('./nlu_model/current')
_tracker_store = Agent.create_tracker_store(None, domain, env_config)
# In[7]:
agent = Agent(domain, ensemble, _interpreter, _tracker_store, env_config)
# In[9]:
def create_app(port, server_ip, server_port, redis_ip, redis_port,
nlu_train_file, path_to_glove, envconfig):
app = Flask(__name__)
server_url = "http://" + server_ip + ':' + server_port
model_directory = envconfig['interpreter_dir']
dst_directory = envconfig['dst_model_dir']
feedback_file_loc = model_directory + '/data/feedback_data.json'
logger.debug('Loading Suprath nlu')
nlu_model = Sinterpreter(path_to_glove=path_to_glove)
domain = OntologyDomain.load(envconfig['domain'], None)
feat = MaxHistoryTrackerFeaturizer(FloatSingleStateFeaturizer(),
max_history=2)
policy = KerasIDarePolicy(feat)
ensemble = SimplePolicyEnsemble([policy])
try:
nlu_model.load(path_to_dir=model_directory)
except OSError:
logger.warning(
'Model doesnt already exist, please train suprath nlu model once')
@app.after_request
def after_request(response):
response.headers.add('Access-Control-Allow-Origin', '*')
response.headers.add('Access-Control-Allow-Headers',
'Content-Type,Authorization')
response.headers.add('Access-Control-Allow-Methods',
'GET,PUT,POST,DELETE,OPTIONS')
return response
@app.route("/run")
def start():
"""Runs the server"""
logger.info("Runnning the server")
try:
r = requests.post(server_url + '/run', timeout=5)
except ReadTimeout:
logger.info("Server now running")
return jsonify({"success": 1})
except Exception as e:
logger.error(str(e))
return "Exception %s" % e
logger.error("Some error occurred")
return "Error in starting the server"
@app.route("/ssg", methods=['GET', 'OPTIONS'])
def ssg():
logger.info("Starting ssg training")
args = request.args
if not args:
return "Invalid request"
ssg_file = args.get('file')
if not ssg_file:
return "Invalid request"
with open(nlu_train_file, 'r') as f:
nlu_data = json.load(f)
try:
with open(ssg_file, 'r') as f:
ssg_data = json.load(f)
except Exception:
with open(ssg_file, 'r') as f:
ssg_data = eval(f.read())
except Exception as e:
logger.error(str(e))
return str(e)
nlu_data['rasa_nlu_data']['common_examples'] += (ssg_data)
with open(nlu_train_file, 'w') as f:
json.dump(nlu_data, f, indent=4, sort_keys=True)
logger.info("Success SSG training")
return jsonify({"success": 1})
@app.route("/train_dst", methods=['GET', 'OPTIONS'])
def train_dst():
"""Processes all the nlu feedback and train the nlu model"""
logger.info("Training DST")
agent = Agent(domain, ensemble, None, None)
training_data = agent.load_data(envconfig['stories_path'])
agent.train(training_data, epochs=50)
agent.policy_ensemble.persist(dst_directory, False)
#Finally setting all the trained data train_status to 1
logger.info("Training nlu successfull")
return jsonify({"success": 1})
@app.route("/train_nlu", methods=['GET', 'OPTIONS'])
def process_and_train_nlu():
"""Processes all the nlu feedback and train the nlu model"""
logger.info("Training nlu")
nlu_model_config_file = "nlu_model_config.yml"
Interpreter.train(nlu_train_file, './nlu_model', nlu_model_config_file,
path_to_glove)
#Finally setting all the trained data train_status to 1
logger.info("Training nlu successfull")
return jsonify({"success": 1})
@app.route('/feedback_train', methods=['GET', 'OPTIONS'])
def feedback_train():
logger.info("Starting feedback training")
raw_data = get_feedback_info()
#if not raw_data or len(raw_data) < 1:
if not raw_data or len(raw_data) < 1:
return jsonify({"Error": "No data available to train"})
if not os.path.exists(os.path.dirname(feedback_file_loc)):
try:
os.makedirs(os.path.dirname(feedback_file_loc))
except OSError as exc: # Guard against race condition
if exc.errno != os.errno.EEXIST:
raise exc
train_data = get_nlu_train_data(raw_data, redis_ip, redis_port)
update_nlu_training(train_data, feedback_file_loc)
nlu_model.train_feedback()
#Finally setting all the trained data train_status to 1
set_train_status(raw_data)
logger.info("Feedback training successfull")
return jsonify({'success': 1})
@app.route("/deploy_nlu", methods=['GET', 'OPTIONS'])
def deploy_nlu():
""" Deploys the nlu model to the server"""
#First compressing
logger.info("Depoying NLU..")
zout = zipfile.ZipFile('nlu.zip', "w", zipfile.ZIP_DEFLATED)
nlu_dir = model_directory + '/'
for fname in os.listdir(nlu_dir):
print("writing: ", fname)
zout.write(nlu_dir + fname, arcname=os.path.join('./', fname))
zout.close()
file = {'nlu_model': open('nlu.zip', 'rb')}
url = 'http'
try:
result = requests.post(server_url + '/load_nlu', files=file)
logger.info("Deployed NLU successfully")
return jsonify(result.json())
except JSONDecodeError:
string = "Got this from server: {}".format(result.status_code)
logger.info(string)
return string
except ConnectionError as e:
logger.error(e.__class__.__name__ + str(e))
return str(e)
def update_nlu_training(train_data, feedback_file_loc):
feedback = []
for dat in train_data:
feedback.append({
'act': dat[0],
'feedback': int(dat[2]),
'query': dat[1]
})
with open(feedback_file_loc, 'w') as f:
json.dump(feedback, f)
return app
def create_tracker_store(store: Optional['TrackerStore'],
domain: Domain) -> 'TrackerStore':
if store is not None:
store.domain = domain
return store
else:
return InMemoryTrackerStore(domain)
@staticmethod
def _create_ensemble(
policies: Union[List[Policy], PolicyEnsemble, None]
) -> Optional[PolicyEnsemble]:
if policies is None:
return None
if isinstance(policies, list):
return SimplePolicyEnsemble(policies)
elif isinstance(policies, PolicyEnsemble):
return policies
else:
passed_type = type(policies).__name__
raise ValueError(
"Invalid param `policies`. Passed object is "
"of type '{}', but should be policy, an array of "
"policies, or a policy ensemble".format(passed_type))
def _form_policy_not_present(self):
# type: () -> bool
"""Check whether form policy is not present
if there is a form action in the domain
"""
return (self.domain and self.domain.form_names and