def save_intents_by_tasks():
""" Fetches intents from all users, divides them by task and save it to a csv """
db = client.Database()
intents_by_task = {
0: [],
1: [],
2: [],
3: [],
4: [],
5: []
}
with open(config.USER_STUDY_QUESTIONNAIRE_PATH.format("post")) as csvfile:
reader = csv.reader(csvfile, delimiter=',',)
next(reader) # skip header
for row in reader:
uuid = row[1]
session_intents = db.get_intents(uuid)
for intent in session_intents:
task_completed = tasks.check_intent(intent)
if task_completed > 0:
intents_by_task[task_completed].append(intent['text'])
print("INTENTS", intents_by_task)
with open(config.USER_STUDY_INTENTS_PATH, 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(['Task', 'Intent'])
for (task, intents) in intents_by_task.items():
for intent in intents:
writer.writerow([task, intent])
def build_accepted(request):
""" Webhook action to deploy Nile intent after user confirmation """
uuid = request.get("session").split("/")[-1]
db = client.Database()
intent = db.get_latest_intent(uuid)
db.update_intent(intent["_id"], {"status": "confirmed"})
# TODO: fix conflict after user study
# conflict = inspector.check(intent, uuid)
# if conflict :
# text = "The intent you described probably conflict s a previous one. Do you want to deploy it anyway or remove the old one?"
# return make_card_response("Possible conflict ", text, text, beautify_intent(conflict ["nile"]),
# suggestions=["Deploy it anyway", "Remove old one and deploy new", "Keep old one"])
merlin_program, compilation_time = compiler.compile(intent["nile"])
if merlin_program:
db.update_intent(intent["_id"], {"status": "compiled", "merlin": merlin_program})
return make_simple_response("Okay! Intent compiled and deployed!")
# TODO: fix deploy API after user study
# res = requests.post(config.DEPLOY_URL, {"intent": intent["nile"]})
# if res.status["code"] == 200:
# return make_simple_response("Okay! Intent compiled and deployed!")
# db.update_intent(intent["_id"], {"status": "deployed"})
return make_simple_response("Sorry. Something went wrong during deployment. :(")
def check(session, task):
"""Given Session UUID and Task number, check if user has completed it."""
db = client.Database()
intents = db.get_intents(session)
checker = {
1: check_task_one,
2: check_task_two,
3: check_task_three,
4: check_task_four,
5: check_task_five
}
return checker[task](intents) if task in checker else False
def check_intents():
""" Fetches intents from sessions in the user study to analyze them """
db = client.Database()
live_sessions = {}
intents = {}
confirmed_intents = {}
tasks_completed_by_session = {}
with open(config.USER_STUDY_QUESTIONNAIRE_PATH.format("pre")) as csvfile:
reader = csv.reader(csvfile, delimiter=',',)
next(reader) # skip header
for row in reader:
uuid = row[1]
if uuid not in live_sessions:
live_sessions[uuid] = 0
live_sessions[uuid] += 1
with open(config.USER_STUDY_QUESTIONNAIRE_PATH.format("post")) as csvfile:
reader = csv.reader(csvfile, delimiter=',',)
next(reader) # skip header
for row in reader:
uuid = row[1]
if uuid not in live_sessions:
live_sessions[uuid] = 0
live_sessions[uuid] += 1
print("NUM LIVE SESSIONS", len(list(live_sessions)))
for session, count in live_sessions.items():
if count == 2:
session_intents = db.get_intents(session)
# print(session_intents)
intents[session] = []
confirmed_intents[session] = []
for intent in session_intents:
intents[session].append(intent)
if intent['status'] != 'pending':
# print(intent['status'])
confirmed_intents[session].append(intent)
print("NUM INTENTS", session, len(intents[session]))
print("CONFIRMED INTENTS", session, len(confirmed_intents[session]))
tasks_completed_by_session[session] = 0
for task in [1, 2, 3, 4, 5]:
result = tasks.check(session, task)
# print("TASK #", task, result['done'])
tasks_completed_by_session[session] += 1 if result['done'] else 0
print("COMPLETED TASKS", tasks_completed_by_session[session])
print("TOTAL INTENTS", sum([len(intents) for (uuid, intents) in intents.items()]))
def feedback_count():
""" Count the total number of intents generated, and count the number of feedbacks given """
db = client.Database()
num_intents = 0
num_intents_accepted_right = 0
num_intents_accepted_wrong = 0
num_intents_pending = 0
num_intents_rejected_no_feedback = 0
num_intents_feedback = 0
num_intents_feedback_accepted = 0
with open(config.USER_STUDY_QUESTIONNAIRE_PATH.format("post")) as csvfile:
reader = csv.reader(csvfile, delimiter=',',)
next(reader) # skip header
for row in reader:
uuid = row[1]
session_intents = db.get_intents(uuid)
for intent in session_intents:
num_intents += 1
if intent['status'] == 'declined':
if 'missingEntities' in intent:
num_intents_feedback += 1
else:
num_intents_rejected_no_feedback += 1
elif 'missingEntities' in intent and 'nileFeedback' in intent:
num_intents_feedback_accepted += 1
elif intent['status'] == 'compiled':
if tasks.check_intent(intent):
num_intents_accepted_right += 1
else:
num_intents_accepted_wrong += 1
else:
num_intents_pending += 1
print("NUM INTENTS", num_intents)
print("CONFIRMED INTENTS RIGHT", num_intents_accepted_right)
print("CONFIRMED INTENTS WRONG", num_intents_accepted_wrong)
print("PENDING INTENTS", num_intents_pending)
# print("REJECTED INTENTS", num_intents_rejected_no_feedback)
print("FEEDBACK REJECTED INTENTS", num_intents_feedback)
print("FEEDBACK ACCEPTED INTENTS", num_intents_feedback_accepted)
plotter.plot_pie_chart(['Confirmed Intents (Right)', 'Confirmed Intents (Wrong)', 'Pending Intents', 'Feedback Accepted', 'Feedback Rejected'],
[num_intents_accepted_right, num_intents_accepted_wrong, num_intents_pending,
num_intents_feedback_accepted, num_intents_feedback],
config.USER_STUDY_PLOTS_PATH.format("feedback"))
def finish(session):
"""
API to finish session and record end timestamp
"""
print("Session: {}".format(session))
try:
db = client.Database()
res = db.finish_session(session)
if res:
res = "Session {} finished and recorded.".format(session)
else:
res = "Session {} does not exist or was already finished.".format(session)
except Exception as err:
traceback.print_exc()
res = "{}".format(err)
print("Response: {}".format(res))
return make_response(res)
def feedback_confirm(request):
""" Webhook action to confirm feedback received from the user """
uuid = request.get("session").split("/")[-1]
db = client.Database()
intent = db.get_latest_intent(uuid)
print("INTENT CONFIRM", intent)
entities = intent['entities']
for entity, values in intent["missingEntities"].items():
entity_key = entity
if entity == "middlebox":
entity_key = "middleboxes"
elif entity == "service":
entity_key = "services"
elif entity == "traffic":
entity_key = "traffics"
elif entity == "protocol":
entity_key = "protocols"
elif entity == "operation":
entity_key = "operations"
elif entity == "location":
entity_key = "locations"
if entity_key not in entities:
entities[entity_key] = list(values.keys())
else:
entities[entity_key] += list(values.keys())
try:
nile = builder.build(entities)
speech = "So, is this what you want then?"
response = make_card_response("Nile Intent", nile, speech, beautify_intent(nile),
suggestions=["Yes", "No"])
# tracking
db.update_intent(intent["_id"], {"status": "pending", "nileFeedback": nile})
except ValueError as err:
traceback.print_exc()
# TODO: use slot-filling to get the missing info
# TODO: use different exceptions to figure out whats missing
response = make_simple_response("{}".format(err))
return response
def gateway():
"""
Gateway for Dialogflow API
request = {
session: <uuid>,
live: <true/false>,
queryInput: {
text: {
text: <text>,
languageCode: 'en'
}
}
}
"""
req = request.get_json(silent=True, force=True)
# print("Request: {}".format(json.dumps(req, indent=4)))
try:
dialogflow = api.Dialogflow()
session = req.get("session")
live = req.get("live")
text = req.get("queryInput").get('text').get('text')
res = dialogflow.detect_intent(text, session)
# tracking
db = client.Database()
db.insert_session(session, live)
query_result_text = res.query_result.fulfillment_text
if res.query_result.fulfillment_messages:
query_result_text = ', '.join([MessageToString(x.text) for x in res.query_result.fulfillment_messages])
db.insert_message(session, text, query_result_text, res.query_result.intent.display_name)
except Exception as err:
traceback.print_exc()
res = "{}".format(err)
res = MessageToJson(res)
# print("Response: {}".format(json.dumps(res, indent=4)))
response = make_response(res)
response.headers["Content-Type"] = "application/json"
return response
def feedback_train(request):
""" Webhook action to train chatbot with feedback information """
uuid = request.get("session").split("/")[-1]
db = client.Database()
intent = db.get_latest_intent(uuid)
dialogflow = Dialogflow()
for entity, values in intent["missingEntities"].items():
try:
entity_type_id = dialogflow.get_entity_type_id(entity)
for value in values.keys():
dialogflow.create_entity(entity_type_id, value, [])
except:
traceback.print_exc()
merlin_program, compilation_time = compiler.compile(intent["nileFeedback"])
if merlin_program:
db.update_intent(intent["_id"], {"status": "compiled", "merlin": merlin_program})
return make_simple_response("Okay! Feedback received. Intent compiled and deployed!")
return make_simple_response("Sorry. Something went wrong during deployment. :(")
def build_feedback(request):
""" Webhook action to receive feedback from user after rejecting built intent """
uuid = request.get("session").split("/")[-1]
db = client.Database()
intent = db.get_latest_intent(uuid)
feedback = parse_feedback(request)
entity_types = ["Location", "Group", "Middlebox", "Service", "Traffic"]
query = request["queryResult"]["queryText"].lower()
if "cancel" in query or "start over" in query:
response = make_simple_response("Okay, cancelled. Please start over.")
return reset_output_context(request, response)
# slot-filling
if "entity" not in feedback and "value" not in feedback:
return make_simple_response("First of all, what entity did I miss?", suggestions=entity_types)
elif "entity" not in feedback:
return make_simple_response("What type of entity is '{}'?".format(feedback["value"]), suggestions=entity_types)
elif "value" not in feedback:
suggestions = []
for word in intent['text'].split():
entities = intent['entities'].values()
if word not in entities:
suggestions.append(word)
return make_simple_response("Great! And what word is a {}?".format(feedback["entity"]), suggestions=suggestions)
missing_entities = {}
if "missingEntities" in intent:
missing_entities = intent["missingEntities"]
if feedback["entity"] not in missing_entities:
missing_entities[feedback["entity"]] = {}
missing_entities[feedback["entity"]][feedback["value"]] = True
db.update_intent(intent["_id"], {"status": "declined", "missingEntities": missing_entities})
print("training feedback", uuid, intent)
return make_simple_response("Okay! And is there anything else I missed?", suggestions=["Yes", "No"])
def build_nile_intent(request):
""" Webhook action to build Nile intent from Dialogflow request """
uuid = request.get("session").split("/")[-1]
text = request.get("queryResult").get("queryText")
response = {}
try:
entities = parse_entities(request)
intent = builder.build(entities)
speech = "Is this what you want?"
response = make_card_response("Nile Intent", intent, speech, beautify_intent(intent),
suggestions=["Yes", "No"])
# tracking
db = client.Database()
db.insert_intent(uuid, text, entities, intent)
except ValueError as err:
traceback.print_exc()
# TODO: use slot-filling to get the missing info
# TODO: use different exceptions to figure out whats missing
response = make_simple_response("{}".format(err))
return response
def check(new_intent, session):
""" Checks new intent for conflicts """
model = ClassificationModel('forest')
contradictory = None
if model.load(10000):
db = client.Database()
confirmed_intents = db.get_confirmed_intents(session)
for intent in confirmed_intents:
if intent['_id'] != new_intent['_id']:
features = get_features(new_intent['nile'], intent['nile'])
print("Extracted features:", features)
res = model.predict([features])
print('Conflict?', res)
db.insert_conflict(session, intent, new_intent, features,
bool(res[0]))
if res[0]:
print("conflict detected!")
contradictory = intent
break
else:
print("Failure loading conflict model. Will continue without it.")
return contradictory
