def _pair_with_human(cursor, userid, my_index, partner_id, scenario, chat_id):
controller, my_session, partner_session = _init_controller(my_index, HumanSystem.name(), scenario, chat_id)
self.controller_map[userid] = controller
self.controller_map[partner_id] = controller
self.sessions[userid] = my_session
self.sessions[partner_id] = partner_session
self._update_user(cursor, partner_id,
status=Status.Chat,
partner_id=userid,
partner_type=HumanSystem.name(),
scenario_id=scenario.uuid,
agent_index=1 - my_index,
selected_index=-1,
message="",
chat_id=chat_id)
self._update_user(cursor, userid,
status=Status.Chat,
partner_id=partner_id,
partner_type=HumanSystem.name(),
scenario_id=scenario_id,
agent_index=my_index,
selected_index=-1,
message="",
chat_id=chat_id)
return True
def add_systems(config_dict, schema, lexicon, realizer):
"""
Params:
config_dict: A dictionary that maps the bot name to a dictionary containing configs for the bot. The
dictionary should contain the bot type (key 'type') and. for bots that use an underlying model for generation,
the path to the directory containing the parameters, vocab, etc. for the model.
Returns:
agents: A dict mapping from the bot name to the System object for that bot.
pairing_probabilities: A dict mapping from the bot name to the probability that a user is paired with that
bot. Also includes the pairing probability for humans (backend.Partner.Human)
"""
total_probs = 0.0
systems = {HumanSystem.name(): HumanSystem()}
pairing_probabilities = {}
for (sys_name, info) in config_dict.iteritems():
if "active" not in info.keys():
warnings.warn("active status not specified for bot %s - assuming that bot is inactive." % sys_name)
if info["active"]:
type = info["type"]
# TODO: add realizer to simple system
if type == SimpleSystem.name():
model = SimpleSystem(lexicon, timed_session=True, realizer=realizer, consecutive_entity=False)
elif type == NeuralSystem.name():
path = info["path"]
decoding = info["decoding"].split()
model = NeuralSystem(schema, lexicon, path, False, decoding, timed_session=True, realizer=realizer, consecutive_entity=False)
else:
warnings.warn(
'Unrecognized model type in {} for configuration '
'{}. Ignoring configuration.'.format(info, sys_name))
continue
systems[sys_name] = model
if 'prob' in info.keys():
prob = float(info['prob'])
pairing_probabilities[sys_name] = prob
total_probs += prob
if total_probs > 1.0:
raise ValueError("Probabilities for active bots can't exceed 1.0.")
if len(pairing_probabilities.keys()) != 0 and len(pairing_probabilities.keys()) != len(systems.keys()):
remaining_prob = (1.0-total_probs)/(len(systems.keys()) - len(pairing_probabilities.keys()))
else:
remaining_prob = 1.0 / len(systems.keys())
inactive_bots = set()
for system_name in systems.keys():
if system_name not in pairing_probabilities.keys():
if remaining_prob == 0.0:
inactive_bots.add(system_name)
else:
pairing_probabilities[system_name] = remaining_prob
for sys_name in inactive_bots:
systems.pop(sys_name, None)
return systems, pairing_probabilities
def aggregate_responses(
question,
completed_scenarios,
surveys,
transcripts,
bots=['human', 'rulebased', 'static-neural', 'dynamic-neural'],
n=5):
agent_type_mappings = surveys[0]
# print agent_type_mappings
responses = surveys[1]
question_responses = dict(
(b, dict((x, 0.0) for x in xrange(0, n))) for b in bots)
for chat in transcripts:
cid = chat["uuid"]
if chat["scenario_uuid"] in completed_scenarios and cid in agent_type_mappings.keys(
):
agents = agent_type_mappings[cid]
agent_type = agents['0'] if agents['1'] == HumanSystem.name(
) else agents['1']
rating = responses[cid][question]
question_responses[agent_type][rating - 1] += 1.0
return question_responses
def _find_disconnected_user_chats():
"""
Find chats with no outcome where at least one human agent has been disconnected longer
than user_timeout seconds
:return:
"""
# print 'Finding chats with no outcome and users with timed out connections'
cursor.execute('''SELECT * FROM chat WHERE outcome=""''')
inc_chats = cursor.fetchall()
disconnected_chats = []
for chat_info in inc_chats:
chat_id, sid, outcome, agent_ids, agent_types, start_time = chat_info
agent_types = json.loads(agent_types)
agent_ids = json.loads(agent_ids)
human_idxes = [
k for k in agent_types.keys()
if agent_types[k] == HumanSystem.name()
]
clean = False
for idx in human_idxes:
userid = agent_ids[idx]
if _is_connection_timed_out(userid):
# print "User %s connection timeout" % userid
clean = True
if clean:
disconnected_chats.append(chat_info)
return disconnected_chats
def convert_events_to_json(chat_id, cursor, scenario_db):
try:
cursor.execute(
'SELECT agent, action, time, data, start_time FROM event WHERE chat_id=? ORDER BY time ASC',
(chat_id, ))
logged_events = cursor.fetchall()
except sqlite3.OperationalError:
cursor.execute(
'SELECT agent, action, time, data FROM event WHERE chat_id=? ORDER BY time ASC',
(chat_id, ))
logged_events = cursor.fetchall()
events = []
for i, (agent, action, time, data) in enumerate(logged_events):
events.append((agent, action, time, data, time))
logged_events = events
cursor.execute('SELECT scenario_id, outcome FROM chat WHERE chat_id=?',
(chat_id, ))
(uuid, outcome) = cursor.fetchone()
try:
outcome = json.loads(outcome)
except ValueError:
outcome = {'reward': 0}
try:
cursor.execute('SELECT agent_types FROM chat WHERE chat_id=?',
(chat_id, ))
agent_types = cursor.fetchone()[0]
agent_types = json.loads(agent_types)
except sqlite3.OperationalError:
agent_types = {0: HumanSystem.name(), 1: HumanSystem.name()}
chat_events = []
for (agent, action, time, data, start_time) in logged_events:
if action == 'join' or action == 'leave':
continue
if action == 'select':
data = json.loads(data)
time = convert_time_format(time)
start_time = convert_time_format(start_time)
event = Event(agent, time, action, data, start_time)
chat_events.append(event)
return Example(scenario_db.get(uuid), uuid, chat_events, outcome, chat_id,
agent_types)
def _init_controller(my_index, partner_type, scenario, chat_id):
my_session = self.systems[HumanSystem.name()].new_session(my_index, scenario.get_kb(my_index))
if partner_type == 'rule_bot':
partner_session = self.systems[partner_type].new_session(1-my_index, scenario.get_kb(1-my_index), scenario.style)
else:
partner_session = self.systems[partner_type].new_session(1-my_index, scenario.get_kb(1-my_index))
controller = Controller(scenario, [my_session, partner_session], chat_id=chat_id, debug=False)
return controller, my_session, partner_session
def _update_inactive_chats(chats):
for chat_info in chats:
chat_id, sid, outcome, _, agent_types, start_time = chat_info
if chat_id not in cleaned_chats:
# if it's been longer than chat_timeout seconds since the chat started, and the chat
# wasn't previously cleaned up, update the scenario DB
agent_types = json.loads(agent_types)
partner_type = agent_types['0'] if agent_types[
'1'] == HumanSystem.name() else agent_types['1']
print "[Cleaner] Cleaned up chat with ID={}, partner_type={}, scenario_id={}".format(
chat_id, partner_type, sid)
cursor.execute(
'''
UPDATE scenario SET active=active-1 WHERE partner_type=? AND scenario_id=?
''', (partner_type, sid))
cleaned_chats.add(chat_id)
def _choose_scenario_and_partner_type(cursor):
# for each scenario, get number of complete dialogues per agent type
non_human_partners = [k for k in self.systems.keys() if k != HumanSystem.name()]
# print 'NON HUMAN PARTNERS!!', non_human_partners
cursor.execute('''SELECT * FROM scenario''')
db_scenarios = cursor.fetchall()
scenario_dialogues = defaultdict(lambda: defaultdict(int))
for (scenario_id, partner_type, num_complete, num_active) in db_scenarios:
# map from scenario ID -> partner type -> # of completed dialogues with that partner
if scenario_id not in scenario_dialogues:
scenario_dialogues[scenario_id] = {}
scenario_dialogues[scenario_id][partner_type] = num_complete + num_active
# find "active" scenarios (scenarios for which at least one agent type has no completed or active dialogues)
active_scenarios = defaultdict(list)
for sid in scenario_dialogues.keys():
for partner_type in non_human_partners:
if scenario_dialogues[sid][partner_type] == 0:
active_scenarios[sid].append(partner_type)
# import pdb; pdb.set_trace()
# print '*'*20
# print 'active_scenarios', active_scenarios
# print 'scenario_dialogues', scenario_dialogues
# if all scenarios have at least one dialogue per agent type (i.e. no active scenarios),
# just select a random scenario and agent type
if len(active_scenarios.keys()) == 0:
sid = np.random.choice(scenario_dialogues.keys())
p = np.random.choice(non_human_partners)
print 'PARTNER', p
return self.scenario_db.get(sid), p
# otherwise, select a random active scenario and an agent type that it's missing
sid = np.random.choice(active_scenarios.keys())
p = np.random.choice(active_scenarios[sid])
print 'PARTNER', p
return self.scenario_db.get(sid), p
def is_user_partner_bot(self, cursor, userid):
u = self._get_user_info_unchecked(cursor, userid)
return len(u.partner_type) > 0 and u.partner_type != HumanSystem.name()
def attempt_join_chat(self, userid):
def _init_controller(my_index, partner_type, scenario, chat_id):
my_session = self.systems[HumanSystem.name()].new_session(my_index, scenario.get_kb(my_index))
if partner_type == 'rule_bot':
partner_session = self.systems[partner_type].new_session(1-my_index, scenario.get_kb(1-my_index), scenario.style)
else:
partner_session = self.systems[partner_type].new_session(1-my_index, scenario.get_kb(1-my_index))
controller = Controller(scenario, [my_session, partner_session], chat_id=chat_id, debug=False)
return controller, my_session, partner_session
def _pair_with_human(cursor, userid, my_index, partner_id, scenario, chat_id):
controller, my_session, partner_session = _init_controller(my_index, HumanSystem.name(), scenario, chat_id)
self.controller_map[userid] = controller
self.controller_map[partner_id] = controller
self.sessions[userid] = my_session
self.sessions[partner_id] = partner_session
self._update_user(cursor, partner_id,
status=Status.Chat,
partner_id=userid,
partner_type=HumanSystem.name(),
scenario_id=scenario.uuid,
agent_index=1 - my_index,
selected_index=-1,
message="",
chat_id=chat_id)
self._update_user(cursor, userid,
status=Status.Chat,
partner_id=partner_id,
partner_type=HumanSystem.name(),
scenario_id=scenario_id,
agent_index=my_index,
selected_index=-1,
message="",
chat_id=chat_id)
return True
def _pair_with_bot(cursor, userid, my_index, bot_type, scenario, chat_id):
controller, my_session, bot_session = _init_controller(my_index, bot_type, scenario, chat_id)
self.controller_map[userid] = controller
self.sessions[userid] = my_session
self._update_user(cursor, userid,
status=Status.Chat,
partner_id=0,
partner_type=bot_type,
scenario_id=scenario_id,
agent_index=my_index,
selected_index=-1,
message="",
chat_id=chat_id)
return True
def _get_other_waiting_users(cursor, userid):
cursor.execute("SELECT name FROM active_user WHERE name!=? AND status=? AND connected_status=1",
(userid, Status.Waiting))
userids = [r[0] for r in cursor.fetchall()]
return userids
def _choose_scenario_and_partner_type(cursor):
# for each scenario, get number of complete dialogues per agent type
non_human_partners = [k for k in self.systems.keys() if k != HumanSystem.name()]
# print 'NON HUMAN PARTNERS!!', non_human_partners
cursor.execute('''SELECT * FROM scenario''')
db_scenarios = cursor.fetchall()
scenario_dialogues = defaultdict(lambda: defaultdict(int))
for (scenario_id, partner_type, num_complete, num_active) in db_scenarios:
# map from scenario ID -> partner type -> # of completed dialogues with that partner
if scenario_id not in scenario_dialogues:
scenario_dialogues[scenario_id] = {}
scenario_dialogues[scenario_id][partner_type] = num_complete + num_active
# find "active" scenarios (scenarios for which at least one agent type has no completed or active dialogues)
active_scenarios = defaultdict(list)
for sid in scenario_dialogues.keys():
for partner_type in non_human_partners:
if scenario_dialogues[sid][partner_type] == 0:
active_scenarios[sid].append(partner_type)
# import pdb; pdb.set_trace()
# print '*'*20
# print 'active_scenarios', active_scenarios
# print 'scenario_dialogues', scenario_dialogues
# if all scenarios have at least one dialogue per agent type (i.e. no active scenarios),
# just select a random scenario and agent type
if len(active_scenarios.keys()) == 0:
sid = np.random.choice(scenario_dialogues.keys())
p = np.random.choice(non_human_partners)
print 'PARTNER', p
return self.scenario_db.get(sid), p
# otherwise, select a random active scenario and an agent type that it's missing
sid = np.random.choice(active_scenarios.keys())
p = np.random.choice(active_scenarios[sid])
print 'PARTNER', p
return self.scenario_db.get(sid), p
def _update_used_scenarios(scenario_id, partner_type):
# cursor.execute('''SELECT active FROM scenario WHERE scenario_id? AND''')
cursor.execute(
'''UPDATE scenario
SET active=active+1
WHERE scenario_id=? AND partner_type=?''',
(scenario_id, partner_type))
try:
with self.conn:
cursor = self.conn.cursor()
others = _get_other_waiting_users(cursor, userid)
my_index = np.random.choice([0, 1])
scenario, partner_type = _choose_scenario_and_partner_type(cursor)
scenario_id = scenario.uuid
chat_id = self._generate_chat_id()
if partner_type == HumanSystem.name():
print "others" * 20
print "others", others
if len(others) == 0:
return None
partner_id = np.random.choice(others)
_update_used_scenarios(scenario_id, HumanSystem.name())
if my_index == 0:
self.add_chat_to_db(chat_id, scenario_id, userid, partner_id, HumanSystem.name(), HumanSystem.name())
else:
self.add_chat_to_db(chat_id, scenario_id, partner_id, userid, HumanSystem.name(), HumanSystem.name())
return _pair_with_human(cursor, userid, my_index, partner_id, scenario, chat_id)
else:
_update_used_scenarios(scenario_id, partner_type)
if my_index == 0:
self.add_chat_to_db(chat_id, scenario_id, userid, 0, HumanSystem.name(), partner_type)
else:
self.add_chat_to_db(chat_id, scenario_id, 0, userid, partner_type, HumanSystem.name())
return _pair_with_bot(cursor, userid, my_index, partner_type, scenario, chat_id)
except sqlite3.IntegrityError:
print("WARNING: Rolled back transaction")