def parley(self):
if self.cnt == 0:
self.topic = self.get_new_topic()
self.acts = [None, None]
self.human_first = random.choice([0, 1])
# possibly get human act first
if self.cnt == 0 and not self.human_first:
self.acts[0] = act = {'text': '', 'episode_done': False}
act = self.acts[0]
else:
self.acts[0] = self.human_agent.act()
act = deepcopy(self.acts[0])
# model agent observe
if self.cnt == 0 and self.topic != NO_TOPIC:
# add the chosen_topic to the message
act['chosen_topic'] = self.topic
act['text'] = '\n'.join([self.topic, act.get('text', 'hi')])
self.model_agent.observe(validate(act))
# model agent act
self.acts[1] = self.model_agent.act()
# human agent observe
self.human_agent.observe(validate(self.acts[1]))
self.update_counters()
self.cnt += 1
if self.episode_done():
print('[ CHAT DONE ]')
print('\n[ Preparing new chat... ]\n')
self.cnt = 0
self.model_agent.reset()
def parley(self):
if self.episode_done():
self.cnt = 0
agents = self.get_agents()
for a in agents:
a.reset()
if self.cnt == 0:
self.acts = [None, None]
# choose speaking order:
if random.choice([0, 1]):
self.agents_ordered = [self.agents[0], self.agents[1]]
else:
self.agents_ordered = [self.agents[1], self.agents[0]]
self.contexts = self.get_contexts()
# initial context
for i in range(0, 2):
context = {
'text': self.contexts[i],
'episode_done': False,
'id': 'context',
}
self.acts[1 - i] = context
self.agents_ordered[i].observe(validate(context))
else:
# do regular loop
acts = self.acts
agents = self.agents_ordered
acts[0] = agents[0].act()
agents[1].observe(validate(acts[0]))
acts[1] = agents[1].act()
agents[0].observe(validate(acts[1]))
self.update_counters()
self.cnt += 1
def parley(self):
# only pick topic if your ID is PERSON_1
if self.role == 'PERSON_1':
seen = random.choice([True, False])
num = random.choice([2, 3])
topics = self.mturk_agent.topics_generator.get_topics(seen=seen, num=num)
self.mturk_agent.observe(
validate(
{'id': 'SYSTEM', 'text': PICK_TOPIC_MSG, 'relevant_topics': topics}
)
)
topic_act = self.mturk_agent.act(timeout=self.max_choice_time)
timed_out = self.check_timeout(topic_act)
if timed_out:
return
pick_msg = AFTER_PICK_TOPIC_MSG
self.mturk_agent.observe({'id': 'SYSTEM', 'text': pick_msg})
self.mturk_agent.chosen_topic = topic_act['text']
self.mturk_agent.topic_choices = topics
self.mturk_agent.seen = seen
self.mturk_agent.observe(validate({'id': 'SYSTEM', 'wait_msg': True}))
else:
self.mturk_agent.observe(validate({'id': 'SYSTEM', 'wait_msg': True}))
return
def eval_or_shutdown(agent):
if self.model_agent is None and agent == self.other_agent:
control_msg = {'episode_done': False}
control_msg['id'] = 'SYSTEM'
control_msg['text'] = OTHER_AGENT_FINISHED_MSG
self.other_agent.observe(validate(control_msg))
# mark eval agent done
self.eval_agent.mturk_manager.mark_workers_done([self.eval_agent])
# shutdown other agent
self.other_agent.shutdown()
else:
control_msg = {'episode_done': False}
control_msg['id'] = 'SYSTEM'
# General mark for this convo
control_msg['text'] = GMARK_MSG
control_msg['general_mark_score'] = True
self.eval_agent.observe(validate(control_msg))
act = self.eval_agent.act(timeout=self.max_resp_time)
timeout = self.check_timeout(act)
if timeout:
return
while act['text'] not in self.ratings:
control_msg['text'] = NAN_MSG
self.eval_agent.observe(validate(control_msg))
act = self.eval_agent.act(timeout=self.max_resp_time)
if 'text' in act and act['text'] in self.ratings:
self.gmark_score = int(act['text'])
def parley(self):
self.turn_index = (self.turn_index + 1) % 2; # Each turn starts from the QA Collector agent
ad = { 'episode_done': False }
ad['id'] = self.__class__.collector_agent_id
if self.turn_index == 0:
# At the first turn, the QA Collector agent provides the context and
# prompts the turker to ask a question regarding the context
# Get context from SQuAD teacher agent
qa = self.task.act()
context = '\n'.join(qa['text'].split('\n')[:-1])
# Wrap the context with a prompt telling the turker what to do next
ad['text'] = (context +
'\n\nPlease provide a question given this context.')
self.mturk_agent.observe(validate(ad))
self.question = self.mturk_agent.act() # Can log the turker's question here
if self.turn_index == 1:
# At the second turn, the QA Collector collects the turker's question from the first turn,
# and then prompts the turker to provide the answer
# A prompt telling the turker what to do next
ad['text'] = 'Thanks. And what is the answer to your question?'
ad['episode_done'] = True # end of episode
self.mturk_agent.observe(validate(ad))
self.answer = self.mturk_agent.act() # Can log the turker's answer here
self.episodeDone = True
def parley(self):
"""Agent 0 goes first. Alternate between the two agents."""
if self.cnt == 0:
self.p1, self.p2 = self.get_new_personas()
acts = self.acts
agents = self.agents
if self.cnt == 0:
# add the persona on to the first message to agent 0
act = {}
act['text'] = self.p1
act['episode_done'] = False
act['id'] = 'persona'
agents[0].observe(validate(act))
act = deepcopy(agents[0].act())
if self.cnt == 0:
# add the persona on to the first message to agent 1
act['text'] = self.p2 + act.get('text', 'hi')
print("gave bot its persona!")
print(act)
agents[1].observe(validate(act))
else:
agents[1].observe(validate(act))
acts[1] = agents[1].act()
agents[0].observe(validate(acts[1]))
self.update_counters()
self.cnt += 1
if self.episode_done():
print("CHAT DONE ")
print("\n... preparing new chat... \n")
self.cnt = 0
def evaluate_repetitiveness(self):
control_msg = self.get_control_msg()
control_msg['text'] = REPETITIVENESS_MSGS[0]
control_msg['button_choices'] = '</ROUND>'.join(REPETITIVENESS_CHOICES)
self.eval_agent.observe(validate(control_msg))
act = self.eval_agent.act(timeout=self.max_resp_time)
timeout = self.check_timeout(act)
if timeout:
return False
act_choice = REPETITIVENESS_CHOICES.index(act.get('text'))
self.repetitiveness_scores.append(act_choice)
if ASK_DETAILED and act_choice != 2:
control_msg = self.get_control_msg()
control_msg['text'] = REPETITIVENESS_MSGS[1]
control_msg['good_rounds'] = True
control_msg['rounds'] = '</ROUND>'.join(self.dialog_list)
self.eval_agent.observe(validate(control_msg))
act = self.eval_agent.act(timeout=self.max_resp_time)
timeout = self.check_timeout(act)
if timeout:
return False
if 'text' in act:
self.repetitiveness_scores.append(
[int(x) - 1 for x in act['text'].split(',')])
return True
def parley(self):
"""Alternate taking turns, until both agents have made a choice
(indicated by a turn starting with <selection>)
"""
if self.first_turn:
# Use NegotiationTeacher to load data for us
data = self.task.episodes[self.num_negotiations %
len(self.task.episodes)].strip().split()
self.num_negotiations += 1
for agent, tag in zip(self.agents, ['input', 'partner_input']):
(book_cnt, book_val, hat_cnt, hat_val, ball_cnt,
ball_val) = get_tag(data, tag)
action = {}
action['text'] = WELCOME_MESSAGE.format(book_cnt=book_cnt,
book_val=book_val,
hat_cnt=hat_cnt,
hat_val=hat_val,
ball_cnt=ball_cnt,
ball_val=ball_val)
action['items'] = {
"book_cnt": book_cnt,
"book_val": book_val,
"hat_cnt": hat_cnt,
"hat_val": hat_val,
"ball_cnt": ball_cnt,
"ball_val": ball_val
}
agent.observe(validate(action))
self.first_turn = False
else:
self.turns += 1
for _index, agent in enumerate(self.agents):
if agent in self.choices:
# This agent has already made a choice
continue
try:
act = agent.act(timeout=None)
except TypeError:
act = agent.act() # not MTurkAgent
for other_agent in self.agents:
if other_agent != agent:
other_agent.observe(validate(act))
if act["text"].startswith("<selection>") and self.turns > 1:
# Making a choice
self.choices[agent] = act["text"]
self.selection = True
if len(self.choices) == len(self.agents):
self.first_turn = True
self.episodeDone = True
elif act['episode_done']:
# Action is not selection but episode ended due to
# disconnection or timeout or returned hit
self.episodeDone = True
def parley(self):
"""
Agent 0 goes first.
Alternate between the two agents.
"""
if self.turn_cnt == 0:
self.p1, self.p2 = self.get_contexts()
acts = self.acts
agents = self.agents
if self.turn_cnt == 0 and self.p1 != '':
# add the context on to the first message to agent 0
context_act = Message(
{'id': 'context', 'text': self.p1, 'episode_done': False}
)
agents[0].observe(validate(context_act))
act = deepcopy(agents[0].act())
acts[0] = act
if self.turn_cnt == 0 and self.p2 != '':
# add the context on to the first message to agent 1
context_act = Message(
{'id': 'context', 'text': self.p2, 'episode_done': False}
)
agents[1].observe(validate(context_act))
agents[1].observe(validate(act))
acts[1] = agents[1].act()
agents[0].observe(validate(acts[1]))
self.update_counters()
self.turn_cnt += 1
if act['episode_done']:
self.finalize_episode()
self.turn_cnt = 0
def parley_observe(self, input_statement, output):
"""
Agent 0 goes first.
Alternate between the two agents.
"""
if self.turn_cnt == 0:
self.p1, self.p2 = self.get_contexts()
acts = self.acts
agents = self.agents
if self.turn_cnt == 0 and self.p1 != '':
# add the context on to the first message to agent 0
context_act = Message(
{'id': 'context', 'text': self.p1, 'episode_done': False}
)
agents[0].observe(validate(context_act))
if self.turn_cnt == 0 and self.p2 != '':
# add the context on to the first message to agent 1
context_act = Message(
{'id': 'context', 'text': self.p2, 'episode_done': False}
)
agents[1].observe(validate(context_act))
agents[1].history.add_reply(input_statement)
agents[1].history.add_reply(output)
print(agents[1].history.get_history_str())
print(agents[1].history.get_history_vec())
self.update_counters()
self.turn_cnt += 1
def evaluate_consistency(self):
control_msg = self.get_control_msg()
control_msg['text'] = CONSISTENCY_MSGS[0]
control_msg['button_choices'] = '</ROUND>'.join(CONSISTENCY_CHOICES)
self.eval_agent.observe(validate(control_msg))
act = self.eval_agent.act(timeout=self.max_resp_time)
timeout = self.check_timeout(act)
if timeout:
return False
act_choice = CONSISTENCY_CHOICES.index(act.get('text'))
self.consistency_scores.append(act_choice)
if act_choice != 0:
control_msg = self.get_control_msg()
control_msg['text'] = CONSISTENCY_MSGS[1]
control_msg['good_rounds'] = True
control_msg['rounds'] = '</ROUND>'.join(self.dialog_list)
self.eval_agent.observe(validate(control_msg))
act = self.eval_agent.act(timeout=self.max_resp_time)
timeout = self.check_timeout(act)
if timeout:
return False
if 'text' in act:
self.consistency_scores.append(
[int(x) - 1 for x in act['text'].split(',')])
return True
def parley(self):
if self.episode_done():
self.turn_cnt = 0
self.episode_cnt += 1
self.contexts = None
self.seed_utterances = None
agents = self.get_agents()
for a in agents:
a.reset()
if self.turn_cnt == 0:
self.acts = [None, None]
# get the beginning of the conversation, which can include contexts
# and/or any number of starting messages
self.contexts = self.get_contexts()
self.seed_utterances = self._get_seed_utt_acts(
self.episode_cnt, self.agents
)
if self.contexts:
assert len(self.contexts) == 2
# initial context
for i in range(0, 2):
context = {
'text': self.contexts[i],
'episode_done': False,
'id': 'context',
}
self.acts[i] = context
self.agents[i].observe(validate(context))
# clear contexts so they are only added once per episode
self.contexts = None
elif self.seed_utterances:
# pop the next two seed messages (there may be less or more than 2 total)
utts = self.seed_utterances[:2]
self.seed_utterances = self.seed_utterances[2:]
# process the turn
for i in [0, 1]:
# if we have a seed utterance, add it to the conversation
if len(utts) > i:
self.acts[i] = utts[i]
if hasattr(self.agents[i], 'self_observe'):
self.agents[i].self_observe(self.acts[i])
else:
self.acts[i] = self.agents[i].act()
self.agents[1 - i].observe(validate(self.acts[i]))
else:
# do regular loop
acts = self.acts
agents = self.agents
acts[0] = agents[0].act()
agents[1].observe(validate(acts[0]))
acts[1] = agents[1].act()
agents[0].observe(validate(acts[1]))
self.update_counters()
self.turn_cnt += 1
def parley(self):
"""
Loop between wizard and apprentice.
Adds knowledge to the wizard observations. Assumes that the model agent is the
wizard model.
"""
if self.cnt == 0:
self.topic = self._get_new_topic()
self.acts = [None, None]
if self.topic != NO_TOPIC:
self.human_first = random.choice([0, 1])
else:
self.human_first = 1
# possibly get human act first
if self.cnt == 0 and not self.human_first:
self.acts[0] = act = Message({'text': '', 'episode_done': False})
act = self.acts[0]
else:
self.acts[0] = self.human_agent.act()
act = deepcopy(self.acts[0])
# model agent observe
if self.cnt == 0 and self.topic != NO_TOPIC:
# add the chosen_topic to the message
act['chosen_topic'] = self.topic
act.force_set('text',
'\n'.join([self.topic,
act.get('text', 'hi')]))
# add knowledge to the model observation
act = self._add_knowledge_to_act(act)
# model observes knowledge and human (apprentice) act
self.model_agent.observe(validate(act))
# model agent act
self.acts[1] = self.model_agent.act()
# add the model reply to the knowledge retriever's dialogue history
self.knowledge_agent.observe(self.acts[1], actor_id='wizard')
# human (apprentice) agent observes model act
self.human_agent.observe(validate(self.acts[1]))
self.update_counters()
self.cnt += 1
if self.episode_done():
print('[ CHAT DONE ]')
print('\n[ Preparing new chat... ]\n')
self.cnt = 0
self.model_agent.reset()
def get_axis_predictions(act_label,
model_agent,
self_threshold,
partner_threshold,
return_acts=False):
act = deepcopy(act_label)
if 'labels' in act:
del act['labels']
if 'eval_labels' in act:
del act['eval_labels']
# SELF (as) pred
act.force_set('label_candidates', gend_utils.SELF_CANDS)
model_agent.observe(validate(act))
self_act = model_agent.act()
if not return_acts:
self_pred = get_threshold_preds(
self_act,
axis='self',
self_threshold=self_threshold,
partner_threshold=partner_threshold,
)
else:
self_pred = self_act
# PARTNER (to) pred
act.force_set('label_candidates', gend_utils.PARTNER_CANDS)
model_agent.observe(validate(act))
partner_act = model_agent.act()
if not return_acts:
partner_pred = get_threshold_preds(
partner_act,
axis='partner',
self_threshold=self_threshold,
partner_threshold=partner_threshold,
)
else:
partner_pred = partner_act
# ABOUT pred
act.force_set('label_candidates', gend_utils.ABOUT_CANDS)
model_agent.observe(validate(act))
about_act = model_agent.act()
if not return_acts:
about_pred = get_threshold_preds(
about_act,
axis='about',
self_threshold=self_threshold,
partner_threshold=partner_threshold,
)
else:
about_pred = about_act
return self_pred, partner_pred, about_pred
def _run_initial_turn(self) -> None:
"""
Show the image to the human and bot, and show the bot's response to the human.
"""
system_id = 'SYSTEM'
system_agent_idx = None
# Show the image to the human
image_act_for_human = {
'episode_done': False,
'id': system_id,
'text':
f"""Welcome! You'll now have a conversation with your partner.
<-- FIRST, YOUR PARTNER WILL SAY SOMETHING ABOUT THIS IMAGE TO YOUR LEFT.
Be sure to talk about this image a little bit before discussing other things!
""",
'task_data': {
'image_src': self.image_src
},
'agent_idx': system_agent_idx,
}
self.agent.observe(validate(image_act_for_human))
# Show the image to the bot
image_act = {
**self.image_act,
'episode_done': False,
'id': system_id,
'agent_idx': system_agent_idx,
}
self.bot.observe(validate(image_act))
del image_act['image']
# Don't save the image features to disk
# Have the bot respond
bot_first_act_raw = self.bot.act()
bot_first_act_raw = Message(
Compatibility.maybe_fix_act(
bot_first_act_raw)).json_safe_payload()
bot_first_act_raw['id'] = self.bot.agent_id
self.agent.observe(validate(bot_first_act_raw))
bot_first_act = {
'episode_done': False,
'id': bot_first_act_raw['id'],
'text': bot_first_act_raw['text'],
'agent_idx': 1,
}
# Record lines of dialogue
self.dialog.append(image_act)
self.dialog.append(bot_first_act)
def parley(self) -> None:
# random initialize human and model persona
if self.turn_cnt == 0:
self.p1, self.p2 = self.get_contexts()
if self.turn_cnt == 0 and self.p1 != '':
# add the context on to the first message to human
context_act = Message({
'id': 'context',
'text': self.p1,
'episode_done': False
})
# human agent observes his/her persona
self.agents[0].observe(validate(context_act))
try:
# human agent act first
act = deepcopy(self.agents[0].act())
except StopIteration:
self.reset()
self.finalize_episode()
self.turn_cnt = 0
return
self.acts[0] = act
if self.turn_cnt == 0 and self.p2 != '':
# add the context on to the first message to agent 1
context_act = Message({
'id': 'context',
'text': self.p2,
'episode_done': False
})
# model observe its persona
self.agents[1].observe(validate(context_act))
# add knowledge to the model observation
if 'text' in act:
act = self._add_knowledge_to_act(act)
# model observe human act and knowledge
self.agents[1].observe(validate(act))
# model agent act
self.acts[1] = self.agents[1].act()
# add the mdoel reply to the knowledge retriever's dialogue history
if 'text' in self.acts[1]:
self.knowledge_agent.observe(validate(self.acts[1]))
# human agent observes model act
self.agents[0].observe(validate(self.acts[1]))
self.update_counters()
self.turn_cnt += 1
if act['episode_done']:
self.finalize_episode()
self.turn_cnt = 0
def parley(self):
"""Alternate taking turns, until both agents have made a choice
(indicated by a turn starting with <selection>)
"""
if self.first_turn:
# Use NegotiationTeacher to load data for us
data = self.task.episodes[
self.num_negotiations % len(self.task.episodes)
].strip().split()
self.num_negotiations += 1
for agent, tag in zip(self.agents, ['input', 'partner_input']):
(book_cnt, book_val, hat_cnt,
hat_val, ball_cnt, ball_val) = get_tag(data, tag)
action = {}
action['text'] = WELCOME_MESSAGE.format(
book_cnt=book_cnt, book_val=book_val,
hat_cnt=hat_cnt, hat_val=hat_val,
ball_cnt=ball_cnt, ball_val=ball_val)
action['items'] = {"book_cnt": book_cnt, "book_val": book_val,
"hat_cnt": hat_cnt, "hat_val": hat_val,
"ball_cnt": ball_cnt, "ball_val": ball_val}
agent.observe(validate(action))
self.first_turn = False
else:
self.turns += 1
for _index, agent in enumerate(self.agents):
if agent in self.choices:
# This agent has already made a choice
continue
try:
act = agent.act(timeout=None)
except TypeError:
act = agent.act() # not MTurkAgent
for other_agent in self.agents:
if other_agent != agent:
other_agent.observe(validate(act))
if act["text"].startswith("<selection>") and self.turns > 1:
# Making a choice
self.choices[agent] = act["text"]
self.selection = True
if len(self.choices) == len(self.agents):
self.first_turn = True
self.episodeDone = True
elif act['episode_done']:
# Action is not selection but episode ended due to disconnection or timeout or returned hit
self.episodeDone = True
def parley(self):
"""
Loop between model and human.
"""
if self.cnt == 0:
self.context_id = self._get_new_intent()
self.acts = [None, None]
self.human_first = random.choice([0, 1])
# possibly get human act first
if self.cnt == 0 and not self.human_first:
self.acts[0] = Message({
'text': '__SILENCE__',
'episode_done': False
})
else:
try:
self.acts[0] = self.human_agent.act()
except StopIteration:
if self.agenttype != 'customer':
self.get_air_score()
print('[ CHAT DONE ]')
print('\n[ Preparing new chat... ]\n')
self.reset()
return
act = deepcopy(self.acts[0])
# add context to the model observation
act = self._add_context(act)
# model observes context and human (apprentice) act
self.model_agent.observe(validate(act))
# model agent act
self.acts[1] = self.model_agent.act()
# human (apprentice) agent observes model act
# remove encoder_states to prevent output
act = deepcopy(self.acts[1])
if 'encoder_states' in act:
del act['encoder_states']
self.human_agent.observe(validate(act))
self.update_counters()
self.cnt += 1
if self.episode_done():
print('[ CHAT DONE ]')
print('\n[ Preparing new chat... ]\n')
self.cnt = 0
self.model_agent.reset()
def parley(self):
from time import time
"""
Agent 0 goes first.
Alternate between the two agents.
"""
if self.turn_cnt == 0:
self.p1, self.p2 = self.get_contexts()
acts = self.acts
agents = self.agents
if self.turn_cnt == 0 and self.p1 != '':
# add the context on to the first message to agent 0
context_act = Message({
'id': 'context',
'text': self.p1,
'episode_done': False
})
agents[0].observe(validate(context_act))
try:
# .act() 에서 입력 받는데??
# parlai/agents/local_human/act
act = deepcopy(agents[0].act())
start = time()
except StopIteration:
self.reset()
self.finalize_episode()
self.turn_cnt = 0
return
acts[0] = act
if self.turn_cnt == 0 and self.p2 != '':
# add the context on to the first message to agent 1
context_act = Message({
'id': 'context',
'text': self.p2,
'episode_done': False
})
agents[1].observe(validate(context_act))
agents[1].observe(validate(act))
acts[1] = agents[1].act()
# agents : [localhuman, transformergenerator]
# [Display Response]
agents[0].observe(validate(acts[1]))
self.update_counters()
self.turn_cnt += 1
# from pprint import pprint
# pprint(acts)
if act['episode_done']:
self.finalize_episode()
self.turn_cnt = 0
print('답변 생성 시간 : {}'.format(time() - start))
def _run_initial_turn(self) -> None:
"""
Run the initial turn for both the human and the bot.
Optionally show the bot its persona. If we are in BST conversation mode, show 2
previous BST utterances to both the human and the bot; if we are in Meena-like
conversation mode, show "Hi!" to the human and the bot and let the bot respond
accordingly.
"""
control_msg = {"episode_done": False}
time.sleep(2)
coordinator_first_msg = {
'episode_done': False,
'id': 'Coordinator',
'text':
'Please chitchat with another worker for 6 turns as if you were catching up since last time you two spoke.',
'fake_start': True,
'agent_idx': 2,
'task_data': self.task_data,
}
self.agent.observe(validate(coordinator_first_msg))
time.sleep(2)
human_first_msg = {
'episode_done': False,
'id': self.agent.id,
'text': self.context_for_bot_prompt,
'fake_start': True,
'agent_idx': 0,
'task_data': self.task_data,
}
for k, v in control_msg.items():
human_first_msg[k] = v
# self.dialog.append(human_first_msg)
# self.agent.observe(validate(human_first_msg))
print(human_first_msg)
self.bot.observe(validate(human_first_msg))
first_bot_act = self.bot.act()
first_bot_act = Compatibility.maybe_fix_act(first_bot_act)
first_bot_act['id'] = 'THEY'
self.agent.observe(validate(first_bot_act))
bot_utterance_data = {
'agent_idx': 1,
'text': first_bot_act['text'],
'id': 'THEY',
}
self.dialog.append(bot_utterance_data)
def parley(self, human_input, history, cands):
"""
Agent 0 goes first.
Alternate between the two agents.
"""
#if self.turn_cnt == 0:
# self.p1, self.p2 = self.get_contexts()
self.turn_cnt = 0
self.p1 = history
self.p2 = history
acts = deepcopy(self.acts)
agents = deepcopy(self.agents)
agents[0].fixedCands_txt = cands
agents[1].fixedCands_txt = cands
if self.turn_cnt == 0 and self.p1 != '':
# add the context on to the first message to agent 0
context_act = Message({
'id': 'context',
'text': self.p1,
'episode_done': False
})
agents[0].observe(validate(context_act))
try:
print('[Human]: ' + human_input)
act = deepcopy(agents[0].act(human_input))
except StopIteration:
self.reset()
self.finalize_episode()
self.turn_cnt = 0
return
acts[0] = act
if self.turn_cnt == 0 and self.p2 != '':
# add the context on to the first message to agent 1
context_act = Message({
'id': 'context',
'text': self.p2,
'episode_done': False
})
agents[1].observe(validate(context_act))
agents[1].observe(validate(act))
acts[1] = agents[1].act()
agents[0].observe(validate(acts[1]))
self.update_counters()
self.turn_cnt += 1
if act['episode_done']:
self.finalize_episode()
self.turn_cnt = 0
return acts[1]['text_candidates'], acts[1]['sorted_scores']
def receive_stegotext(agent_id: int, text: str) -> bytes:
assert(state.agents is not None and len(state.agents) > agent_id) # Unknown agent otherwise
assert(not state.agent_ownership[agent_id]) # Can only receive stegotext for non-owned agents
observation = {'text': text, 'episode_done': False}
secret = state.agents[agent_id].receiveMessage(observation)
for i, agent in enumerate(state.agents):
if i == agent_id:
agent.self_observe(validate(observation))
else:
agent.observe(validate(observation))
if secret is None:
return None
else:
return secret
def parley(self):
"""
Agent 0 goes first.
Alternate between the two agents.
"""
if self.turn_cnt == 0:
self.p1, self.p2 = self.get_contexts()
acts = self.acts
agents = self.agents
if self.turn_cnt == 0 and self.p1 != '':
# add the context on to the first message to agent 0
context_act = Message({
'id': 'context',
'text': self.p1,
'episode_done': False
})
agents[0].observe(validate(context_act))
try:
act = deepcopy(agents[0].act())
except StopIteration:
self.reset()
self.finalize_episode()
self.turn_cnt = 0
return
acts[0] = act
if self.turn_cnt == 0 and self.p2 != '':
# add the context on to the first message to agent 1
context_act = Message({
'id': 'context',
'text': self.p2,
'episode_done': False
})
agents[1].observe(validate(context_act))
agents[1].observe(validate(act))
acts[1] = agents[1].act()
agents[0].observe(validate(acts[1]))
self.update_counters()
self.turn_cnt += 1
print("Bob:\t", acts[0]["text"])
print("Alice:\t", acts[1]["text"])
print()
if act['episode_done']:
self.finalize_episode()
self.turn_cnt = 0
def is_exact_match(self, act, ag, persona_data, tolerance=0):
if act['episode_done']:
return False
control_msg = {'episode_done': False}
control_msg['id'] = 'SYSTEM'
text = act['text']
if text not in ['', ' ', ' ', ' ']:
n_word_match = 0
for per in [persona_data]:
per_parse = per.split(' ')
regular_words = ['', ' ', 'I', 'I\'m', 'My', 'i']
for r_w in regular_words:
if r_w in per_parse:
per_parse.remove(r_w)
per_subseq = [
' '.join(per_parse[i:i + len(per_parse) - tolerance])
for i in range(tolerance + 1)
]
for pp in per_subseq:
if pp in ['', ' ', ' ', ' ']:
per_subseq.remove(pp)
n_word_match += sum([(paa in text) for paa in per_subseq])
if n_word_match > 0:
control_msg['text'] = (
'We found that you <b><span style="color:red">trivially '
'copied character descriptions</span></b>. Please '
'rephrase your message again.'
)
ag.observe(validate(control_msg))
return True
else:
return False
def is_close_match(self, act, ag, persona_data, tolerance=0.7):
if act['episode_done']:
return False
control_msg = {'episode_done': False}
control_msg['id'] = 'SYSTEM'
text = act['text']
if text not in ['', ' ', ' ', ' ']:
n_word_match = 0
per_parse = persona_data.split(' ')
regular_words = ['', ' ', 'I', 'I\'m', 'My', 'i']
for r_w in regular_words:
if r_w in per_parse:
per_parse.remove(r_w)
n_word_match += sum([word in text for word in per_parse])
if n_word_match / (len(per_parse) + 1) > tolerance:
control_msg['text'] = (
'We found that you <b><span style="color:red">'
'trivially copied character descriptions</span></b>. '
'Please rephrase your message again.'
)
ag.observe(validate(control_msg))
return True
else:
return False
def evaluate_persona(self):
if self.model_agent is not None:
other_persona = self.model_personas
else:
other_persona = self.other_agent.personas
fake_persona = self.eval_agent.personas_generator.get_persona()
while fake_persona == other_persona:
fake_persona = self.eval_agent.personas_generator.get_persona()
cand_text = []
for dt in [other_persona, fake_persona]:
if dt == other_persona:
is_correct = True
else:
is_correct = False
_text = ''
for s in dt:
_text += '<b><span style="color:blue">' + \
s.strip() + '</span></b><br>'
cand_text.append((is_correct, _text))
random.shuffle(cand_text)
control_msg = self.get_control_msg()
control_msg['text'] = PERSONA_MSG.format(cand_text[0][1],
cand_text[1][1])
control_msg['button_choices'] = '</ROUND>'.join(PERSONA_CHOICES)
self.eval_agent.observe(validate(control_msg))
act = self.eval_agent.act(timeout=self.max_resp_time)
timeout = self.check_timeout(act)
if timeout:
return False
self.persona_scores.append(cand_text[int(act['text']) - 1][0])
return True
def is_msg_tooshortlong(self, act, ag, th_min=3, th_max=20):
if act['episode_done']:
return False
control_msg = self.get_control_msg()
msg_len = len(act['text'].split(' '))
if msg_len < th_min:
control_msg['text'] = TOO_SHORT_MSG.format(th_min)
ag.observe(validate(control_msg))
return True
if msg_len > th_max:
control_msg['text'] = TOO_LONG_MSG.format(th_max)
ag.observe(validate(control_msg))
return True
return False
def is_exact_match(self, act, ag, tolerance=0):
if act['episode_done']:
return False
control_msg = {'episode_done': False}
control_msg['id'] = 'SYSTEM'
text = act['text']
if text not in ['', ' ', ' ', ' ']:
n_word_match = 0
for per in self.agentpersona_data:
per_parse = per.split(' ')
regular_words = ['', ' ', 'I', 'I\'m', 'My', 'i']
for r_w in regular_words:
if r_w in per_parse:
per_parse.remove(r_w)
per_subseq = [' '.join(per_parse[i:i + len(per_parse) -
tolerance]) for i in range(tolerance + 1)]
for pp in per_subseq:
if pp in ['', ' ', ' ', ' ']:
per_subseq.remove(pp)
n_word_match += sum([(paa in text) for paa in per_subseq])
if n_word_match > 0:
control_msg['text'] = COPIED_CHARACTER_MSG
self.agentobserve(validate(control_msg))
return True
else:
return False
def parley(self):
persona_idx, data = self.mturk_agent.persona_generator.pop_persona()
self.mturk_agent.persona_idx = persona_idx
self.mturk_agent.persona_data = data
persona_text = ''
for s in data:
persona_text += '<b><span style="color:blue">' \
'{}\n</span></b>'.format(s.strip())
self.mturk_agent.observe({
'id':
'SYSTEM',
'show_persona':
True,
'text':
ONBOARD_MSG + '<br>' + persona_text + '<br>'
})
act = self.mturk_agent.act(timeout=self.max_persona_time)
# timeout
if act['episode_done'] or (('text' in act
and act['text'] == TIMEOUT_MESSAGE)):
self.mturk_agent.persona_generator.push_persona(
self.mturk_agent.persona_idx)
self.mturk_agent.persona_generator.save_idx_stack()
self.episodeDone = True
return
if 'text' not in act:
control_msg = {'id': 'SYSTEM', 'text': WAITING_MSG}
self.mturk_agent.observe(validate(control_msg))
self.episodeDone = True
def run_conversation(mturk_manager, opt, workers):
mturk_manager.left_pane_refresh(task_config['task_description'])
mturk_agent = workers[0]
if (mturk_agent.worker_id in user_ids):
print("USER_ID: ", mturk_agent.worker_id, " DIALS: ",
user_ids[mturk_agent.worker_id])
else:
print("USER_ID: ", mturk_agent.worker_id, " DIALS: 0")
if (mturk_agent.worker_id
in user_ids) and user_ids[mturk_agent.worker_id] >= 15:
ad = {'episode_done': False}
ad['id'] = 'Restaurant bot'
ad['text'] = "We are closing this HIT, since you've already had over 15 dialogues with our restaurant bot in this session. We are very appreciated for your help. Welcome to join the next session."
mturk_agent.observe(validate(ad))
return
else:
world = QADataCollectionWorld(opt=opt, mturk_agent=mturk_agent)
btime = time.time()
world.parley()
etime = time.time()
logger.debug("DialTime: " + str(etime - btime))
if mturk_agent.worker_id not in user_ids:
user_ids[mturk_agent.worker_id] = 1
else:
user_ids[mturk_agent.worker_id] += 1
world.shutdown()
world.review_work()
return
def eval_or_shutdown(agent):
if self.model_agent is None and agent == self.other_agent:
control_msg = self.get_control_msg()
control_msg['text'] = OTHER_AGENT_FINISHED_MSG
self.other_agent.observe(validate(control_msg))
# mark eval agent done
self.eval_agent.mturk_manager.mark_workers_done(
[self.eval_agent])
# shutdown other agent
self.other_agent.shutdown()
else:
evaluations = [
self.evaluate_engagingness,
self.evaluate_interestingness,
self.evaluate_inquisitiveness,
self.evaluate_listening,
self.evaluate_repetitiveness,
self.evaluate_fluency,
self.evaluate_consistency,
self.evaluate_humanness,
self.evaluate_persona,
]
for evaluation in evaluations:
fin = evaluation()
if not fin:
return
return
def parley(self):
"""
Agent 0 goes first.
Alternate between the two agents.
"""
acts = self.acts
human_agent, model_agent = self.agents
# human act
act = deepcopy(human_agent.act())
self_pred, partner_pred, about_pred = get_axis_predictions(
act,
model_agent,
self_threshold=self.opt['self_threshold'],
partner_threshold=self.opt['partner_threshold'],
)
pred_text = f'SELF: {self_pred}\nPARTNER: {partner_pred}\nABOUT: {about_pred}'
acts[1] = {
'id': 'MDGender Classifier',
'text': pred_text,
'episode_done': True
}
human_agent.observe(validate(acts[1]))
self.update_counters()
def parley(self):
self.turn_index = (self.turn_index + 1) % 2
ad = { 'episode_done': False }
ad['id'] = self.__class__.TEST_ID
if self.turn_index == 0:
# Take a first turn
ad['text'] = self.TEST_TEXT_1
self.mturk_agent.observe(validate(ad))
self.response1 = self.mturk_agent.act()
if self.turn_index == 1:
# Complete after second turn
ad['text'] = self.TEST_TEXT_2
ad['episode_done'] = True # end of episode
self.mturk_agent.observe(validate(ad))
self.response2 = self.mturk_agent.act()
time.sleep(1)
self.episodeDone = True
def parley(self):
"""For each agent, get an observation of the last action each of the
other agents took. Then take an action yourself.
"""
acts = self.acts
for index, agent in enumerate(self.agents):
try:
acts[index] = agent.act(timeout=None)
except TypeError:
acts[index] = agent.act() # not MTurkAgent
if acts[index]['episode_done']:
self.episodeDone = True
for other_agent in self.agents:
if other_agent != agent:
other_agent.observe(validate(acts[index]))
def parley(self):
"""For each agent, act, then force other agents to observe your act
"""
acts = self.acts
for index, agent in enumerate(self.agents):
try:
acts[index] = agent.act(timeout=None)
except TypeError:
acts[index] = agent.act() # not MTurkAgent
if acts[index]['episode_done']:
self.episodeDone = True
for other_agent in self.agents:
if other_agent != agent:
other_agent.observe(validate(acts[index]))
self.rounds += 1
if self.rounds >= 2:
time.sleep(2)
self.episodeDone = True
def parley(self):
self.task_world.parley()
ad = {}
# Show the dialog model's response to the context, and ask the turker to rate the response
ad['id'] = self.__class__.evaluator_agent_id
ad['text'] = (
self.task_world.get_acts()[0]['text'] + "\n\n" +
"How would you rate the following response (from 0 to 10):\n\n" +
self.task_world.get_acts()[1]['text'])
# TODO: deal with multi-turn dialogs, for now we will just deal
# with 1-turn dialogs in this task.
ad['episode_done'] = True # self.world.episode_done()
self.mturk_agent.observe(validate(ad))
rating = self.mturk_agent.act()
self.episodeDone = True
