def parley(self): if self.cnt == 0: self.topic = self.get_new_topic() self.acts = [None, None] self.human_first = random.choice([0, 1]) # possibly get human act first if self.cnt == 0 and not self.human_first: self.acts[0] = act = {'text': '', 'episode_done': False} act = self.acts[0] else: self.acts[0] = self.human_agent.act() act = deepcopy(self.acts[0]) # model agent observe if self.cnt == 0 and self.topic != NO_TOPIC: # add the chosen_topic to the message act['chosen_topic'] = self.topic act['text'] = '\n'.join([self.topic, act.get('text', 'hi')]) self.model_agent.observe(validate(act)) # model agent act self.acts[1] = self.model_agent.act() # human agent observe self.human_agent.observe(validate(self.acts[1])) self.update_counters() self.cnt += 1 if self.episode_done(): print('[ CHAT DONE ]') print('\n[ Preparing new chat... ]\n') self.cnt = 0 self.model_agent.reset()
def parley(self): if self.episode_done(): self.cnt = 0 agents = self.get_agents() for a in agents: a.reset() if self.cnt == 0: self.acts = [None, None] # choose speaking order: if random.choice([0, 1]): self.agents_ordered = [self.agents[0], self.agents[1]] else: self.agents_ordered = [self.agents[1], self.agents[0]] self.contexts = self.get_contexts() # initial context for i in range(0, 2): context = { 'text': self.contexts[i], 'episode_done': False, 'id': 'context', } self.acts[1 - i] = context self.agents_ordered[i].observe(validate(context)) else: # do regular loop acts = self.acts agents = self.agents_ordered acts[0] = agents[0].act() agents[1].observe(validate(acts[0])) acts[1] = agents[1].act() agents[0].observe(validate(acts[1])) self.update_counters() self.cnt += 1
def parley(self): # only pick topic if your ID is PERSON_1 if self.role == 'PERSON_1': seen = random.choice([True, False]) num = random.choice([2, 3]) topics = self.mturk_agent.topics_generator.get_topics(seen=seen, num=num) self.mturk_agent.observe( validate( {'id': 'SYSTEM', 'text': PICK_TOPIC_MSG, 'relevant_topics': topics} ) ) topic_act = self.mturk_agent.act(timeout=self.max_choice_time) timed_out = self.check_timeout(topic_act) if timed_out: return pick_msg = AFTER_PICK_TOPIC_MSG self.mturk_agent.observe({'id': 'SYSTEM', 'text': pick_msg}) self.mturk_agent.chosen_topic = topic_act['text'] self.mturk_agent.topic_choices = topics self.mturk_agent.seen = seen self.mturk_agent.observe(validate({'id': 'SYSTEM', 'wait_msg': True})) else: self.mturk_agent.observe(validate({'id': 'SYSTEM', 'wait_msg': True})) return
def eval_or_shutdown(agent): if self.model_agent is None and agent == self.other_agent: control_msg = {'episode_done': False} control_msg['id'] = 'SYSTEM' control_msg['text'] = OTHER_AGENT_FINISHED_MSG self.other_agent.observe(validate(control_msg)) # mark eval agent done self.eval_agent.mturk_manager.mark_workers_done([self.eval_agent]) # shutdown other agent self.other_agent.shutdown() else: control_msg = {'episode_done': False} control_msg['id'] = 'SYSTEM' # General mark for this convo control_msg['text'] = GMARK_MSG control_msg['general_mark_score'] = True self.eval_agent.observe(validate(control_msg)) act = self.eval_agent.act(timeout=self.max_resp_time) timeout = self.check_timeout(act) if timeout: return while act['text'] not in self.ratings: control_msg['text'] = NAN_MSG self.eval_agent.observe(validate(control_msg)) act = self.eval_agent.act(timeout=self.max_resp_time) if 'text' in act and act['text'] in self.ratings: self.gmark_score = int(act['text'])
def parley(self): self.turn_index = (self.turn_index + 1) % 2; # Each turn starts from the QA Collector agent ad = { 'episode_done': False } ad['id'] = self.__class__.collector_agent_id if self.turn_index == 0: # At the first turn, the QA Collector agent provides the context and # prompts the turker to ask a question regarding the context # Get context from SQuAD teacher agent qa = self.task.act() context = '\n'.join(qa['text'].split('\n')[:-1]) # Wrap the context with a prompt telling the turker what to do next ad['text'] = (context + '\n\nPlease provide a question given this context.') self.mturk_agent.observe(validate(ad)) self.question = self.mturk_agent.act() # Can log the turker's question here if self.turn_index == 1: # At the second turn, the QA Collector collects the turker's question from the first turn, # and then prompts the turker to provide the answer # A prompt telling the turker what to do next ad['text'] = 'Thanks. And what is the answer to your question?' ad['episode_done'] = True # end of episode self.mturk_agent.observe(validate(ad)) self.answer = self.mturk_agent.act() # Can log the turker's answer here self.episodeDone = True
def parley(self): """Agent 0 goes first. Alternate between the two agents.""" if self.cnt == 0: self.p1, self.p2 = self.get_new_personas() acts = self.acts agents = self.agents if self.cnt == 0: # add the persona on to the first message to agent 0 act = {} act['text'] = self.p1 act['episode_done'] = False act['id'] = 'persona' agents[0].observe(validate(act)) act = deepcopy(agents[0].act()) if self.cnt == 0: # add the persona on to the first message to agent 1 act['text'] = self.p2 + act.get('text', 'hi') print("gave bot its persona!") print(act) agents[1].observe(validate(act)) else: agents[1].observe(validate(act)) acts[1] = agents[1].act() agents[0].observe(validate(acts[1])) self.update_counters() self.cnt += 1 if self.episode_done(): print("CHAT DONE ") print("\n... preparing new chat... \n") self.cnt = 0
def evaluate_repetitiveness(self): control_msg = self.get_control_msg() control_msg['text'] = REPETITIVENESS_MSGS[0] control_msg['button_choices'] = '</ROUND>'.join(REPETITIVENESS_CHOICES) self.eval_agent.observe(validate(control_msg)) act = self.eval_agent.act(timeout=self.max_resp_time) timeout = self.check_timeout(act) if timeout: return False act_choice = REPETITIVENESS_CHOICES.index(act.get('text')) self.repetitiveness_scores.append(act_choice) if ASK_DETAILED and act_choice != 2: control_msg = self.get_control_msg() control_msg['text'] = REPETITIVENESS_MSGS[1] control_msg['good_rounds'] = True control_msg['rounds'] = '</ROUND>'.join(self.dialog_list) self.eval_agent.observe(validate(control_msg)) act = self.eval_agent.act(timeout=self.max_resp_time) timeout = self.check_timeout(act) if timeout: return False if 'text' in act: self.repetitiveness_scores.append( [int(x) - 1 for x in act['text'].split(',')]) return True
def parley(self): """Alternate taking turns, until both agents have made a choice (indicated by a turn starting with <selection>) """ if self.first_turn: # Use NegotiationTeacher to load data for us data = self.task.episodes[self.num_negotiations % len(self.task.episodes)].strip().split() self.num_negotiations += 1 for agent, tag in zip(self.agents, ['input', 'partner_input']): (book_cnt, book_val, hat_cnt, hat_val, ball_cnt, ball_val) = get_tag(data, tag) action = {} action['text'] = WELCOME_MESSAGE.format(book_cnt=book_cnt, book_val=book_val, hat_cnt=hat_cnt, hat_val=hat_val, ball_cnt=ball_cnt, ball_val=ball_val) action['items'] = { "book_cnt": book_cnt, "book_val": book_val, "hat_cnt": hat_cnt, "hat_val": hat_val, "ball_cnt": ball_cnt, "ball_val": ball_val } agent.observe(validate(action)) self.first_turn = False else: self.turns += 1 for _index, agent in enumerate(self.agents): if agent in self.choices: # This agent has already made a choice continue try: act = agent.act(timeout=None) except TypeError: act = agent.act() # not MTurkAgent for other_agent in self.agents: if other_agent != agent: other_agent.observe(validate(act)) if act["text"].startswith("<selection>") and self.turns > 1: # Making a choice self.choices[agent] = act["text"] self.selection = True if len(self.choices) == len(self.agents): self.first_turn = True self.episodeDone = True elif act['episode_done']: # Action is not selection but episode ended due to # disconnection or timeout or returned hit self.episodeDone = True
def parley(self): """ Agent 0 goes first. Alternate between the two agents. """ if self.turn_cnt == 0: self.p1, self.p2 = self.get_contexts() acts = self.acts agents = self.agents if self.turn_cnt == 0 and self.p1 != '': # add the context on to the first message to agent 0 context_act = Message( {'id': 'context', 'text': self.p1, 'episode_done': False} ) agents[0].observe(validate(context_act)) act = deepcopy(agents[0].act()) acts[0] = act if self.turn_cnt == 0 and self.p2 != '': # add the context on to the first message to agent 1 context_act = Message( {'id': 'context', 'text': self.p2, 'episode_done': False} ) agents[1].observe(validate(context_act)) agents[1].observe(validate(act)) acts[1] = agents[1].act() agents[0].observe(validate(acts[1])) self.update_counters() self.turn_cnt += 1 if act['episode_done']: self.finalize_episode() self.turn_cnt = 0
def parley_observe(self, input_statement, output): """ Agent 0 goes first. Alternate between the two agents. """ if self.turn_cnt == 0: self.p1, self.p2 = self.get_contexts() acts = self.acts agents = self.agents if self.turn_cnt == 0 and self.p1 != '': # add the context on to the first message to agent 0 context_act = Message( {'id': 'context', 'text': self.p1, 'episode_done': False} ) agents[0].observe(validate(context_act)) if self.turn_cnt == 0 and self.p2 != '': # add the context on to the first message to agent 1 context_act = Message( {'id': 'context', 'text': self.p2, 'episode_done': False} ) agents[1].observe(validate(context_act)) agents[1].history.add_reply(input_statement) agents[1].history.add_reply(output) print(agents[1].history.get_history_str()) print(agents[1].history.get_history_vec()) self.update_counters() self.turn_cnt += 1
def evaluate_consistency(self): control_msg = self.get_control_msg() control_msg['text'] = CONSISTENCY_MSGS[0] control_msg['button_choices'] = '</ROUND>'.join(CONSISTENCY_CHOICES) self.eval_agent.observe(validate(control_msg)) act = self.eval_agent.act(timeout=self.max_resp_time) timeout = self.check_timeout(act) if timeout: return False act_choice = CONSISTENCY_CHOICES.index(act.get('text')) self.consistency_scores.append(act_choice) if act_choice != 0: control_msg = self.get_control_msg() control_msg['text'] = CONSISTENCY_MSGS[1] control_msg['good_rounds'] = True control_msg['rounds'] = '</ROUND>'.join(self.dialog_list) self.eval_agent.observe(validate(control_msg)) act = self.eval_agent.act(timeout=self.max_resp_time) timeout = self.check_timeout(act) if timeout: return False if 'text' in act: self.consistency_scores.append( [int(x) - 1 for x in act['text'].split(',')]) return True
def parley(self): if self.episode_done(): self.turn_cnt = 0 self.episode_cnt += 1 self.contexts = None self.seed_utterances = None agents = self.get_agents() for a in agents: a.reset() if self.turn_cnt == 0: self.acts = [None, None] # get the beginning of the conversation, which can include contexts # and/or any number of starting messages self.contexts = self.get_contexts() self.seed_utterances = self._get_seed_utt_acts( self.episode_cnt, self.agents ) if self.contexts: assert len(self.contexts) == 2 # initial context for i in range(0, 2): context = { 'text': self.contexts[i], 'episode_done': False, 'id': 'context', } self.acts[i] = context self.agents[i].observe(validate(context)) # clear contexts so they are only added once per episode self.contexts = None elif self.seed_utterances: # pop the next two seed messages (there may be less or more than 2 total) utts = self.seed_utterances[:2] self.seed_utterances = self.seed_utterances[2:] # process the turn for i in [0, 1]: # if we have a seed utterance, add it to the conversation if len(utts) > i: self.acts[i] = utts[i] if hasattr(self.agents[i], 'self_observe'): self.agents[i].self_observe(self.acts[i]) else: self.acts[i] = self.agents[i].act() self.agents[1 - i].observe(validate(self.acts[i])) else: # do regular loop acts = self.acts agents = self.agents acts[0] = agents[0].act() agents[1].observe(validate(acts[0])) acts[1] = agents[1].act() agents[0].observe(validate(acts[1])) self.update_counters() self.turn_cnt += 1
def parley(self): """ Loop between wizard and apprentice. Adds knowledge to the wizard observations. Assumes that the model agent is the wizard model. """ if self.cnt == 0: self.topic = self._get_new_topic() self.acts = [None, None] if self.topic != NO_TOPIC: self.human_first = random.choice([0, 1]) else: self.human_first = 1 # possibly get human act first if self.cnt == 0 and not self.human_first: self.acts[0] = act = Message({'text': '', 'episode_done': False}) act = self.acts[0] else: self.acts[0] = self.human_agent.act() act = deepcopy(self.acts[0]) # model agent observe if self.cnt == 0 and self.topic != NO_TOPIC: # add the chosen_topic to the message act['chosen_topic'] = self.topic act.force_set('text', '\n'.join([self.topic, act.get('text', 'hi')])) # add knowledge to the model observation act = self._add_knowledge_to_act(act) # model observes knowledge and human (apprentice) act self.model_agent.observe(validate(act)) # model agent act self.acts[1] = self.model_agent.act() # add the model reply to the knowledge retriever's dialogue history self.knowledge_agent.observe(self.acts[1], actor_id='wizard') # human (apprentice) agent observes model act self.human_agent.observe(validate(self.acts[1])) self.update_counters() self.cnt += 1 if self.episode_done(): print('[ CHAT DONE ]') print('\n[ Preparing new chat... ]\n') self.cnt = 0 self.model_agent.reset()
def get_axis_predictions(act_label, model_agent, self_threshold, partner_threshold, return_acts=False): act = deepcopy(act_label) if 'labels' in act: del act['labels'] if 'eval_labels' in act: del act['eval_labels'] # SELF (as) pred act.force_set('label_candidates', gend_utils.SELF_CANDS) model_agent.observe(validate(act)) self_act = model_agent.act() if not return_acts: self_pred = get_threshold_preds( self_act, axis='self', self_threshold=self_threshold, partner_threshold=partner_threshold, ) else: self_pred = self_act # PARTNER (to) pred act.force_set('label_candidates', gend_utils.PARTNER_CANDS) model_agent.observe(validate(act)) partner_act = model_agent.act() if not return_acts: partner_pred = get_threshold_preds( partner_act, axis='partner', self_threshold=self_threshold, partner_threshold=partner_threshold, ) else: partner_pred = partner_act # ABOUT pred act.force_set('label_candidates', gend_utils.ABOUT_CANDS) model_agent.observe(validate(act)) about_act = model_agent.act() if not return_acts: about_pred = get_threshold_preds( about_act, axis='about', self_threshold=self_threshold, partner_threshold=partner_threshold, ) else: about_pred = about_act return self_pred, partner_pred, about_pred
def _run_initial_turn(self) -> None: """ Show the image to the human and bot, and show the bot's response to the human. """ system_id = 'SYSTEM' system_agent_idx = None # Show the image to the human image_act_for_human = { 'episode_done': False, 'id': system_id, 'text': f"""Welcome! You'll now have a conversation with your partner. <-- FIRST, YOUR PARTNER WILL SAY SOMETHING ABOUT THIS IMAGE TO YOUR LEFT. Be sure to talk about this image a little bit before discussing other things! """, 'task_data': { 'image_src': self.image_src }, 'agent_idx': system_agent_idx, } self.agent.observe(validate(image_act_for_human)) # Show the image to the bot image_act = { **self.image_act, 'episode_done': False, 'id': system_id, 'agent_idx': system_agent_idx, } self.bot.observe(validate(image_act)) del image_act['image'] # Don't save the image features to disk # Have the bot respond bot_first_act_raw = self.bot.act() bot_first_act_raw = Message( Compatibility.maybe_fix_act( bot_first_act_raw)).json_safe_payload() bot_first_act_raw['id'] = self.bot.agent_id self.agent.observe(validate(bot_first_act_raw)) bot_first_act = { 'episode_done': False, 'id': bot_first_act_raw['id'], 'text': bot_first_act_raw['text'], 'agent_idx': 1, } # Record lines of dialogue self.dialog.append(image_act) self.dialog.append(bot_first_act)
def parley(self) -> None: # random initialize human and model persona if self.turn_cnt == 0: self.p1, self.p2 = self.get_contexts() if self.turn_cnt == 0 and self.p1 != '': # add the context on to the first message to human context_act = Message({ 'id': 'context', 'text': self.p1, 'episode_done': False }) # human agent observes his/her persona self.agents[0].observe(validate(context_act)) try: # human agent act first act = deepcopy(self.agents[0].act()) except StopIteration: self.reset() self.finalize_episode() self.turn_cnt = 0 return self.acts[0] = act if self.turn_cnt == 0 and self.p2 != '': # add the context on to the first message to agent 1 context_act = Message({ 'id': 'context', 'text': self.p2, 'episode_done': False }) # model observe its persona self.agents[1].observe(validate(context_act)) # add knowledge to the model observation if 'text' in act: act = self._add_knowledge_to_act(act) # model observe human act and knowledge self.agents[1].observe(validate(act)) # model agent act self.acts[1] = self.agents[1].act() # add the mdoel reply to the knowledge retriever's dialogue history if 'text' in self.acts[1]: self.knowledge_agent.observe(validate(self.acts[1])) # human agent observes model act self.agents[0].observe(validate(self.acts[1])) self.update_counters() self.turn_cnt += 1 if act['episode_done']: self.finalize_episode() self.turn_cnt = 0
def parley(self): """Alternate taking turns, until both agents have made a choice (indicated by a turn starting with <selection>) """ if self.first_turn: # Use NegotiationTeacher to load data for us data = self.task.episodes[ self.num_negotiations % len(self.task.episodes) ].strip().split() self.num_negotiations += 1 for agent, tag in zip(self.agents, ['input', 'partner_input']): (book_cnt, book_val, hat_cnt, hat_val, ball_cnt, ball_val) = get_tag(data, tag) action = {} action['text'] = WELCOME_MESSAGE.format( book_cnt=book_cnt, book_val=book_val, hat_cnt=hat_cnt, hat_val=hat_val, ball_cnt=ball_cnt, ball_val=ball_val) action['items'] = {"book_cnt": book_cnt, "book_val": book_val, "hat_cnt": hat_cnt, "hat_val": hat_val, "ball_cnt": ball_cnt, "ball_val": ball_val} agent.observe(validate(action)) self.first_turn = False else: self.turns += 1 for _index, agent in enumerate(self.agents): if agent in self.choices: # This agent has already made a choice continue try: act = agent.act(timeout=None) except TypeError: act = agent.act() # not MTurkAgent for other_agent in self.agents: if other_agent != agent: other_agent.observe(validate(act)) if act["text"].startswith("<selection>") and self.turns > 1: # Making a choice self.choices[agent] = act["text"] self.selection = True if len(self.choices) == len(self.agents): self.first_turn = True self.episodeDone = True elif act['episode_done']: # Action is not selection but episode ended due to disconnection or timeout or returned hit self.episodeDone = True
def parley(self): """ Loop between model and human. """ if self.cnt == 0: self.context_id = self._get_new_intent() self.acts = [None, None] self.human_first = random.choice([0, 1]) # possibly get human act first if self.cnt == 0 and not self.human_first: self.acts[0] = Message({ 'text': '__SILENCE__', 'episode_done': False }) else: try: self.acts[0] = self.human_agent.act() except StopIteration: if self.agenttype != 'customer': self.get_air_score() print('[ CHAT DONE ]') print('\n[ Preparing new chat... ]\n') self.reset() return act = deepcopy(self.acts[0]) # add context to the model observation act = self._add_context(act) # model observes context and human (apprentice) act self.model_agent.observe(validate(act)) # model agent act self.acts[1] = self.model_agent.act() # human (apprentice) agent observes model act # remove encoder_states to prevent output act = deepcopy(self.acts[1]) if 'encoder_states' in act: del act['encoder_states'] self.human_agent.observe(validate(act)) self.update_counters() self.cnt += 1 if self.episode_done(): print('[ CHAT DONE ]') print('\n[ Preparing new chat... ]\n') self.cnt = 0 self.model_agent.reset()
def parley(self): from time import time """ Agent 0 goes first. Alternate between the two agents. """ if self.turn_cnt == 0: self.p1, self.p2 = self.get_contexts() acts = self.acts agents = self.agents if self.turn_cnt == 0 and self.p1 != '': # add the context on to the first message to agent 0 context_act = Message({ 'id': 'context', 'text': self.p1, 'episode_done': False }) agents[0].observe(validate(context_act)) try: # .act() 에서 입력 받는데?? # parlai/agents/local_human/act act = deepcopy(agents[0].act()) start = time() except StopIteration: self.reset() self.finalize_episode() self.turn_cnt = 0 return acts[0] = act if self.turn_cnt == 0 and self.p2 != '': # add the context on to the first message to agent 1 context_act = Message({ 'id': 'context', 'text': self.p2, 'episode_done': False }) agents[1].observe(validate(context_act)) agents[1].observe(validate(act)) acts[1] = agents[1].act() # agents : [localhuman, transformergenerator] # [Display Response] agents[0].observe(validate(acts[1])) self.update_counters() self.turn_cnt += 1 # from pprint import pprint # pprint(acts) if act['episode_done']: self.finalize_episode() self.turn_cnt = 0 print('답변 생성 시간 : {}'.format(time() - start))
def _run_initial_turn(self) -> None: """ Run the initial turn for both the human and the bot. Optionally show the bot its persona. If we are in BST conversation mode, show 2 previous BST utterances to both the human and the bot; if we are in Meena-like conversation mode, show "Hi!" to the human and the bot and let the bot respond accordingly. """ control_msg = {"episode_done": False} time.sleep(2) coordinator_first_msg = { 'episode_done': False, 'id': 'Coordinator', 'text': 'Please chitchat with another worker for 6 turns as if you were catching up since last time you two spoke.', 'fake_start': True, 'agent_idx': 2, 'task_data': self.task_data, } self.agent.observe(validate(coordinator_first_msg)) time.sleep(2) human_first_msg = { 'episode_done': False, 'id': self.agent.id, 'text': self.context_for_bot_prompt, 'fake_start': True, 'agent_idx': 0, 'task_data': self.task_data, } for k, v in control_msg.items(): human_first_msg[k] = v # self.dialog.append(human_first_msg) # self.agent.observe(validate(human_first_msg)) print(human_first_msg) self.bot.observe(validate(human_first_msg)) first_bot_act = self.bot.act() first_bot_act = Compatibility.maybe_fix_act(first_bot_act) first_bot_act['id'] = 'THEY' self.agent.observe(validate(first_bot_act)) bot_utterance_data = { 'agent_idx': 1, 'text': first_bot_act['text'], 'id': 'THEY', } self.dialog.append(bot_utterance_data)
def parley(self, human_input, history, cands): """ Agent 0 goes first. Alternate between the two agents. """ #if self.turn_cnt == 0: # self.p1, self.p2 = self.get_contexts() self.turn_cnt = 0 self.p1 = history self.p2 = history acts = deepcopy(self.acts) agents = deepcopy(self.agents) agents[0].fixedCands_txt = cands agents[1].fixedCands_txt = cands if self.turn_cnt == 0 and self.p1 != '': # add the context on to the first message to agent 0 context_act = Message({ 'id': 'context', 'text': self.p1, 'episode_done': False }) agents[0].observe(validate(context_act)) try: print('[Human]: ' + human_input) act = deepcopy(agents[0].act(human_input)) except StopIteration: self.reset() self.finalize_episode() self.turn_cnt = 0 return acts[0] = act if self.turn_cnt == 0 and self.p2 != '': # add the context on to the first message to agent 1 context_act = Message({ 'id': 'context', 'text': self.p2, 'episode_done': False }) agents[1].observe(validate(context_act)) agents[1].observe(validate(act)) acts[1] = agents[1].act() agents[0].observe(validate(acts[1])) self.update_counters() self.turn_cnt += 1 if act['episode_done']: self.finalize_episode() self.turn_cnt = 0 return acts[1]['text_candidates'], acts[1]['sorted_scores']
def receive_stegotext(agent_id: int, text: str) -> bytes: assert(state.agents is not None and len(state.agents) > agent_id) # Unknown agent otherwise assert(not state.agent_ownership[agent_id]) # Can only receive stegotext for non-owned agents observation = {'text': text, 'episode_done': False} secret = state.agents[agent_id].receiveMessage(observation) for i, agent in enumerate(state.agents): if i == agent_id: agent.self_observe(validate(observation)) else: agent.observe(validate(observation)) if secret is None: return None else: return secret
def parley(self): """ Agent 0 goes first. Alternate between the two agents. """ if self.turn_cnt == 0: self.p1, self.p2 = self.get_contexts() acts = self.acts agents = self.agents if self.turn_cnt == 0 and self.p1 != '': # add the context on to the first message to agent 0 context_act = Message({ 'id': 'context', 'text': self.p1, 'episode_done': False }) agents[0].observe(validate(context_act)) try: act = deepcopy(agents[0].act()) except StopIteration: self.reset() self.finalize_episode() self.turn_cnt = 0 return acts[0] = act if self.turn_cnt == 0 and self.p2 != '': # add the context on to the first message to agent 1 context_act = Message({ 'id': 'context', 'text': self.p2, 'episode_done': False }) agents[1].observe(validate(context_act)) agents[1].observe(validate(act)) acts[1] = agents[1].act() agents[0].observe(validate(acts[1])) self.update_counters() self.turn_cnt += 1 print("Bob:\t", acts[0]["text"]) print("Alice:\t", acts[1]["text"]) print() if act['episode_done']: self.finalize_episode() self.turn_cnt = 0
def is_exact_match(self, act, ag, persona_data, tolerance=0): if act['episode_done']: return False control_msg = {'episode_done': False} control_msg['id'] = 'SYSTEM' text = act['text'] if text not in ['', ' ', ' ', ' ']: n_word_match = 0 for per in [persona_data]: per_parse = per.split(' ') regular_words = ['', ' ', 'I', 'I\'m', 'My', 'i'] for r_w in regular_words: if r_w in per_parse: per_parse.remove(r_w) per_subseq = [ ' '.join(per_parse[i:i + len(per_parse) - tolerance]) for i in range(tolerance + 1) ] for pp in per_subseq: if pp in ['', ' ', ' ', ' ']: per_subseq.remove(pp) n_word_match += sum([(paa in text) for paa in per_subseq]) if n_word_match > 0: control_msg['text'] = ( 'We found that you <b><span style="color:red">trivially ' 'copied character descriptions</span></b>. Please ' 'rephrase your message again.' ) ag.observe(validate(control_msg)) return True else: return False
def is_close_match(self, act, ag, persona_data, tolerance=0.7): if act['episode_done']: return False control_msg = {'episode_done': False} control_msg['id'] = 'SYSTEM' text = act['text'] if text not in ['', ' ', ' ', ' ']: n_word_match = 0 per_parse = persona_data.split(' ') regular_words = ['', ' ', 'I', 'I\'m', 'My', 'i'] for r_w in regular_words: if r_w in per_parse: per_parse.remove(r_w) n_word_match += sum([word in text for word in per_parse]) if n_word_match / (len(per_parse) + 1) > tolerance: control_msg['text'] = ( 'We found that you <b><span style="color:red">' 'trivially copied character descriptions</span></b>. ' 'Please rephrase your message again.' ) ag.observe(validate(control_msg)) return True else: return False
def evaluate_persona(self): if self.model_agent is not None: other_persona = self.model_personas else: other_persona = self.other_agent.personas fake_persona = self.eval_agent.personas_generator.get_persona() while fake_persona == other_persona: fake_persona = self.eval_agent.personas_generator.get_persona() cand_text = [] for dt in [other_persona, fake_persona]: if dt == other_persona: is_correct = True else: is_correct = False _text = '' for s in dt: _text += '<b><span style="color:blue">' + \ s.strip() + '</span></b><br>' cand_text.append((is_correct, _text)) random.shuffle(cand_text) control_msg = self.get_control_msg() control_msg['text'] = PERSONA_MSG.format(cand_text[0][1], cand_text[1][1]) control_msg['button_choices'] = '</ROUND>'.join(PERSONA_CHOICES) self.eval_agent.observe(validate(control_msg)) act = self.eval_agent.act(timeout=self.max_resp_time) timeout = self.check_timeout(act) if timeout: return False self.persona_scores.append(cand_text[int(act['text']) - 1][0]) return True
def is_msg_tooshortlong(self, act, ag, th_min=3, th_max=20): if act['episode_done']: return False control_msg = self.get_control_msg() msg_len = len(act['text'].split(' ')) if msg_len < th_min: control_msg['text'] = TOO_SHORT_MSG.format(th_min) ag.observe(validate(control_msg)) return True if msg_len > th_max: control_msg['text'] = TOO_LONG_MSG.format(th_max) ag.observe(validate(control_msg)) return True return False
def is_exact_match(self, act, ag, tolerance=0): if act['episode_done']: return False control_msg = {'episode_done': False} control_msg['id'] = 'SYSTEM' text = act['text'] if text not in ['', ' ', ' ', ' ']: n_word_match = 0 for per in self.agentpersona_data: per_parse = per.split(' ') regular_words = ['', ' ', 'I', 'I\'m', 'My', 'i'] for r_w in regular_words: if r_w in per_parse: per_parse.remove(r_w) per_subseq = [' '.join(per_parse[i:i + len(per_parse) - tolerance]) for i in range(tolerance + 1)] for pp in per_subseq: if pp in ['', ' ', ' ', ' ']: per_subseq.remove(pp) n_word_match += sum([(paa in text) for paa in per_subseq]) if n_word_match > 0: control_msg['text'] = COPIED_CHARACTER_MSG self.agentobserve(validate(control_msg)) return True else: return False
def parley(self): persona_idx, data = self.mturk_agent.persona_generator.pop_persona() self.mturk_agent.persona_idx = persona_idx self.mturk_agent.persona_data = data persona_text = '' for s in data: persona_text += '<b><span style="color:blue">' \ '{}\n</span></b>'.format(s.strip()) self.mturk_agent.observe({ 'id': 'SYSTEM', 'show_persona': True, 'text': ONBOARD_MSG + '<br>' + persona_text + '<br>' }) act = self.mturk_agent.act(timeout=self.max_persona_time) # timeout if act['episode_done'] or (('text' in act and act['text'] == TIMEOUT_MESSAGE)): self.mturk_agent.persona_generator.push_persona( self.mturk_agent.persona_idx) self.mturk_agent.persona_generator.save_idx_stack() self.episodeDone = True return if 'text' not in act: control_msg = {'id': 'SYSTEM', 'text': WAITING_MSG} self.mturk_agent.observe(validate(control_msg)) self.episodeDone = True
def run_conversation(mturk_manager, opt, workers): mturk_manager.left_pane_refresh(task_config['task_description']) mturk_agent = workers[0] if (mturk_agent.worker_id in user_ids): print("USER_ID: ", mturk_agent.worker_id, " DIALS: ", user_ids[mturk_agent.worker_id]) else: print("USER_ID: ", mturk_agent.worker_id, " DIALS: 0") if (mturk_agent.worker_id in user_ids) and user_ids[mturk_agent.worker_id] >= 15: ad = {'episode_done': False} ad['id'] = 'Restaurant bot' ad['text'] = "We are closing this HIT, since you've already had over 15 dialogues with our restaurant bot in this session. We are very appreciated for your help. Welcome to join the next session." mturk_agent.observe(validate(ad)) return else: world = QADataCollectionWorld(opt=opt, mturk_agent=mturk_agent) btime = time.time() world.parley() etime = time.time() logger.debug("DialTime: " + str(etime - btime)) if mturk_agent.worker_id not in user_ids: user_ids[mturk_agent.worker_id] = 1 else: user_ids[mturk_agent.worker_id] += 1 world.shutdown() world.review_work() return
def eval_or_shutdown(agent): if self.model_agent is None and agent == self.other_agent: control_msg = self.get_control_msg() control_msg['text'] = OTHER_AGENT_FINISHED_MSG self.other_agent.observe(validate(control_msg)) # mark eval agent done self.eval_agent.mturk_manager.mark_workers_done( [self.eval_agent]) # shutdown other agent self.other_agent.shutdown() else: evaluations = [ self.evaluate_engagingness, self.evaluate_interestingness, self.evaluate_inquisitiveness, self.evaluate_listening, self.evaluate_repetitiveness, self.evaluate_fluency, self.evaluate_consistency, self.evaluate_humanness, self.evaluate_persona, ] for evaluation in evaluations: fin = evaluation() if not fin: return return
def parley(self): """ Agent 0 goes first. Alternate between the two agents. """ acts = self.acts human_agent, model_agent = self.agents # human act act = deepcopy(human_agent.act()) self_pred, partner_pred, about_pred = get_axis_predictions( act, model_agent, self_threshold=self.opt['self_threshold'], partner_threshold=self.opt['partner_threshold'], ) pred_text = f'SELF: {self_pred}\nPARTNER: {partner_pred}\nABOUT: {about_pred}' acts[1] = { 'id': 'MDGender Classifier', 'text': pred_text, 'episode_done': True } human_agent.observe(validate(acts[1])) self.update_counters()
def parley(self): self.turn_index = (self.turn_index + 1) % 2 ad = { 'episode_done': False } ad['id'] = self.__class__.TEST_ID if self.turn_index == 0: # Take a first turn ad['text'] = self.TEST_TEXT_1 self.mturk_agent.observe(validate(ad)) self.response1 = self.mturk_agent.act() if self.turn_index == 1: # Complete after second turn ad['text'] = self.TEST_TEXT_2 ad['episode_done'] = True # end of episode self.mturk_agent.observe(validate(ad)) self.response2 = self.mturk_agent.act() time.sleep(1) self.episodeDone = True
def parley(self): """For each agent, get an observation of the last action each of the other agents took. Then take an action yourself. """ acts = self.acts for index, agent in enumerate(self.agents): try: acts[index] = agent.act(timeout=None) except TypeError: acts[index] = agent.act() # not MTurkAgent if acts[index]['episode_done']: self.episodeDone = True for other_agent in self.agents: if other_agent != agent: other_agent.observe(validate(acts[index]))
def parley(self): """For each agent, act, then force other agents to observe your act """ acts = self.acts for index, agent in enumerate(self.agents): try: acts[index] = agent.act(timeout=None) except TypeError: acts[index] = agent.act() # not MTurkAgent if acts[index]['episode_done']: self.episodeDone = True for other_agent in self.agents: if other_agent != agent: other_agent.observe(validate(acts[index])) self.rounds += 1 if self.rounds >= 2: time.sleep(2) self.episodeDone = True
def parley(self): self.task_world.parley() ad = {} # Show the dialog model's response to the context, and ask the turker to rate the response ad['id'] = self.__class__.evaluator_agent_id ad['text'] = ( self.task_world.get_acts()[0]['text'] + "\n\n" + "How would you rate the following response (from 0 to 10):\n\n" + self.task_world.get_acts()[1]['text']) # TODO: deal with multi-turn dialogs, for now we will just deal # with 1-turn dialogs in this task. ad['episode_done'] = True # self.world.episode_done() self.mturk_agent.observe(validate(ad)) rating = self.mturk_agent.act() self.episodeDone = True