def test_mturk_racehistory(self): """ Emulate a setting where batch_act misappropriately handles mturk. """ agent = get_agent(batchsize=16, interactive_mode=True, echo=True) share1 = create_agent_from_shared(agent.share()) share1.observe(Message({'text': 'thread1-msg1', 'episode_done': False})) share2 = create_agent_from_shared(agent.share()) share2.observe(Message({'text': 'thread2-msg1', 'episode_done': False})) share1.act() share2.act() share1.observe(Message({'text': 'thread1-msg2', 'episode_done': False})) share2.observe(Message({'text': 'thread2-msg2', 'episode_done': False})) share2.act() share1.act() share2.observe(Message({'text': 'thread2-msg3', 'episode_done': False})) share1.observe(Message({'text': 'thread1-msg3', 'episode_done': False})) self.assertNotIn('thread1-msg1', share2.history.get_history_str()) self.assertNotIn('thread2-msg1', share1.history.get_history_str()) self.assertNotIn('thread1-msg2', share2.history.get_history_str()) self.assertNotIn('thread2-msg2', share1.history.get_history_str())
def test_interactive_mode(self): """ Test if conversation history is destroyed in MTurk mode. """ # both manually setting bs to 1 and interactive mode true agent = get_agent(batchsize=1, interactive_mode=True) agent.observe(Message({'text': 'foo', 'episode_done': True})) response = agent.act() self.assertIn( 'Evaluating 0', response['text'], 'Incorrect output in single act()' ) shared = create_agent_from_shared(agent.share()) shared.observe(Message({'text': 'bar', 'episode_done': True})) response = shared.act() self.assertIn( 'Evaluating 0', response['text'], 'Incorrect output in single act()' ) # now just bs 1 agent = get_agent(batchsize=1, interactive_mode=False) agent.observe(Message({'text': 'foo', 'episode_done': True})) response = agent.act() self.assertIn( 'Evaluating 0', response['text'], 'Incorrect output in single act()' ) shared = create_agent_from_shared(agent.share()) shared.observe(Message({'text': 'bar', 'episode_done': True})) response = shared.act() self.assertIn( 'Evaluating 0', response['text'], 'Incorrect output in single act()' ) # now just interactive shared = create_agent_from_shared(agent.share()) agent.observe(Message({'text': 'foo', 'episode_done': True})) response = agent.act() self.assertIn( 'Evaluating 0', response['text'], 'Incorrect output in single act()' ) shared = create_agent_from_shared(agent.share()) shared.observe(Message({'text': 'bar', 'episode_done': True})) response = shared.act() self.assertIn( 'Evaluating 0', response['text'], 'Incorrect output in single act()' ) # finally, actively attempt to sabotage agent = get_agent(batchsize=16, interactive_mode=False) agent.observe(Message({'text': 'foo', 'episode_done': True})) response = agent.act() self.assertIn( 'Evaluating 0', response['text'], 'Incorrect output in single act()' ) shared = create_agent_from_shared(agent.share()) shared.observe(Message({'text': 'bar', 'episode_done': True})) response = shared.act() self.assertIn( 'Evaluating 0', response['text'], 'Incorrect output in single act()' )
def __init__( self, opt, agents=None, shared=None, range_turn=(3, 5), max_turn=5, max_resp_time=120, model_agent_opt=None, world_tag='', agent_timeout_shutdown=120, knowledge_retriever_opt=None, ): self.opt = opt # TURN CONTROL self.turn_idx = 0 self.range_turn = range_turn self.max_turn = max_turn self.n_turn = np.random.randint(self.range_turn[0], self.range_turn[1]) + 1 self.chat_done = False self.other_first = random.choice([True, False]) # DATA self.dialog = [] self.dialog_list = [] self.gmark_score = -1 self.task_type = 'sandbox' if opt['is_sandbox'] else 'live' self.world_tag = world_tag self.ratings = ['1', '2', '3', '4', '5'] super().__init__(opt, agents, shared) # MODEL AGENT SET UP if model_agent_opt is not None: self.model_agent = create_agent_from_shared(model_agent_opt) self.knowledge_agent = create_agent_from_shared( knowledge_retriever_opt) else: # case where we test against a human self.model_agent = None # TIMEOUT PROTOCOLS self.max_resp_time = max_resp_time # in secs self.agent_timeout_shutdown = agent_timeout_shutdown # TOPIC CHOICES if self.model_agent is None: for idx in range(len(self.agents)): if self.agents[idx].id == 'PERSON_1': self.eval_agent = self.agents[idx] self.other_agent = self.agents[idx - 1] break else: self.eval_agent = self.agents[0] self.chosen_topic = self.eval_agent.chosen_topic self.seen = self.eval_agent.seen self.topic_choices = self.eval_agent.topic_choices
def __init__(self, opt): if isinstance(opt, ParlaiParser): opt = opt.parse_args() # Possibly build a dictionary (not all models do this). if opt['dict_build_first'] and 'dict_file' in opt: if opt['dict_file'] is None and opt.get('model_file_transmitter') and opt.get('model_file_receiver'): opt['dict_file'] = opt['model_file_transmitter'] + '_' + opt['model_file_receiver'] + '.dict' print("[ building dictionary first... ]") build_dict(opt, skip_if_built=False) # Create model and assign it to the specified task print("[ create meta-agent ... ]") self.agent = create_agent(opt) print("[ create agent A ... ]") shared = self.agent.share() self.agent_a = create_agent_from_shared(shared) self.agent_a.set_id(suffix=' A') print("[ create agent B ... ]") self.agent_b = create_agent_from_shared(shared) # self.agent_b = create_agent(opt) self.agent_b.set_id(suffix=' B') # self.agent_a.copy(self.agent, 'transmitter') # self.agent_b.copy(self.agent, 'transmitter') self.world = create_selfplay_world(opt, [self.agent_a, self.agent_b]) # TODO: if batch, it is also not parallel # self.world = BatchSelfPlayWorld(opt, self_play_world) self.train_time = Timer() self.train_dis_time = Timer() self.validate_time = Timer() self.log_time = Timer() self.save_time = Timer() print('[ training... ]') self.parleys_episode = 0 self.max_num_epochs = opt['num_epochs'] if opt['num_epochs'] > 0 else float('inf') self.max_train_time = opt['max_train_time'] if opt['max_train_time'] > 0 else float('inf') self.log_every_n_secs = opt['log_every_n_secs'] if opt['log_every_n_secs'] > 0 else float('inf') self.train_dis_every_n_secs = opt['train_display_every_n_secs'] if opt['train_display_every_n_secs'] > 0 else float('inf') self.val_every_n_secs = opt['validation_every_n_secs'] if opt['validation_every_n_secs'] > 0 else float('inf') self.save_every_n_secs = opt['save_every_n_secs'] if opt['save_every_n_secs'] > 0 else float('inf') self.valid_optim = 1 if opt['validation_metric_mode'] == 'max' else -1 self.best_valid = None if opt.get('model_file_transmitter') and os.path.isfile(opt['model_file_transmitter'] + '.best_valid'): with open(opt['model_file_transmitter'] + ".best_valid", 'r') as f: x = f.readline() self.best_valid = float(x) f.close() self.impatience = 0 self.saved = False self.valid_world = None self.opt = opt if opt['tensorboard_log'] is True: self.writer = TensorboardLogger(opt)
def __init__(self, opt: Opt): self.opt = opt self.agents = [] self.agent_dict = None self.generations = [] self.input_type = 'Memory' self.delimiter = opt.get('memory_decoder_delimiter', '\n') self.one_line_memories = opt.get('memory_decoder_one_line_memories', False) model_file = modelzoo_path(opt['datapath'], opt['memory_decoder_model_file']) if model_file and os.path.exists(model_file): logging.info(f'Building Memory Decoder from file: {model_file}') logging.disable() overrides = { 'skip_generation': False, 'inference': 'beam', 'beam_size': opt.get('memory_decoder_beam_size', 3), 'beam_min_length': opt.get('memory_decoder_beam_min_length', 10), 'beam_block_ngram': 3, } if self.opt.get('memory_decoder_truncate', -1) > 0: overrides['text_truncate'] = self.opt['memory_decoder_truncate'] overrides['truncate'] = self.opt['memory_decoder_truncate'] base_agent = create_agent_from_model_file( model_file, opt_overrides=overrides ) assert isinstance(base_agent, TorchAgent) self.agents = [base_agent] assert isinstance(self.agents[0], TorchAgent) copies = max(100, (opt['batchsize'] * opt.get('rag_turn_n_turns', 1))) self.agents += [ create_agent_from_shared(self.agents[0].share()) for _ in range(copies) ] self.agent_dict = self.agents[0].build_dictionary() logging.enable()
def __init__(self, opt, shared=None): super().__init__(opt, shared) self.debug = opt['debug'] self.model_path = os.path.join( opt['datapath'], 'models', 'wizard_of_wikipedia', 'full_dialogue_retrieval_model', ) if not shared: # Create responder self._set_up_responder(opt) # Create retriever self._set_up_retriever(opt) else: self.opt = shared['opt'] self.retriever = shared['retriever'] self.responder = create_agent_from_shared( shared['responder_shared_opt']) self.sent_tok = shared['sent_tok'] self.wiki_map = shared['wiki_map'] self.id = 'WizardGenerativeInteractiveAgent' self.ret_history = {}
def __init__(self, opt: Opt): self.opt = opt self.agents = [] self.agent_dict = None self.generations = [] self.input_type = 'Search' self.knowledge_access_method = KnowledgeAccessMethod( opt['knowledge_access_method'] ) model_file = modelzoo_path(opt['datapath'], opt['query_generator_model_file']) if model_file and os.path.exists(model_file): logging.info(f'Building Query Generator from file: {model_file}') logging.disable() overrides: Dict[str, Any] = {'skip_generation': False} overrides['inference'] = opt['query_generator_inference'] overrides['beam_size'] = opt.get('query_generator_beam_size', 3) overrides['beam_min_length'] = opt.get('query_generator_beam_min_length', 2) if self.opt['query_generator_truncate'] > 0: overrides['text_truncate'] = self.opt['query_generator_truncate'] overrides['truncate'] = self.opt['query_generator_truncate'] base_agent = create_agent_from_model_file( model_file, opt_overrides=overrides ) assert isinstance(base_agent, TorchAgent) self.agents = [base_agent] bsz = opt.get('batchsize', 1) rag_turn_n_turns = opt.get('rag_turn_n_turns', 1) if bsz > 1 or rag_turn_n_turns > 1: self.agents += [ create_agent_from_shared(self.agents[0].share()) for _ in range((bsz * rag_turn_n_turns) - 1) ] self.agent_dict = self.agents[0].build_dictionary() logging.enable()
def __init__(self, opt, shared=None): self.opt = opt self.add_token_knowledge = opt['add_token_knowledge'] self.model_path = os.path.join( opt['datapath'], 'models', 'wizard_of_wikipedia', 'full_dialogue_retrieval_model', ) if not shared: # Create retriever download( opt['datapath']) # make sure to download all relevant files self._set_up_tfidf_retriever(opt) self._set_up_selector(opt) else: self.selector = create_agent_from_shared(shared['selector']) self.retriever = shared['retriever'] self.sent_tok = shared['sent_tok'] self.wiki_map = shared['wiki_map'] self.id = 'KnowledgeRetrieverAgent' # NOTE: dialogue history should NOT be shared between instances self.retriever_history = {'episode_done': False} self.dialogue_history = [] self.checked_sentence_history = []
def run_conversation(manager, opt, agents, task_id): agent = agents[0] this_bot = create_agent_from_shared(shared_bot_params) world = MessengerBotChatTaskWorld(opt=opt, agent=agent, bot=this_bot) while not world.episode_done(): world.parley() world.shutdown()
def run_eval(agent, opt, datatype, max_exs=-1, write_log=False, valid_world=None): """Eval on validation/test data. - Agent is the agent to use for the evaluation. - opt is the options that specific the task, eval_task, etc - datatype is the datatype to use, such as "valid" or "test" - write_log specifies to write metrics to file if the model_file is set - max_exs limits the number of examples if max_exs > 0 - valid_world can be an existing world which will be reset instead of reinitialized """ print('[ running eval: ' + datatype + ' ]') if 'stream' in opt['datatype']: datatype += ':stream' if valid_world is None: opt = opt.copy() opt['datatype'] = datatype if opt.get('evaltask'): opt['task'] = opt['evaltask'] if opt.get('validation_share_agent', False): valid_agent = create_agent_from_shared(agent.share()) else: valid_agent = agent valid_world = create_task(opt, valid_agent) valid_world.reset() cnt = 0 i_example = 0 print('== Before valid ==') while not valid_world.epoch_done( ): # and opt['validation_max_examples'] > i_example: valid_world.parley() if cnt == 0 and opt['display_examples']: print(valid_world.display() + '\n~~') print(valid_world.report()) cnt += opt['batchsize'] if max_exs > 0 and cnt > max_exs + opt.get('numthreads', 1): # note this max_exs is approximate--some batches won't always be # full depending on the structure of the data break i_example += 1 valid_report = valid_world.report() valid_world.reset() # this makes sure agent doesn't remember valid data metrics = datatype + ':' + str(valid_report) print(metrics) if write_log and opt.get('model_file'): # Write out metrics f = open(opt['model_file'] + '.' + datatype, 'a+') f.write(metrics + '\n') f.close() return valid_report, valid_world
def generate_world(opt, agents): if opt['models'] is None: raise RuntimeError("Model must be specified") return ChattyGooseMessengerTaskWorld( opt, agents[0], create_agent_from_shared(opt['shared_bot_params'][ ChattyGooseMessengerTaskWorld.MODEL_KEY]), )
def generate_world(opt, agents): if opt['models'] is None: raise RuntimeError("Model must be specified") return ConversationBotTaskWorld( opt, agents[0], create_agent_from_shared( opt['shared_bot_params'][ConversationBotTaskWorld.MODEL_KEY]), )
def __init__( self, shared: TShared = None, custom_model_file='zoo:dialogue_safety/single_turn/model', ): if not shared: self.model = self._create_safety_model(custom_model_file) else: self.model = create_agent_from_shared(shared['model']) self.classes = {OK_CLASS: False, NOT_OK_CLASS: True}
def __init__( self, opt, agents=None, shared=None, range_turn=(5, 6), max_turn=10, max_resp_time=120, model_agent_opt=None, world_tag='', agent_timeout_shutdown=120, ): self.turn_idx = 0 self.hit_id = None self.range_turn = range_turn self.max_turn = max_turn self.n_turn = np.random.randint(self.range_turn[0], self.range_turn[1]) + 1 self.model_name = opt.get('model_name') self.dialog = [] self.task_type = 'sandbox' if opt['is_sandbox'] else 'live' self.chat_done = False self.n_personas = [] self.fluency_score = len(agents) * [-1] self.fluency_reason = len(agents) * [None] self.eng_score = len(agents) * [-1] self.eng_reason = len(agents) * [None] self.consistent_score = len(agents) * [-1] self.consistent_reason = len(agents) * [None] self.persona_picked = len(agents) * [None] self.world_tag = world_tag self.ratings = ['1', '2', '3', '4', '5'] super().__init__(opt, agents, shared) # set up model agent if model_agent_opt is not None: self.model_agent = create_agent_from_shared(model_agent_opt) else: # case where we test against a human self.model_agent = self.agents[1] # below are timeout protocols self.max_resp_time = max_resp_time # in secs self.agent_timeout_shutdown = agent_timeout_shutdown # set up personas self.personas = [ (ag.persona_data if hasattr(ag, 'persona_data') else None) for ag in self.agents ] self.model_persona_text = '\n'.join([ 'your persona:' + pers for pers in self.agents[0].model_persona[1] ]) print(self.model_persona_text)
def run_conversation(mturk_manager, opt, workers): remaining_counts_needed = [ (m, c - run_statistics[m]) for (m, c) in opt['conversations_needed'].items() ] remaining_counts_needed.sort(reverse=True, key=lambda x: x[1]) model_name = remaining_counts_needed[0][0] print( f'Remaining conversation counts needed: {remaining_counts_needed}' ) # Get a bot and add it to the list of "workers" print(f'Choosing the "{model_name}" model for the bot.') agent = create_agent_from_shared(shared_bot_agents[model_name]) bot_worker = TurkLikeAgent( opt, model_name=model_name, model_agent=agent, num_turns=opt['num_turns'], semaphore=semaphore, ) workers_including_bot = workers + [bot_worker] assert len(workers_including_bot) == 2 # Get context: personas, previous utterances, etc. if context_generator is not None: context_info = context_generator.get_context() else: context_info = None conv_idx = mturk_manager.conversation_index world = TurnAnnotationsChatWorld( opt=opt, agents=workers_including_bot, num_turns=opt['num_turns'], max_resp_time=opt['max_resp_time'], tag='conversation t_{}'.format(conv_idx), context_info=context_info, ) while not world.episode_done(): print('About to parley') world.parley() model_nickname, worker_is_unacceptable, convo_finished = world.save_data( ) if worker_is_unacceptable: print(f'Soft-blocking worker {workers[0].worker_id}') mturk_manager.soft_block_worker(workers[0].worker_id) time.sleep(0.1) if not worker_is_unacceptable and convo_finished: run_statistics[model_nickname] += 1 world.shutdown() world.review_work()
def __init__(self, opt: Opt, shared=None): if ',' in opt['task']: raise ValueError( 'LabelToTextTeacher cannot be used with multiple tasks!') self.id = opt['task'] self.opt = opt if shared and 'task' in shared: self.task = create_agent_from_shared(shared['task']) else: opt_singletask = copy.deepcopy(opt) opt_singletask['task'] = opt['label_to_text_task'] self.task = create_task_agent_from_taskname(opt_singletask)[0]
def __init__(self, opt: Opt, shared=None): if ',' in opt['task']: raise ValueError( 'AbstractWrapperTeacher cannot be used with multiple tasks!') self.id = opt['task'] self.opt = opt if shared: self.task = create_agent_from_shared(shared['task']) else: opt_singletask = copy.deepcopy(opt) opt_singletask['task'] = opt['wrapper_task'] self.task = create_task_agent_from_taskname(opt_singletask)[0]
def load_eval_worlds(agent, opt, datatype): """ Create a new eval world for the agent and the given opt. Overrides the datatype options for doing this. Handles some magic overrides of other special options for the training script. :param Agent agent: The model being trained. :param Opt opt: The global CLI opts. :param string datatype: The new datatype. """ if 'stream' in opt['datatype']: datatype += ':stream' opt = opt.copy() opt['datatype'] = datatype if opt.get('evaltask'): # if a different eval task is specified, use it. opt['task'] = opt['evaltask'] if opt.get('eval_batchsize'): # override eval time batchsize opt['batchsize'] = opt['eval_batchsize'] if opt.get('eval_dynamic_batching'): # FIXME: see issue tracked in https://github.com/facebookresearch/ParlAI/issues/3367 # override eval time dynamic batching settings eval_dyn_batch = ( None if opt['eval_dynamic_batching'] == 'off' else opt['eval_dynamic_batching'] ) opt['dynamic_batching'] = eval_dyn_batch tasks = opt['task'].split(',') worlds = [] # possibly load agent if opt.get('validation_share_agent', False): valid_agent = create_agent_from_shared(agent.share()) else: valid_agent = agent # create worlds for task in tasks: task_opt = opt.copy() # copy opt since we edit the task task_opt['task'] = task valid_world = create_task(task_opt, valid_agent) worlds.append(valid_world) return worlds
async def parlai(bot, userstate, initial=None): user_text = initial or await bot.listen_text(timeout=10) agent = create_agent_from_shared(root_agent.share()) while True: if language.get() != 'en': user_text = await translate(language.get(), 'en', user_text) msg = dict(type='message', text=user_text, episode_done=False) agent.observe(validate(msg)) response = agent.act() bot_text = response['text'].replace(" ' ", "'").replace(' ,', ',') if language.get() != 'en': bot_text = await translate('en', language.get(), bot_text) user_text = await bot.ask(bot_text, timeout=10)
def __init__(self, opt, shared=None): if 'dictionary_agent' in shared: # use this first--maybe be overriding an original dictionary self.dict = create_agent_from_shared(shared['dictionary_agent']) elif 'dictionary' in shared: # otherwise use this dictionary self.dict = shared['dictionary'] else: raise RuntimeError('ParsedRemoteAgent needs a dictionary to parse' + ' text with--pass in a dictionary using shared' + '["dictionary"] or pass in the arguments to ' + 'instantiate one using shared["dictionary_args' + '"] = (class, options, shared).') super().__init__(opt, shared)
def load_eval_worlds(agent, opt, datatype): """ Create a new eval world for the agent and the given opt. Overrides the datatype options for doing this. Handles some magic overrides of other special options for the training script. :param Agent agent: The model being trained. :param Opt opt: The global CLI opts. :param string datatype: The new datatype. """ if 'stream' in opt['datatype']: datatype += ':stream' opt = opt.copy() opt['datatype'] = datatype if opt.get('pytorch_teacher_task'): # never use pytorch teachers for evaluation # but don't forget what we were normally using opt['task'] = opt['pytorch_teacher_task'] del opt['pytorch_teacher_task'] if opt.get('evaltask'): # if a different eval task is specified, use it. opt['task'] = opt['evaltask'] if opt.get('eval_batchsize'): # override eval time batchsize opt['batchsize'] = opt['eval_batchsize'] tasks = opt['task'].split(',') worlds = [] # possibly load agent if opt.get('validation_share_agent', False): valid_agent = create_agent_from_shared(agent.share()) else: valid_agent = agent # create worlds for task in tasks: task_opt = opt.copy() # copy opt since we edit the task task_opt['task'] = task valid_world = create_task(task_opt, valid_agent) worlds.append(valid_world) return worlds
def load_eval_worlds(agent, opt, datatype): """ Create a new eval world for the agent and the given opt. Overrides the datatype options for doing this. Handles some magic overrides of other special options for the training script. :param Agent agent: The model being trained. :param Opt opt: The global CLI opts. :param string datatype: The new datatype. """ if not is_primary_worker(): # don't load worlds in workers # TODO(MW): this block will need to be removed return None if 'stream' in opt['datatype']: datatype += ':stream' opt = opt.copy() opt['datatype'] = datatype if opt.get('evaltask'): # if a different eval task is specified, use it. opt['task'] = opt['evaltask'] if opt.get('eval_batchsize'): # override eval time batchsize opt['batchsize'] = opt['eval_batchsize'] tasks = opt['task'].split(',') worlds = [] # possibly load agent if opt.get('validation_share_agent', False): valid_agent = create_agent_from_shared(agent.share()) else: valid_agent = agent # create worlds for task in tasks: task_opt = opt.copy() # copy opt since we edit the task task_opt['task'] = task valid_world = create_task(task_opt, valid_agent) worlds.append(valid_world) return worlds
def load_eval_world(agent, opt, datatype): if 'stream' in opt['datatype']: datatype += ':stream' opt = opt.copy() opt['datatype'] = datatype if opt.get('evaltask'): opt['task'] = opt['evaltask'] if opt.get('eval_batchsize'): # override eval time batchsize opt['batchsize'] = opt['eval_batchsize'] if opt.get('validation_share_agent', False): valid_agent = create_agent_from_shared(agent.share()) else: valid_agent = agent valid_world = create_task(opt, valid_agent) return valid_world
def __init__(self, opt: Opt): self.opt = opt self.agents = [] self.agent_dict = None self.generations = [] self.input_type = 'Search' self.knowledge_access_method = KnowledgeAccessMethod( opt['knowledge_access_method']) model_file = modelzoo_path(opt['datapath'], opt['query_generator_model_file']) if (self.knowledge_access_method is KnowledgeAccessMethod.SEARCH_ONLY and 'blenderbot2/query_generator/model' in model_file): raise ValueError( 'You cannot use the blenderbot2 query generator with search_only. Please ' 'consider setting --query-generator-model-file zoo:sea/bart_sq_gen/model ' 'instead.') if model_file and os.path.exists(model_file): logging.info(f'Building Query Generator from file: {model_file}') logging.disable() overrides: Dict[str, Any] = {'skip_generation': False} overrides['inference'] = opt['query_generator_inference'] overrides['beam_size'] = opt.get('query_generator_beam_size', 3) overrides['beam_min_length'] = opt.get( 'query_generator_beam_min_length', 2) overrides['model_parallel'] = opt['model_parallel'] overrides['no_cuda'] = opt['no_cuda'] if self.opt['query_generator_truncate'] > 0: overrides['text_truncate'] = self.opt[ 'query_generator_truncate'] overrides['truncate'] = self.opt['query_generator_truncate'] base_agent = create_agent_from_model_file(model_file, opt_overrides=overrides) assert isinstance(base_agent, TorchAgent) self.agents = [base_agent] bsz = max( opt.get('batchsize') or 1, opt.get('eval_batchsize') or 1) rag_turn_n_turns = opt.get('rag_turn_n_turns', 1) if bsz > 1 or rag_turn_n_turns > 1: self.agents += [ create_agent_from_shared(self.agents[0].share()) for _ in range((bsz * rag_turn_n_turns) - 1) ] self.agent_dict = self.agents[0].build_dictionary() logging.enable()
def _init_chat(self, chatID): """ Create new chat for new dialog. Sets up a new instantiation of the agent so that each chat has its own local state. :param chatID: chat id :return: new instance of your local agent """ agent_info = self.shared # Add refs to current world instance and chat id to agent 'opt' parameter if 'opt' not in agent_info.keys() or agent_info['opt'] is None: agent_info['opt'] = {} agent_info['opt']['convai_chatID'] = chatID local_agent = create_agent_from_shared(agent_info) self.chats[chatID] = local_agent return self.chats[chatID]
def _init_chat(self, chatID): """Create new chat for new dialog. Sets up a new instantiation of the agent so that each chat has its own local state. :param chatID: chat id :return: new instance of your local agent """ agent_info = self.shared # Add refs to current world instance and chat id to agent 'opt' parameter if 'opt' not in agent_info.keys() or agent_info['opt'] is None: agent_info['opt'] = {} agent_info['opt']['convai_chatID'] = chatID local_agent = create_agent_from_shared(agent_info) self.chats[chatID] = local_agent return self.chats[chatID]
def get_bot_worker(opt: Dict[str, Any], model_name: str) -> TurkLikeAgent: """ Return a bot agent. Agent behaves like a crowdsource worker but actually wraps around a dialogue model. """ semaphore = opt['semaphore'] shared_bot_agents = opt['shared_bot_agents'] num_turns = opt['num_turns'] bot_agent = create_agent_from_shared(shared_bot_agents[model_name]) bot_worker = TurkLikeAgent( opt, model_name=model_name, model_agent=bot_agent, num_turns=num_turns, semaphore=semaphore, ) return bot_worker
def run_conversation(mturk_manager, opt, workers): remaining_counts_needed = [ (m, c - run_statistics[m]) for (m, c) in opt['conversations_needed'].items() ] remaining_counts_needed.sort(reverse=True, key=lambda x: x[1]) model_name = remaining_counts_needed[0][0] print( f'Remaining conversation counts needed: {remaining_counts_needed}' ) # Get a bot and add it to the list of "workers" print(f'Choosing the "{model_name}" model for the bot.') agent = create_agent_from_shared(shared_bot_agents[model_name]) bot_worker = TurkLikeAgent( opt, model_name=model_name, model_agent=agent, num_turns=opt['num_turns'], semaphore=semaphore, ) workers_including_bot = workers + [bot_worker] assert len(workers_including_bot) == 2 conv_idx = mturk_manager.conversation_index world = TurnAnnotationsChatWorld( opt=opt, agents=workers_including_bot, num_turns=opt['num_turns'], max_resp_time=opt['max_resp_time'], tag='conversation t_{}'.format(conv_idx), annotations_config=ANNOTATIONS_CONFIG, ) while not world.episode_done(): print('About to parley') world.parley() model_nickname, convo_finished = world.save_data() if convo_finished: run_statistics[model_nickname] += 1 world.shutdown() world.review_work()
def make_world(opt, agents): # Extract important components from opt semaphore = opt['semaphore'] shared_bot_agents = opt['shared_bot_agents'] statistics_condition = opt['statistics_condition'] context_generator = opt['context_generator'] num_turns = opt['num_turns'] # Decide on a bot to use run_statistics = opt['run_statistics'] with statistics_condition: remaining_counts_needed = [ (m, c - run_statistics[m]) for (m, c) in opt['conversations_needed'].items() ] remaining_counts_needed.sort(reverse=True, key=lambda x: x[1]) model_name = remaining_counts_needed[0][0] print( f'Remaining conversation counts needed: {remaining_counts_needed}') print(f'Choosing the "{model_name}" model for the bot.') # Create the bot bot_agent = create_agent_from_shared(shared_bot_agents[model_name]) bot_worker = TurkLikeAgent( opt, model_name=model_name, model_agent=bot_agent, num_turns=num_turns, semaphore=semaphore, ) # Get context: personas, previous utterances, etc. if context_generator is not None: context_info = context_generator.get_context() else: context_info = None agents[0].agent_id = "Worker" return TurnAnnotationsChatWorld(opt, agent=agents[0], bot=bot_worker, context_info=context_info)
def _init_chat(self, chat): """ Create new chat for new dialog. Chat consists of new instance of ConvAIAgent, new instance of your own Agent and new instance DialogPartnerWorld where this two agents will communicate. Information about class of your local agent getting from shared data. :param chat: chat id :return: tuple with instances of ConvAIAgent, your local agent, DialogPartnerWorld """ agent_info = self.shared["agents"][0] # Add refs to current world instance and chat id to agent 'opt' parameter if 'opt' not in agent_info.keys() or agent_info['opt'] is None: agent_info['opt'] = {} agent_info['opt']['convai_world'] = self agent_info['opt']['convai_chat'] = chat local_agent = create_agent_from_shared(agent_info) remote_agent = ConvAIAgent({'chat': chat}) world = DialogPartnerWorld({'task': 'ConvAI Dialog'}, [remote_agent, local_agent]) self.chats[chat] = (remote_agent, local_agent, world) return self.chats[chat]
def load_eval_world(agent, opt, datatype): if 'stream' in opt['datatype']: datatype += ':stream' opt = opt.copy() opt['datatype'] = datatype if opt.get('pytorch_teacher_task'): # never use pytorch teachers for evaluation # but don't forget what we were normally using opt['task'] = opt['pytorch_teacher_task'] del opt['pytorch_teacher_task'] if opt.get('evaltask'): # if a different eval task is specified, use it. opt['task'] = opt['evaltask'] if opt.get('eval_batchsize'): # override eval time batchsize opt['batchsize'] = opt['eval_batchsize'] if opt.get('validation_share_agent', False): valid_agent = create_agent_from_shared(agent.share()) else: valid_agent = agent valid_world = create_task(opt, valid_agent) return valid_world