def setup_args(): """ Set up args. """ parser = ParlaiParser(False, False) parser.add_parlai_data_path() parser.add_messenger_args() return parser.parse_args()
def setup_args(): """ Set up args. """ parser = ParlaiParser(False, False) parser.add_parlai_data_path() parser.add_websockets_args() return parser.parse_args()
def setup_args(): """ Set up args. """ parser = ParlaiParser(False, False) parser.add_parlai_data_path() reddit = parser.add_argument_group('Download Reddit Docs') reddit.add_argument('-sy', '--start_year', default=2011, type=int, metavar='N', help='starting year') reddit.add_argument('-ey', '--end_year', default=2018, type=int, metavar='N', help='end year') reddit.add_argument('-sm', '--start_month', default=7, type=int, metavar='N', help='starting year') reddit.add_argument('-em', '--end_month', default=7, type=int, metavar='N', help='end year') reddit.add_argument( '-sr_l', '--subreddit_list', default='["explainlikeimfive"]', type=str, help='subreddit name', ) reddit.add_argument( '-id_l', '--id_list', type=str, help='json file path of base36 post IDs (in a list format)', ) reddit.add_argument('-Q', '--questions_only', action='store_true', help='only download submissions') reddit.add_argument('-A', '--answers_only', action='store_true', help='only download comments') reddit.add_argument('-o', '--output_dir', default='eli5/', type=str, help='where to save the output') return parser.parse_args()
def create_agent(opt, requireModelExists=False): """ Create an agent from the options ``model``, ``model_params`` and ``model_file``. The input is either of the form ``parlai.agents.ir_baseline.agents:IrBaselineAgent`` (i.e. the path followed by the class name) or else just ``ir_baseline`` which assumes the path above, and a class name suffixed with 'Agent'. If ``model-file`` is available in the options this function can also attempt to load the model from that location instead. This avoids having to specify all the other options necessary to set up the model including its name as they are all loaded from the options file if it exists (the file opt['model_file'] + '.opt' must exist and contain a pickled or json dict containing the model's options). """ if opt.get('datapath', None) is None: # add datapath, it is missing from parlai.core.params import ParlaiParser, get_model_name parser = ParlaiParser(add_parlai_args=False) parser.add_parlai_data_path() # add model args if they are missing model = get_model_name(opt) if model is not None: parser.add_model_subargs(model) opt_parser = parser.parse_args("", print_args=False) for k, v in opt_parser.items(): if k not in opt: opt[k] = v if opt.get('model_file'): opt['model_file'] = modelzoo_path(opt.get('datapath'), opt['model_file']) if requireModelExists and not os.path.isfile(opt['model_file']): raise RuntimeError( 'WARNING: Model file does not exist, check to make ' 'sure it is correct: {}'.format(opt['model_file']) ) # Attempt to load the model from the model file first (this way we do # not even have to specify the model name as a parameter) model = load_agent_module(opt) if model is not None: return model else: print("[ no model with opt yet at: " + opt.get('model_file') + "(.opt) ]") if opt.get('model'): model_class = get_agent_module(opt['model']) # if we want to load weights from --init-model, compare opts with # loaded ones compare_init_model_opts(opt, opt) model = model_class(opt) if requireModelExists and hasattr(model, 'load') and not opt.get('model_file'): # double check that we didn't forget to set model_file on loadable model print('WARNING: model_file unset but model has a `load` function.') return model else: raise RuntimeError('Need to set `model` argument to use create_agent.')
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) mturk_agent_1_id = 'mturk_agent_1' mturk_agent_2_id = 'mturk_agent_2' human_agent_1_id = 'human_1' human_agent_2_id = 'human_2' mturk_manager = MTurkManager( opt=opt, mturk_agent_ids=[mturk_agent_1_id, mturk_agent_2_id], all_agent_ids=[ human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id ] # In speaking order ) mturk_manager.init_aws(opt=opt) global run_hit def run_hit(hit_index, assignment_index, opt, mturk_manager): conversation_id = str(hit_index) + '_' + str(assignment_index) # Create mturk agents mturk_agent_1 = MTurkAgent(id=mturk_agent_1_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) mturk_agent_2 = MTurkAgent(id=mturk_agent_2_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = human_agent_1_id human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = human_agent_2_id world = MultiAgentDialogWorld(opt=opt, agents=[ human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2 ]) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.create_hits(opt=opt) results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \ (delayed(run_hit)(hit_index, assignment_index, opt, mturk_manager) \ for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1))) mturk_manager.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_messenger_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) # Initialize a SQuAD teacher agent, which we will get context from module_name = 'parlai.tasks.squad.agents' class_name = 'DefaultTeacher' my_module = importlib.import_module(module_name) task_class = getattr(my_module, class_name) task_opt = {} task_opt['datatype'] = 'train' task_opt['datapath'] = opt['datapath'] messenger_manager = MessengerManager(opt=opt) messenger_manager.setup_server() messenger_manager.init_new_state() def get_overworld(agent): return MessengerOverworld(None, agent) def assign_agent_role(agent): agent[0].disp_id = 'Agent' def run_conversation(manager, opt, agents, task_id): task = task_class(task_opt) agent = agents[0] world = QADataCollectionWorld( opt=opt, task=task, agent=agent ) while not world.episode_done(): world.parley() world.shutdown() # World with no onboarding messenger_manager.set_onboard_functions({'default': None}) task_functions = {'default': run_conversation} assign_agent_roles = {'default': assign_agent_role} messenger_manager.set_agents_required({'default': 1}) messenger_manager.set_overworld_func(get_overworld) messenger_manager.setup_socket() try: messenger_manager.start_new_run() messenger_manager.start_task( assign_role_functions=assign_agent_roles, task_functions=task_functions, ) except BaseException: raise finally: messenger_manager.shutdown()
def main_bak(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) # Initialize a SQuAD teacher agent, which we will get context from module_name = 'parlai.tasks.squad.agents' class_name = 'DefaultTeacher' my_module = importlib.import_module(module_name) task_class = getattr(my_module, class_name) task_opt = opt.copy() task_opt['datatype'] = 'train' task_opt['datapath'] = opt['datapath'] mturk_agent_id = 'Worker' mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=[mturk_agent_id]) mturk_manager.setup_server() mturk_manager.set_onboard_function(onboard_function=None) mturk_manager.start_new_run() mturk_manager.create_hits() mturk_manager.ready_to_accept_workers() def check_workers_eligibility(workers): return workers eligibility_function = { 'func': check_workers_eligibility, 'multiple': True, } def assign_worker_roles(worker): worker[0].id = mturk_agent_id global run_conversation def run_conversation(mturk_manager, opt, workers): mturk_agent = workers[0] world = QADataCollectionWorld(opt=opt, mturk_agent=mturk_agent) btime = time.time() world.parley() etime = time.time() logger.debug("DialTime: ", (etime - btime)) world.shutdown() world.review_work() mturk_manager.start_task(eligibility_function=eligibility_function, assign_role_function=assign_worker_roles, task_function=run_conversation) mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) opt.update(task_config) global run_hit def run_hit(i, opt, mturk_manager): # Create mturk agents mturk_agent_1 = MTurkAgent(id='mturk_agent_1', manager=mturk_manager, conversation_id=i, opt=opt) mturk_agent_2 = MTurkAgent(id='mturk_agent_2', manager=mturk_manager, conversation_id=i, opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = 'human_1' human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = 'human_2' world = MultiAgentDialogWorld(opt=opt, agents=[ human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2 ]) # Since we are using the regular MultiAgentDialogWorld, we do the following outside of the world instead. mturk_agent_ids = [mturk_agent_1.id, mturk_agent_2.id] all_agent_ids = [human_agent_1.id, human_agent_2.id] + mturk_agent_ids mturk_agent_1.mturk_agent_ids = mturk_agent_ids mturk_agent_1.all_agent_ids = all_agent_ids mturk_agent_2.mturk_agent_ids = mturk_agent_ids mturk_agent_2.all_agent_ids = all_agent_ids mturk_agent_1.create_hit() mturk_agent_2.create_hit() while not world.episode_done(): world.parley() world.shutdown() mturk_manager = MTurkManager() mturk_manager.init_aws(opt=opt) results = Parallel(n_jobs=opt['num_hits'], backend='threading')( delayed(run_hit)(i, opt, mturk_manager) for i in range(1, opt['num_hits'] + 1)) mturk_manager.review_hits() mturk_manager.shutdown()
def setUp(self): self.work_state_1 = WorkerState(TEST_WORKER_ID_1, 10) self.work_state_2 = WorkerState(TEST_WORKER_ID_2) argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() self.opt = argparser.parse_args([]) self.opt['task'] = 'unittest' self.opt['assignment_duration_in_seconds'] = 6 mturk_agent_ids = ['mturk_agent_1'] self.mturk_manager = MTurkManager(opt=self.opt, mturk_agent_ids=mturk_agent_ids) self.worker_manager = WorkerManager(self.mturk_manager, self.opt)
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() # The dialog model we want to evaluate from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent IrBaselineAgent.add_cmdline_args(argparser) opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) opt.update(task_config) # The task that we will evaluate the dialog model on task_opt = {} task_opt['datatype'] = 'test' task_opt['datapath'] = opt['datapath'] task_opt['task'] = '#MovieDD-Reddit' mturk_manager = MTurkManager() mturk_manager.init_aws(opt=opt) mturk_agent_id = 'Worker' mturk_manager.mturk_agent_ids = [mturk_agent_id] mturk_manager.all_agent_ids = [ ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id ] # In speaking order global run_hit def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager): conversation_id = str(hit_index) + '_' + str(assignment_index) model_agent = IrBaselineAgent(opt=opt) # Create the MTurk agent which provides a chat interface to the Turker mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.create_hits(opt=opt) results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \ (delayed(run_hit)(hit_index, assignment_index, opt, task_opt, mturk_manager) \ for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1))) mturk_manager.review_hits() mturk_manager.shutdown()
def setUp(self): self.agent_state1 = AssignState() self.agent_state2 = AssignState(status=AssignState.STATUS_IN_TASK) argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() self.opt = argparser.parse_args() self.opt['task'] = 'unittest' self.opt['assignment_duration_in_seconds'] = 6 mturk_agent_ids = ['mturk_agent_1'] self.mturk_manager = MTurkManager(opt=self.opt, mturk_agent_ids=mturk_agent_ids) self.worker_manager = self.mturk_manager.worker_manager
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) opt.update(task_config) # Initialize a SQuAD teacher agent, which we will get context from module_name = 'parlai.tasks.squad.agents' class_name = 'DefaultTeacher' my_module = importlib.import_module(module_name) task_class = getattr(my_module, class_name) task_opt = {} task_opt['datatype'] = 'train' task_opt['datapath'] = opt['datapath'] mturk_manager = MTurkManager() mturk_manager.init_aws(opt=opt) mturk_agent_id = 'Worker' mturk_manager.mturk_agent_ids = [mturk_agent_id] mturk_manager.all_agent_ids = [ QADataCollectionWorld.collector_agent_id, mturk_agent_id ] # In speaking order global run_hit def run_hit(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager): conversation_id = str(hit_index) + '_' + str(assignment_index) task = task_class(task_opt) # Create the MTurk agent which provides a chat interface to the Turker mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.create_hits(opt=opt) results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \ (delayed(run_hit)(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager) \ for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1))) mturk_manager.review_hits() mturk_manager.shutdown()
def get_context_generator( override_opt: Optional[Dict[str, Any]] = None) -> ContextGenerator: """ Return an object to return BlendedSkillTalk-style context info (personas, etc.). """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() if override_opt is not None: argparser.set_params(**override_opt) opt = argparser.parse_args([]) context_generator = ContextGenerator(opt, datatype='test', seed=0) # We pull from the test set so that the model can't regurgitate # memorized conversations return context_generator
def add_datapath_and_model_args(opt: Opt): # add datapath, it is missing from parlai.core.params import ParlaiParser, get_model_name parser = ParlaiParser(add_parlai_args=False) parser.add_parlai_data_path() # add model args if they are missing model = get_model_name(opt) if model is not None: parser.add_model_subargs(model, opt) opt_parser = parser.parse_args("") for k, v in opt_parser.items(): if k not in opt: opt[k] = v
def setup_args(): """ Set up args. :return: A parser that takes in command line arguments for chat services (debug, config-path, password), and a port. """ parser = ParlaiParser(False, False) parser.add_parlai_data_path() parser.add_chatservice_args() parser_grp = parser.add_argument_group('Discord Chat') parser_grp.add_argument( '--port', default=35496, type=int, help='Port to run the discord chat server' ) return parser.parse_args()
def setUp(self): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() self.opt = argparser.parse_args(print_args=False) self.opt['task'] = 'unittest' self.opt['assignment_duration_in_seconds'] = 6 mturk_agent_ids = ['mturk_agent_1'] self.mturk_manager = MTurkManager(opt=self.opt.copy(), mturk_agent_ids=mturk_agent_ids) self.worker_manager = self.mturk_manager.worker_manager self.turk_agent = MTurkAgent(self.opt.copy(), self.mturk_manager, TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1, TEST_WORKER_ID_1)
def main(): global run_hit argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) opt['mturk_agent_1_id'] = 'mturk_agent_1' opt['mturk_agent_2_id'] = 'mturk_agent_2' opt['human_agent_1_id'] = 'human_1' opt['human_agent_2_id'] = 'human_2' opt.update(task_config) opt['mturk_agent_ids'] = [opt['mturk_agent_1_id'], opt['mturk_agent_2_id']] opt['all_agent_ids'] = [opt['human_agent_1_id'], opt['human_agent_2_id'] ] + opt['mturk_agent_ids'] opt['run_id'] = str(int(time.time())) def run_hit(i, opt): opt['conversation_id'] = str(i) opt['agent_id'] = opt['mturk_agent_1_id'] mturk_agent_1 = MTurkAgent(opt=opt) opt['agent_id'] = opt['mturk_agent_2_id'] mturk_agent_2 = MTurkAgent(opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = opt['human_agent_1_id'] human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = opt['human_agent_2_id'] world = MultiAgentDialogWorld(opt=opt, agents=[ human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2 ]) while not world.episode_done(): world.parley() world.shutdown() MTurkAgent.init_aws(opt) results = Parallel(n_jobs=opt['num_hits'], backend='threading')( delayed(run_hit)(i, copy.deepcopy(opt)) for i in range(1, opt['num_hits'] + 1)) MTurkAgent.review_hits(opt=opt)
def setup_args(): parser = ParlaiParser(False, False) parser.add_parlai_data_path() merge = parser.add_argument_group('Merge Support Docs') merge.add_argument( '-c', '--c', default=True, type=str, help='Finalize the output (or slice number to merge)', ) merge.add_argument('-n', '--name', default='explainlikeimfive', type=str, help='Slices to merge') return parser.parse_args()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_messenger_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt['password'] = '******' # If password is none anyone can chat messenger_manager = MessengerManager(opt=opt) messenger_manager.setup_server() messenger_manager.init_new_state() def get_overworld(agent): return MessengerOverworld(None, agent) onboard_functions = { name: worlds[0].run for (name, worlds) in MessengerOverworld.DEMOS.items() } messenger_manager.set_onboard_functions(onboard_functions) task_functions = { name: worlds[1].run for (name, worlds) in MessengerOverworld.DEMOS.items() } assign_agent_roles = { name: worlds[1].assign_roles for (name, worlds) in MessengerOverworld.DEMOS.items() } agents_required = { name: worlds[1].MAX_AGENTS for (name, worlds) in MessengerOverworld.DEMOS.items() } messenger_manager.set_agents_required(agents_required) messenger_manager.set_overworld_func(get_overworld) messenger_manager.setup_socket() try: messenger_manager.start_new_run() messenger_manager.start_task( assign_role_functions=assign_agent_roles, task_functions=task_functions, ) except BaseException: raise finally: messenger_manager.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() # The dialog model we want to evaluate from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent IrBaselineAgent.add_cmdline_args(argparser) opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) # The task that we will evaluate the dialog model on task_opt = {} task_opt['datatype'] = 'test' task_opt['datapath'] = opt['datapath'] task_opt['task'] = '#MovieDD-Reddit' mturk_agent_id = 'Worker' mturk_manager = MTurkManager( opt=opt, mturk_agent_ids = [mturk_agent_id], all_agent_ids = [ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id] # In speaking order ) mturk_manager.init_aws(opt=opt) global run_hit def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager): conversation_id = str(hit_index) + '_' + str(assignment_index) model_agent = IrBaselineAgent(opt=opt) # Create the MTurk agent which provides a chat interface to the Turker mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown() world.review_work() mturk_manager.create_hits(opt=opt) results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \ (delayed(run_hit)(hit_index, assignment_index, opt, task_opt, mturk_manager) \ for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1))) mturk_manager.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) mturk_agent_1_id = 'mturk_agent_1' mturk_agent_2_id = 'mturk_agent_2' human_agent_1_id = 'human_1' human_agent_2_id = 'human_2' mturk_manager = MTurkManager( opt=opt, mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id], all_agent_ids = [human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id] # In speaking order ) mturk_manager.init_aws(opt=opt) global run_hit def run_hit(hit_index, assignment_index, opt, mturk_manager): conversation_id = str(hit_index) + '_' + str(assignment_index) # Create mturk agents mturk_agent_1 = MTurkAgent(id=mturk_agent_1_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) mturk_agent_2 = MTurkAgent(id=mturk_agent_2_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = human_agent_1_id human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = human_agent_2_id world = MultiAgentDialogWorld(opt=opt, agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2]) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.create_hits(opt=opt) results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \ (delayed(run_hit)(hit_index, assignment_index, opt, mturk_manager) \ for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1))) mturk_manager.shutdown()
def main(): global run_hit argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) # Initialize a SQuAD teacher agent, which we will later get context from module_name = 'parlai.tasks.squad.agents' class_name = 'DefaultTeacher' my_module = importlib.import_module(module_name) task_class = getattr(my_module, class_name) task_opt = {} task_opt['datatype'] = 'train' task_opt['datapath'] = opt['datapath'] # Create the MTurk agent which provides a chat interface to the Turker opt.update(task_config) mturk_agent_id = 'Worker' opt['agent_id'] = mturk_agent_id opt['mturk_agent_ids'] = [mturk_agent_id] opt['all_agent_ids'] = [ QADataCollectionWorld.collector_agent_id, mturk_agent_id ] opt['run_id'] = str(int(time.time())) def run_hit(i, task_class, task_opt, opt): task = task_class(task_opt) opt['conversation_id'] = str(i) mturk_agent = MTurkAgent(opt=opt) world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown() MTurkAgent.init_aws(opt) results = Parallel(n_jobs=opt['num_hits'], backend='threading')( delayed(run_hit)(i, task_class, task_opt, copy.deepcopy(opt)) for i in range(1, opt['num_hits'] + 1)) MTurkAgent.review_hits(opt=opt)
def setup_args(parser=None): if parser is None: parser = ParlaiParser(True, True, 'Evaluate a model') parser.add_parlai_data_path() # Get command line arguments parser.add_argument('-tmf', '--tourist-model-file', type=str) parser.add_argument('-gmf', '--guide-model-file', type=str) parser.add_argument('-ne', '--num-examples', type=int, default=-1) parser.add_argument('-d', '--display-examples', type='bool', default=False) parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=2) parser.add_argument('--metrics', type=str, default="all", help="list of metrics to show/compute, e.g. " "ppl,f1,accuracy,hits@1." "If 'all' is specified [default] all are shown.") TensorboardLogger.add_cmdline_args(parser) parser.set_defaults(datatype='valid') return parser
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) # Initialize a SQuAD teacher agent, which we will get context from module_name = 'parlai.tasks.squad.agents' class_name = 'DefaultTeacher' my_module = importlib.import_module(module_name) task_class = getattr(my_module, class_name) task_opt = {} task_opt['datatype'] = 'train' task_opt['datapath'] = opt['datapath'] mturk_agent_id = 'Worker' mturk_manager = MTurkManager( opt=opt, mturk_agent_ids = [mturk_agent_id], all_agent_ids = [QADataCollectionWorld.collector_agent_id, mturk_agent_id] # In speaking order ) mturk_manager.init_aws(opt=opt) global run_hit def run_hit(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager): conversation_id = str(hit_index) + '_' + str(assignment_index) task = task_class(task_opt) # Create the MTurk agent which provides a chat interface to the Turker mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown() world.review_work() mturk_manager.create_hits(opt=opt) results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \ (delayed(run_hit)(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager) \ for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1))) mturk_manager.shutdown()
def get_context_generator( override_opt: Optional[Dict[str, Any]] = None, task: Optional[str] = 'blended_skill_talk', **kwargs, ) -> ContextGenerator: """ Return an object to return BlendedSkillTalk-style context info (personas, etc.). """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() if override_opt is not None: argparser.set_params(**override_opt) opt = argparser.parse_args([]) task_module = load_task_module(task) context_generator_class = getattr(task_module, 'ContextGenerator', None) context_generator = context_generator_class(opt, datatype='test', seed=0, **kwargs) # We pull from the test set so that the model can't regurgitate # memorized conversations return context_generator
def main(): """ Main function for the DMG pilot data collection task :return: Nothing. """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() opt = argparser.parse_args() opt['task'] = 'dmg_pilot_dev' opt['datatype'] = 'dmg_pilot_data_1' opt.update(task_config) local_agent_1 = LocalHumanAgent(opt=None) local_agent_1.id = 'A' local_agent_2 = LocalHumanAgent(opt=None) local_agent_2.id = 'B' agents = [local_agent_1, local_agent_2] world = LocalDMGDialogWorld(opt=opt, agents=agents) # Loop over all five rounds of the game for r in range(5): print("--- Starting round {} ---".format(r + 1)) while not world.episode_done(): world.parley() # Write the log data to file with open('logs/dmg_pilot_data_{}.json'.format(world.game_nr), 'w') as f: json.dump(world.conversation_log, f) # Reset the world for the next round world.round_log['data'] = [] world.round_log['score'] = None world.selections = defaultdict(lambda: dict()) world.turn_nr = -1 world.round_nr += 1 world.episodeDone = False world.shutdown()
def setUp(self): disconnect_path = os.path.join(parent_dir, 'disconnect-test.pickle') if os.path.exists(disconnect_path): os.remove(disconnect_path) argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() self.opt = argparser.parse_args([]) self.opt['task'] = 'unittest' self.opt['assignment_duration_in_seconds'] = 6 mturk_agent_ids = ['mturk_agent_1'] self.mturk_manager = MTurkManager( opt=self.opt.copy(), mturk_agent_ids=mturk_agent_ids ) self.worker_manager = self.mturk_manager.worker_manager self.worker_state_1 = self.worker_manager.worker_alive(TEST_WORKER_ID_1) self.worker_state_2 = self.worker_manager.worker_alive(TEST_WORKER_ID_2) self.worker_state_3 = self.worker_manager.worker_alive(TEST_WORKER_ID_3)
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) mturk_agent_1_id = 'mturk_agent_1' mturk_agent_2_id = 'mturk_agent_2' human_agent_1_id = 'human_1' human_agent_2_id = 'human_2' # Create the MTurk agents opt.update(task_config) opt['conversation_id'] = str(int(time.time())) opt['mturk_agent_ids'] = [mturk_agent_1_id, mturk_agent_2_id] opt['all_agent_ids'] = [ human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id ] opt['agent_id'] = mturk_agent_1_id mturk_agent_1 = MTurkAgent(opt=opt) opt['agent_id'] = mturk_agent_2_id mturk_agent_2 = MTurkAgent(opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = human_agent_1_id human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = human_agent_2_id world = MultiAgentDialogWorld( opt=opt, agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2]) while not world.episode_done(): world.parley() world.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() # The dialog model we want to evaluate from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent IrBaselineAgent.add_cmdline_args(argparser) opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) model_agent = IrBaselineAgent(opt=opt) # The task that we will evaluate the dialog model on task_opt = {} task_opt['datatype'] = 'test' task_opt['datapath'] = opt['datapath'] task_opt['task'] = '#MovieDD-Reddit' # Create the MTurk agent which provides a chat interface to the Turker opt.update(task_config) mturk_agent_id = 'Worker' opt['agent_id'] = mturk_agent_id opt['mturk_agent_ids'] = [mturk_agent_id] opt['all_agent_ids'] = [ ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id ] opt['conversation_id'] = str(int(time.time())) mturk_agent = MTurkAgent(opt=opt) world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown()
argparser.add_arg('--room_ablation', type=bool, default=False) # ============ above copied from projects/graph_world2/train.py ============ argparser.add_argument('--num_machines', type=int, default=1) argparser.add_argument('--job_timeout', type=float, default=3600*4) argparser.add_argument('--split', action='store_true', default=False) argparser.add_argument('--train', action='store_true', default=False) argparser.add_argument('--eval', action='store_true', default=False) argparser.add_argument('--seq2seq', action='store_true', default=False) argparser.add_argument('--constrain', action='store_true', default=False) argparser.add_argument('--rounds_breakdown', action='store_true', default=False) argparser.add_argument('--data_breakdown', action='store_true', default=False) argparser.add_argument('--ablation', action='store_true', default=False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() # ============ below copied from projects/graph_world2/train.py ============ opt['bidir'] = True opt['action_type_emb_dim'] = 5 opt['counter_max'] = 3 opt['counter_emb_dim'] = 5 # ============ above copied from projects/graph_world2/train.py ============ if opt['split']: overall_split(opt) quit() if opt['train']: overall_run(opt, seq2seq=opt['seq2seq'])
def main(): """ This task consists of one local human agent and two MTurk agents, each MTurk agent will go through the onboarding step to provide information about themselves, before being put into a conversation. You can end the conversation by sending a message ending with `[DONE]` from human_1. """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) mturk_agent_1_id = 'mturk_agent_1' mturk_agent_2_id = 'mturk_agent_2' human_agent_1_id = 'human_1' mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id] mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids) mturk_manager.setup_server() try: mturk_manager.start_new_run() mturk_manager.create_hits() def run_onboard(worker): world = MTurkMultiAgentDialogOnboardWorld(opt=opt, mturk_agent=worker) while not world.episode_done(): world.parley() world.shutdown() # You can set onboard_function to None to skip onboarding mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): # Create mturk agents mturk_agent_1 = workers[0] mturk_agent_2 = workers[1] # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = human_agent_1_id world = MTurkMultiAgentDialogWorld( opt=opt, agents=[human_agent_1, mturk_agent_1, mturk_agent_2]) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.start_task(eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of one agent, model or MTurk worker, talking to an MTurk worker to negotiate a deal. """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('-min_t', '--min_turns', default=5, type=int, help='minimum number of turns') argparser.add_argument('-mt', '--max_turns', default=10, type=int, help='maximal number of chat turns') argparser.add_argument( '-mx_rsp_time', '--max_resp_time', default=150, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '--ag_shutdown_time', default=120, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '--persona-type', default='both', type=str, choices=['both', 'self', 'other'], help='Which personas to load from personachat', ) opt = argparser.parse_args() directory_path = os.path.dirname(os.path.abspath(__file__)) opt['task'] = os.path.basename(directory_path) opt['extract_personas_path'] = os.path.join(opt['datapath'], opt['task']) opt.update(task_config) mturk_agent_ids = ['PERSON_1'] mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids) mturk_manager.setup_server(task_directory_path=directory_path) personas_generator = PersonasGenerator(opt) opt['personas_generator'] = personas_generator try: mturk_manager.start_new_run() mturk_manager.create_hits() if not opt['is_sandbox']: # ADD BLOCKED WORKERS HERE blocked_worker_list = [] for w in blocked_worker_list: mturk_manager.block_worker( w, 'We found that you have unexpected behaviors in our ' 'previous HITs. For more questions please email us.', ) def run_onboard(worker): pass mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): worker = workers[0] world = RephrasePersonaWorld(opt, worker) while not world.episode_done(): world.parley() world.save_data() world.shutdown() world.review_work() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation, ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def setUp(self): patcher = mock.patch('builtins.input', return_value='y') self.addCleanup(patcher.stop) patcher.start() # Mock functions that hit external APIs and such self.server_utils = MTurkManagerFile.server_utils self.mturk_utils = MTurkManagerFile.mturk_utils self.server_utils.setup_server = mock.MagicMock( return_value='https://127.0.0.1') self.server_utils.setup_legacy_server = mock.MagicMock( return_value='https://127.0.0.1') self.server_utils.delete_server = mock.MagicMock() self.mturk_utils.setup_aws_credentials = mock.MagicMock() self.mturk_utils.calculate_mturk_cost = mock.MagicMock(return_value=1) self.mturk_utils.check_mturk_balance = mock.MagicMock( return_value=True) self.mturk_utils.create_hit_config = mock.MagicMock() self.mturk_utils.setup_sns_topic = mock.MagicMock( return_value=TOPIC_ARN) self.mturk_utils.delete_sns_topic = mock.MagicMock() self.mturk_utils.delete_qualification = mock.MagicMock() self.mturk_utils.find_or_create_qualification = mock.MagicMock( return_value=QUALIFICATION_ID) self.mturk_utils.find_qualification = mock.MagicMock( return_value=QUALIFICATION_ID) self.mturk_utils.give_worker_qualification = mock.MagicMock() self.mturk_utils.remove_worker_qualification = mock.MagicMock() self.mturk_utils.create_hit_type = mock.MagicMock( return_value=HIT_TYPE_ID) self.mturk_utils.subscribe_to_hits = mock.MagicMock() self.mturk_utils.create_hit_with_hit_type = mock.MagicMock( return_value=(MTURK_PAGE_URL, FAKE_HIT_ID, 'MTURK_HIT_DATA')) self.mturk_utils.get_mturk_client = mock.MagicMock( return_value=mock.MagicMock()) self.onboarding_agents = {} self.worlds_agents = {} # Set up an MTurk Manager and get it ready for accepting workers self.fake_socket = MockSocket() time.sleep(0.1) argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() self.opt = argparser.parse_args() self.opt['task'] = 'unittest' self.opt['assignment_duration_in_seconds'] = 1 self.opt['hit_title'] = 'test_hit_title' self.opt['hit_description'] = 'test_hit_description' self.opt['task_description'] = 'test_task_description' self.opt['hit_keywords'] = 'test_hit_keywords' self.opt['reward'] = 0.1 self.opt['is_debug'] = True self.opt['log_level'] = 0 self.opt['num_conversations'] = 1 self.mturk_agent_ids = ['mturk_agent_1', 'mturk_agent_2'] self.mturk_manager = MTurkManager(opt=self.opt, mturk_agent_ids=self.mturk_agent_ids, is_test=True) self.mturk_manager.port = self.fake_socket.port self.mturk_manager.setup_server() self.mturk_manager.start_new_run() self.mturk_manager.ready_to_accept_workers() self.mturk_manager.set_onboard_function(self.onboard_agent) self.mturk_manager.create_hits() def assign_worker_roles(workers): workers[0].id = 'mturk_agent_1' workers[1].id = 'mturk_agent_2' def run_task_wait(): self.mturk_manager.start_task(lambda w: True, assign_worker_roles, self.run_conversation) self.task_thread = threading.Thread(target=run_task_wait) self.task_thread.start() self.agent_1 = MockAgent(TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1, TEST_WORKER_ID_1, TASK_GROUP_ID_1) self.agent_1_2 = MockAgent(TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_3, TEST_WORKER_ID_1, TASK_GROUP_ID_1) self.agent_2 = MockAgent(TEST_HIT_ID_2, TEST_ASSIGNMENT_ID_2, TEST_WORKER_ID_2, TASK_GROUP_ID_1)
def main(): """ This task consists of one local human agent and two MTurk agents, each MTurk agent will go through the onboarding step to provide information about themselves, before being put into a conversation. You can end the conversation by sending a message ending with `[DONE]` from human_1. """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) mturk_agent_1_id = 'mturk_agent_1' mturk_agent_2_id = 'mturk_agent_2' human_agent_1_id = 'human_1' mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id] mturk_manager = MTurkManager( opt=opt, mturk_agent_ids=mturk_agent_ids ) mturk_manager.setup_server() try: mturk_manager.start_new_run() mturk_manager.create_hits() def run_onboard(worker): world = MTurkMultiAgentDialogOnboardWorld( opt=opt, mturk_agent=worker ) while not world.episode_done(): world.parley() world.shutdown() # You can set onboard_function to None to skip onboarding mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True eligibility_function = { 'func': check_worker_eligibility, 'multiple': False, } def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): # Create mturk agents mturk_agent_1 = workers[0] mturk_agent_2 = workers[1] # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = human_agent_1_id world = MTurkMultiAgentDialogWorld( opt=opt, agents=[human_agent_1, mturk_agent_1, mturk_agent_2] ) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.start_task( eligibility_function=eligibility_function, assign_role_function=assign_worker_roles, task_function=run_conversation ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): completed_workers = [] argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) mturk_agent_id = 'Worker' mturk_manager = MTurkManager( opt=opt, mturk_agent_ids=[mturk_agent_id] ) mturk_manager.setup_server() qual_name = 'ParlAIExcludeQual{}t{}'.format( random.randint(10000, 99999), random.randint(10000, 99999)) qual_desc = ( 'Qualification for a worker not correctly completing the ' 'first iteration of a task. Used to filter to different task pools.' ) qualification_id = \ mturk_utils.find_or_create_qualification(qual_name, qual_desc) print('Created qualification: ', qualification_id) def run_onboard(worker): world = QualificationFlowOnboardWorld(opt, worker) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) try: mturk_manager.start_new_run() agent_qualifications = [{ 'QualificationTypeId': qualification_id, 'Comparator': 'DoesNotExist', 'RequiredToPreview': True }] mturk_manager.create_hits(qualifications=agent_qualifications) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(worker): worker[0].id = mturk_agent_id global run_conversation def run_conversation(mturk_manager, opt, workers): mturk_agent = workers[0] world = QualificationFlowSoloWorld( opt=opt, mturk_agent=mturk_agent, qualification_id=qualification_id, firstTime=(mturk_agent.worker_id not in completed_workers), ) while not world.episode_done(): world.parley() completed_workers.append(mturk_agent.worker_id) world.shutdown() world.review_work() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation ) except BaseException: raise finally: mturk_utils.delete_qualification(qualification_id) mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of one agent, model or MTurk worker, talking to an MTurk worker to negotiate a deal. """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('--two_mturk_agents', dest='two_mturk_agents', action='store_true', help='data collection mode ' 'with converations between two MTurk agents') opt = argparser.parse_args() opt['task'] = 'dealnodeal' opt['datatype'] = 'valid' opt.update(task_config) local_agent_1_id = 'local_1' mturk_agent_ids = ['mturk_agent_1'] if opt['two_mturk_agents']: mturk_agent_ids.append('mturk_agent_2') mturk_manager = MTurkManager( opt=opt, mturk_agent_ids=mturk_agent_ids ) mturk_manager.setup_server() try: mturk_manager.start_new_run() mturk_manager.create_hits() mturk_manager.set_onboard_function(onboard_function=None) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): agents = workers[:] # Create a local agent if not opt['two_mturk_agents']: if 'model' in opt: local_agent = create_agent(opt) else: local_agent = LocalHumanAgent(opt=None) local_agent.id = local_agent_1_id agents.append(local_agent) opt["batchindex"] = mturk_manager.started_conversations world = MTurkDealNoDealDialogWorld( opt=opt, agents=agents ) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() # The dialog model we want to evaluate from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent IrBaselineAgent.add_cmdline_args(argparser) opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) # The task that we will evaluate the dialog model on task_opt = {} task_opt['datatype'] = 'test' task_opt['datapath'] = opt['datapath'] task_opt['task'] = '#MovieDD-Reddit' mturk_agent_id = 'Worker' mturk_manager = MTurkManager( opt=opt, mturk_agent_ids=[mturk_agent_id] ) mturk_manager.setup_server() try: mturk_manager.start_new_run() mturk_manager.create_hits() def run_onboard(worker): world = ModelEvaluatorOnboardWorld(opt=opt, mturk_agent=worker) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(worker): worker[0].id = mturk_agent_id global run_conversation def run_conversation(mturk_manager, opt, workers): mturk_agent = workers[0] model_agent = IrBaselineAgent(opt=opt) world = ModelEvaluatorWorld( opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent ) while not world.episode_done(): world.parley() world.shutdown() world.review_work() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) # Initialize a SQuAD teacher agent, which we will get context from module_name = 'parlai.tasks.squad.agents' class_name = 'DefaultTeacher' my_module = importlib.import_module(module_name) task_class = getattr(my_module, class_name) task_opt = {} task_opt['datatype'] = 'train' task_opt['datapath'] = opt['datapath'] mturk_agent_id = 'Worker' mturk_manager = MTurkManager( opt=opt, mturk_agent_ids=[mturk_agent_id] ) mturk_manager.setup_server() def run_onboard(worker): world = QADataCollectionOnboardWorld(opt=opt, mturk_agent=worker) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=None) try: mturk_manager.start_new_run() mturk_manager.create_hits() mturk_manager.ready_to_accept_workers() def check_workers_eligibility(workers): return workers eligibility_function = { 'func': check_workers_eligibility, 'multiple': True, } def assign_worker_roles(worker): worker[0].id = mturk_agent_id global run_conversation def run_conversation(mturk_manager, opt, workers): task = task_class(task_opt) mturk_agent = workers[0] world = QADataCollectionWorld( opt=opt, task=task, mturk_agent=mturk_agent ) while not world.episode_done(): world.parley() world.shutdown() world.review_work() mturk_manager.start_task( eligibility_function=eligibility_function, assign_role_function=assign_worker_roles, task_function=run_conversation ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of an MTurk agent evaluating a chit-chat model. They are asked to chat to the model adopting a specific persona. After their conversation, they are asked to evaluate their partner on several metrics. """ argparser = ParlaiParser(False, add_model_args=True) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('-mt', '--max-turns', default=10, type=int, help='maximal number of chat turns') argparser.add_argument('--max-resp-time', default=240, type=int, help='time limit for entering a dialog message') argparser.add_argument('--max-persona-time', type=int, default=300, help='time limit for turker' 'entering the persona') argparser.add_argument('--ag-shutdown-time', default=120, type=int, help='time limit for entering a dialog message') argparser.add_argument('--persona-type', default='both', type=str, choices=['both', 'self', 'other'], help='Which personas to load from personachat') argparser.add_argument('--revised', default=False, type='bool', help='Whether to use revised personas') argparser.add_argument('-rt', '--range-turn', default='5,6', help='sample range of number of turns') argparser.add_argument('--auto-approve-delay', type=int, default=3600 * 24 * 1, help='how long to wait for auto approval') argparser.add_argument('--only-masters', type='bool', default=False, help='Set to True to use only master turks for this' + ' test eval, default is %(default)s') # ADD MODEL ARGS HERE, UNCOMMENT TO USE KVMEMNN MODEL AS AN EXAMPLE # argparser.set_defaults( # model='projects.personachat.kvmemnn.kvmemnn:Kvmemnn', # model_file='models:convai2/kvmemnn/model', # ) opt = argparser.parse_args() # add additional model args opt['override'] = { 'no_cuda': True, 'interactive_mode': True, 'tensorboard_log': False } bot = create_agent(opt) shared_bot_params = bot.share() print( '=== Actual bot opt === :\n {}'.format( '\n'.join(["[{}] : {}".format(k, v) for k, v in bot.opt.items()]) ) ) folder_name = ( 'master_{}_YOURCOMMENT__'.format(opt['only_masters']) + '__'.join(['{}_{}'.format(k, v) for k, v in opt['override'].items()]) ) # this is mturk task, not convai2 task from ParlAI opt['task'] = 'convai2:self' if 'data_path' not in opt: opt['data_path'] = os.getcwd() + '/data/' + folder_name opt.update(task_config) mturk_agent_ids = ['PERSON_1'] mturk_manager = MTurkManager( opt=opt, mturk_agent_ids=mturk_agent_ids ) persona_generator = PersonasGenerator(opt) mturk_manager.setup_server() try: mturk_manager.start_new_run() agent_qualifications = [] if opt['only_masters']: if opt['is_sandbox']: agent_qualifications.append(MASTER_QUALIF_SDBOX) else: agent_qualifications.append(MASTER_QUALIF) mturk_manager.create_hits(qualifications=agent_qualifications) if not opt['is_sandbox']: # ADD SOFT-BLOCKED WORKERS HERE # NOTE: blocking qual *must be* specified blocked_worker_list = [] for w in blocked_worker_list: print('Soft Blocking {}\n'.format(w)) mturk_manager.soft_block_worker(w) time.sleep(0.1) # do the sleep to prevent amazon query drop def run_onboard(worker): worker.persona_generator = persona_generator world = PersonaProfileWorld(opt, worker) world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): agents = workers[0] conv_idx = mturk_manager.conversation_index world = Convai2EvalWorld( opt=opt, agents=[agents], range_turn=[int(s) for s in opt['range_turn'].split(',')], max_turn=opt['max_turns'], max_resp_time=opt['max_resp_time'], model_agent_opt=shared_bot_params, world_tag='conversation t_{}'.format(conv_idx), agent_timeout_shutdown=opt['ag_shutdown_time'], ) world.reset_random() while not world.episode_done(): world.parley() world.save_data() world.shutdown() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) mturk_agent_1_id = 'mturk_agent_1' mturk_agent_2_id = 'mturk_agent_2' human_agent_1_id = 'human_1' mturk_agent_ids = [mturk_agent_1_id, mturk_agent_2_id] mturk_manager = MTurkManager( opt=opt, mturk_agent_ids = mturk_agent_ids ) mturk_manager.setup_server() try: mturk_manager.start_new_run() mturk_manager.create_hits() def run_onboard(worker): world = MTurkMultiAgentDialogOnboardWorld( opt=opt, mturk_agent=worker ) while not world.episode_done(): world.parley() world.shutdown() # You can set onboard_function to None to skip onboarding mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): # Create mturk agents mturk_agent_1 = workers[0] mturk_agent_2 = workers[1] # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = human_agent_1_id world = MTurkMultiAgentDialogWorld( opt=opt, agents=[human_agent_1, mturk_agent_1, mturk_agent_2] ) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation ) except: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()