def main(): """This task consists of one agent, model or MTurk worker, talking to an MTurk worker to negotiate a deal. """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('-min_t', '--min_turns', default=5, type=int, help='minimum number of turns') argparser.add_argument('-mt', '--max_turns', default=10, type=int, help='maximal number of chat turns') argparser.add_argument( '-mx_rsp_time', '--max_resp_time', default=150, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '--ag_shutdown_time', default=120, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '--persona-type', default='both', type=str, choices=['both', 'self', 'other'], help='Which personas to load from personachat', ) opt = argparser.parse_args() directory_path = os.path.dirname(os.path.abspath(__file__)) opt['task'] = os.path.basename(directory_path) opt['extract_personas_path'] = os.path.join(opt['datapath'], opt['task']) opt.update(task_config) mturk_agent_ids = ['PERSON_1'] mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids) mturk_manager.setup_server(task_directory_path=directory_path) personas_generator = PersonasGenerator(opt) opt['personas_generator'] = personas_generator try: mturk_manager.start_new_run() mturk_manager.create_hits() if not opt['is_sandbox']: # ADD BLOCKED WORKERS HERE blocked_worker_list = [] for w in blocked_worker_list: mturk_manager.block_worker( w, 'We found that you have unexpected behaviors in our ' 'previous HITs. For more questions please email us.', ) def run_onboard(worker): pass mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): worker = workers[0] world = RephrasePersonaWorld(opt, worker) while not world.episode_done(): world.parley() world.save_data() world.shutdown() world.review_work() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation, ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of an MTurk agent evaluating a chit-chat model. They are asked to chat to the model adopting a specific persona. After their conversation, they are asked to evaluate their partner on several metrics. """ argparser = ParlaiParser(False, add_model_args=True) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('-mt', '--max-turns', default=10, type=int, help='maximal number of chat turns') argparser.add_argument('--max-resp-time', default=240, type=int, help='time limit for entering a dialog message') argparser.add_argument('--max-persona-time', type=int, default=300, help='time limit for turker' 'entering the persona') argparser.add_argument('--ag-shutdown-time', default=120, type=int, help='time limit for entering a dialog message') argparser.add_argument('--persona-type', default='both', type=str, choices=['both', 'self', 'other'], help='Which personas to load from personachat') argparser.add_argument('--revised', default=False, type='bool', help='Whether to use revised personas') argparser.add_argument('-rt', '--range-turn', default='5,6', help='sample range of number of turns') argparser.add_argument('--auto-approve-delay', type=int, default=3600 * 24 * 1, help='how long to wait for auto approval') argparser.add_argument('--only-masters', type='bool', default=False, help='Set to True to use only master turks for this' + ' test eval, default is %(default)s') # ADD MODEL ARGS HERE, UNCOMMENT TO USE KVMEMNN MODEL AS AN EXAMPLE # argparser.set_defaults( # model='projects.personachat.kvmemnn.kvmemnn:Kvmemnn', # model_file='models:convai2/kvmemnn/model', # ) opt = argparser.parse_args() # add additional model args opt['override'] = { 'no_cuda': True, 'interactive_mode': True, 'tensorboard_log': False } bot = create_agent(opt) shared_bot_params = bot.share() print( '=== Actual bot opt === :\n {}'.format( '\n'.join(["[{}] : {}".format(k, v) for k, v in bot.opt.items()]) ) ) folder_name = ( 'master_{}_YOURCOMMENT__'.format(opt['only_masters']) + '__'.join(['{}_{}'.format(k, v) for k, v in opt['override'].items()]) ) # this is mturk task, not convai2 task from ParlAI opt['task'] = 'convai2:self' if 'data_path' not in opt: opt['data_path'] = os.getcwd() + '/data/' + folder_name opt.update(task_config) mturk_agent_ids = ['PERSON_1'] mturk_manager = MTurkManager( opt=opt, mturk_agent_ids=mturk_agent_ids ) persona_generator = PersonasGenerator(opt) mturk_manager.setup_server() try: mturk_manager.start_new_run() agent_qualifications = [] if opt['only_masters']: if opt['is_sandbox']: agent_qualifications.append(MASTER_QUALIF_SDBOX) else: agent_qualifications.append(MASTER_QUALIF) mturk_manager.create_hits(qualifications=agent_qualifications) if not opt['is_sandbox']: # ADD SOFT-BLOCKED WORKERS HERE # NOTE: blocking qual *must be* specified blocked_worker_list = [] for w in blocked_worker_list: print('Soft Blocking {}\n'.format(w)) mturk_manager.soft_block_worker(w) time.sleep(0.1) # do the sleep to prevent amazon query drop def run_onboard(worker): worker.persona_generator = persona_generator world = PersonaProfileWorld(opt, worker) world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): agents = workers[0] conv_idx = mturk_manager.conversation_index world = Convai2EvalWorld( opt=opt, agents=[agents], range_turn=[int(s) for s in opt['range_turn'].split(',')], max_turn=opt['max_turns'], max_resp_time=opt['max_resp_time'], model_agent_opt=shared_bot_params, world_tag='conversation t_{}'.format(conv_idx), agent_timeout_shutdown=opt['ag_shutdown_time'], ) world.reset_random() while not world.episode_done(): world.parley() world.save_data() world.shutdown() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of an MTurk agent evaluating a Controllable Dialog model. """ start_time = datetime.datetime.today().strftime('%Y-%m-%d-%H-%M') argparser = ParlaiParser(False, add_model_args=True) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument( '--max-resp-time', default=240, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '--max-choice-time', type=int, default=300, help='time limit for turker' 'choosing the topic', ) argparser.add_argument( '--ag-shutdown-time', default=120, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '--num-turns', default=6, type=int, help='number of turns of dialogue' ) argparser.add_argument( '--human-eval', type='bool', default=False, help='human vs human eval, no models involved', ) argparser.add_argument( '--auto-approve-delay', type=int, default=3600 * 24 * 2, help='how long to wait for auto approval', ) argparser.add_argument( '--only-masters', type='bool', default=False, help='Set to true to use only master turks for ' 'this test eval', ) argparser.add_argument( '--create-model-qualif', type='bool', default=True, help='Create model qualif so unique eval between' 'models.', ) argparser.add_argument( '--limit-workers', type=int, default=len(SETTINGS_TO_RUN), help='max HITs a worker can complete', ) argparser.add_argument( '--mturk-log', type=str, default=('data/mturklogs/controllable/{}.log'.format(start_time)), ) argparser.add_argument( '--short-eval', type='bool', default=True, help='Only ask engagingness question and persona' 'question.', ) # persona specific arguments argparser.add_argument( '--persona-type', type=str, default='self', choices=['self', 'other', 'none'] ) argparser.add_argument( '--persona-datatype', type=str, default='valid', choices=['train', 'test', 'valid'], ) argparser.add_argument( '--max-persona-time', type=int, default=360, help='max time to view persona' ) def get_logger(opt): fmt = '%(asctime)s: [ %(message)s ]' logfn = None if 'mturk_log' in opt: logfn = opt['mturk_log'] if not os.path.isdir(os.path.dirname(logfn)): os.makedirs(os.path.dirname(logfn), exist_ok=True) logger = ParlaiLogger( name="mturk_controllable", console_level=INFO, file_level=INFO, console_format=fmt, file_format=fmt, filename=logfn, ) logger.info('COMMAND: %s' % ' '.join(sys.argv)) logger.info('-' * 100) logger.info('CONFIG:\n%s' % json.dumps(opt, indent=4, sort_keys=True)) return logger start_opt = argparser.parse_args() task_config['task_description'] = task_config['task_description'].format( start_opt['reward'] ) # set options start_opt['limit_workers'] = len(SETTINGS_TO_RUN) start_opt['allowed_conversations'] = 1 start_opt['max_hits_per_worker'] = start_opt['limit_workers'] start_opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) start_opt.update(task_config) logger = get_logger(start_opt) model_share_params = {} worker_models_seen = {} model_opts = {} model_counts = {} lock = Lock() for setup in SETTINGS_TO_RUN: assert 'human' not in setup model_counts[setup] = 0 agent_config = getattr(mcf, setup) combined_config = copy.deepcopy(start_opt) for k, v in agent_config.items(): combined_config[k] = v combined_config['override'][k] = v folder_name = '{}-{}'.format(setup, start_time) combined_config['save_data_path'] = os.path.join( start_opt['datapath'], 'local_controllable_dialogue', folder_name ) model_opts[setup] = combined_config bot = create_agent(combined_config, True) model_share_params[setup] = bot.share() if not start_opt.get('human_eval'): mturk_agent_ids = ['PERSON_1'] else: mturk_agent_ids = ['PERSON_1', 'PERSON_2'] mturk_manager = MTurkManager(opt=start_opt, mturk_agent_ids=mturk_agent_ids) personas_generator = PersonasGenerator(start_opt) directory_path = os.path.dirname(os.path.abspath(__file__)) mturk_manager.setup_server(task_directory_path=directory_path) try: mturk_manager.start_new_run() agent_qualifications = [] # assign qualifications if start_opt['create_model_qualif']: qual_name = 'ControlEvalRound2' qual_desc = ( 'Qualification to ensure workers complete only a certain' 'number of these HITs' ) qualification_id = mturk_utils.find_or_create_qualification( qual_name, qual_desc, False ) print('Created qualification: ', qualification_id) start_opt['unique_qualif_id'] = qualification_id def run_onboard(worker): worker.personas_generator = personas_generator world = PersonaAssignWorld(start_opt, worker) world.parley() world.shutdown() def check_worker_eligibility(worker): worker_id = worker.worker_id lock.acquire() retval = len(worker_models_seen.get(worker_id, [])) < len(SETTINGS_TO_RUN) lock.release() return retval def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() mturk_manager.create_hits(qualifications=agent_qualifications) def run_conversation(mturk_manager, opt, workers): conv_idx = mturk_manager.conversation_index # gotta find a bot this worker hasn't seen yet assert len(workers) == 1 worker_id = workers[0].worker_id lock.acquire() if worker_id not in worker_models_seen: worker_models_seen[worker_id] = set() print("MODELCOUNTS:") print(pprint.pformat(model_counts)) logger.info("MODELCOUNTS\n" + pprint.pformat(model_counts)) model_options = [ (model_counts[setup_name] + 10 * random.random(), setup_name) for setup_name in SETTINGS_TO_RUN if setup_name not in worker_models_seen[worker_id] ] if not model_options: lock.release() logger.error( "Worker {} already finished all settings! Returning none".format( worker_id ) ) return None _, model_choice = min(model_options) worker_models_seen[worker_id].add(model_choice) model_counts[model_choice] += 1 lock.release() world = ControllableDialogEval( opt=model_opts[model_choice], agents=workers, num_turns=start_opt['num_turns'], max_resp_time=start_opt['max_resp_time'], model_agent_opt=model_share_params[model_choice], world_tag='conversation t_{}'.format(conv_idx), agent_timeout_shutdown=opt['ag_shutdown_time'], model_config=model_choice, ) world.reset_random() while not world.episode_done(): world.parley() world.save_data() lock.acquire() if not world.convo_finished: model_counts[model_choice] -= 1 worker_models_seen[worker_id].remove(model_choice) lock.release() world.shutdown() gc.collect() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation, ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """ Wizard of Wikipedia Data Collection Task. The task involves two people holding a conversation. One dialog partner chooses a topic to discuss, and then dialog proceeds. One partner is the Wizard, who has access to retrieved external information conditioned on the last two utterances, as well as information regarding the chosen topic. The other partner is the Apprentice, who assumes the role of someone eager to learn about the chosen topic. """ argparser = ParlaiParser(False, False) DictionaryAgent.add_cmdline_args(argparser) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument( '-min_t', '--min_turns', default=3, type=int, help='minimum number of turns' ) argparser.add_argument( '-max_t', '--max_turns', default=5, type=int, help='maximal number of chat turns', ) argparser.add_argument( '-mx_rsp_time', '--max_resp_time', default=120, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '-mx_onb_time', '--max_onboard_time', type=int, default=300, help='time limit for turker' 'in onboarding', ) argparser.add_argument( '--persona-type', default='both', type=str, choices=['both', 'self', 'other'], help='Which personas to load from personachat', ) argparser.add_argument( '--auto-approve-delay', type=int, default=3600 * 24 * 1, help='how long to wait for \ auto approval', ) argparser.add_argument( '--word-overlap-threshold', type=int, default=2, help='How much word overlap we want between message \ and checked sentence', ) argparser.add_argument( '--num-good-sentence-threshold', type=int, default=2, help='How many good sentences with sufficient overlap \ are necessary for turker to be considered good.', ) argparser.add_argument( '--num-passages-retrieved', type=int, default=7, help='How many passages to retrieve per dialog \ message', ) opt = argparser.parse_args() directory_path = os.path.dirname(os.path.abspath(__file__)) opt['task'] = os.path.basename(directory_path) if 'data_path' not in opt: opt['data_path'] = os.getcwd() + '/data/' + opt['task'] opt['current_working_dir'] = os.getcwd() opt.update(task_config) mturk_agent_ids = [APPRENTICE, WIZARD] opt['min_messages'] = 2 mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids) setup_personas_with_wiki_links(opt) ir_agent, task = setup_retriever(opt) persona_generator = PersonasGenerator(opt) wiki_title_to_passage = setup_title_to_passage(opt) mturk_manager.setup_server(task_directory_path=directory_path) worker_roles = {} connect_counter = AttrDict(value=0) try: mturk_manager.start_new_run() if not opt['is_sandbox']: with open(os.path.join(opt['current_working_dir'], 'mtdont.txt')) as f: lines = [l.replace('\n', '') for l in f.readlines()] for w in lines: mturk_manager.soft_block_worker(w) def run_onboard(worker): role = mturk_agent_ids[connect_counter.value % len(mturk_agent_ids)] connect_counter.value += 1 worker_roles[worker.worker_id] = role worker.persona_generator = persona_generator world = RoleOnboardWorld(opt, worker, role) world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() mturk_manager.create_hits() def check_workers_eligibility(workers): if opt['is_sandbox']: return workers valid_workers = {} for worker in workers: worker_id = worker.worker_id if worker_id not in worker_roles: """ Something went wrong... """ continue role = worker_roles[worker_id] if role not in valid_workers: valid_workers[role] = worker if len(valid_workers) == 2: break return valid_workers.values() if len(valid_workers) == 2 else [] eligibility_function = {'func': check_workers_eligibility, 'multiple': True} def assign_worker_roles(workers): if opt['is_sandbox']: for i, worker in enumerate(workers): worker.id = mturk_agent_ids[i % len(mturk_agent_ids)] else: for worker in workers: worker.id = worker_roles[worker.worker_id] def run_conversation(mturk_manager, opt, workers): agents = workers[:] if not opt['is_sandbox']: for agent in agents: worker_roles.pop(agent.worker_id) conv_idx = mturk_manager.conversation_index world = MTurkWizardOfWikipediaWorld( opt, agents=agents, world_tag='conversation t_{}'.format(conv_idx), ir_agent=ir_agent, wiki_title_to_passage=wiki_title_to_passage, task=task, ) world.reset_random() while not world.episode_done(): world.parley() world.save_data() if world.convo_finished and not world.good_wiz and not opt['is_sandbox']: mturk_manager.soft_block_worker(world.wizard_worker) world.shutdown() world.review_work() mturk_manager.start_task( eligibility_function=eligibility_function, assign_role_function=assign_worker_roles, task_function=run_conversation, ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of one agent, model or MTurk worker, talking to an MTurk worker to negotiate a deal. """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('-min_t', '--min_turns', default=5, type=int, help='minimum number of turns') argparser.add_argument('-mt', '--max_turns', default=10, type=int, help='maximal number of chat turns') argparser.add_argument( '-mx_rsp_time', '--max_resp_time', default=150, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '-mx_psn_time', '--max_persona_time', type=int, default=300, help='time limit for turker' 'entering the persona', ) argparser.add_argument( '--ag_shutdown_time', default=120, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '--persona-type', default='both', type=str, choices=['both', 'self', 'other'], help='Which personas to load from personachat', ) argparser.add_argument('--revised', default=False, type='bool', help='Whether to use revised personas') argparser.add_argument('-rt', '--range_turn', default='5,7', help='sample range of number of turns') argparser.add_argument('--personas-path', default=None, help='specify path for personas data') opt = argparser.parse_args() directory_path = os.path.dirname(os.path.abspath(__file__)) opt['task'] = os.path.basename(directory_path) if not opt.get('personas_path'): opt['personas_path'] = (argparser.parlai_home + '/parlai/mturk/personachat_chat/data') opt.update(task_config) opt['extract_personas_path'] = os.path.join(opt['datapath'], 'personachat_chat') mturk_agent_ids = ['PERSON_1', 'PERSON_2'] mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids) persona_generator = PersonasGenerator(opt) mturk_manager.setup_server(task_directory_path=directory_path) try: mturk_manager.start_new_run() mturk_manager.create_hits() if not opt['is_sandbox']: blocked_worker_list = [] for w in blocked_worker_list: mturk_manager.block_worker( w, 'We found that you have unexpected behaviors in our previous ' 'HITs. For more questions please email us.', ) def run_onboard(worker): worker.persona_generator = persona_generator world = PersonaProfileWorld(opt, worker) world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): agents = [workers[0], workers[1]] conv_idx = mturk_manager.conversation_index world = PersonaChatWorld( opt=opt, agents=agents, range_turn=[int(s) for s in opt['range_turn'].split(',')], max_turn=opt['max_turns'], max_resp_time=opt['max_resp_time'], world_tag='conversation t_{}'.format(conv_idx), ) world.reset_random() while not world.episode_done(): world.parley() world.save_data() world.shutdown() world.review_work() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation, ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of an MTurk agent evaluating a ConvAI2 model. """ start_time = datetime.datetime.today().strftime('%Y-%m-%d-%H-%M') argparser = ParlaiParser(False, add_model_args=True) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('-mt', '--max-turns', default=10, type=int, help='maximal number of chat turns') argparser.add_argument('--max-resp-time', default=240, type=int, help='time limit for entering a dialog message') argparser.add_argument('--max-choice-time', type=int, default=300, help='time limit for turker' 'choosing the topic') argparser.add_argument('--ag-shutdown-time', default=120, type=int, help='time limit for entering a dialog message') argparser.add_argument('-rt', '--range-turn', default='3,5', help='sample range of number of turns') argparser.add_argument('--human-eval', type='bool', default=False, help='human vs human eval, no models involved') argparser.add_argument('--auto-approve-delay', type=int, default=3600 * 24 * 2, help='how long to wait for auto approval') argparser.add_argument('--only-masters', type='bool', default=False, help='Set to true to use only master turks for ' 'this test eval') argparser.add_argument('--unique-workers', type='bool', default=False, help='Each worker must be unique') argparser.add_argument('--create-model-qualif', type='bool', default=True, help='Create model qualif so unique eval between' 'models.') argparser.add_argument('--limit-workers', type=int, default=5, help='max HITs a worker can complete') argparser.add_argument( '--mturk-log', type=str, default='/home/kulikov/projects/mtbeam/human-eval/mturklogs/{}.log'. format(start_time)) argparser.add_argument('--short-eval', type='bool', default=True, help='Only ask engagingness question and persona' 'question.') # persona specific arguments argparser.add_argument('--persona-type', type=str, default='self', choices=['self', 'other', 'none']) argparser.add_argument('--persona-datatype', type=str, default='test', choices=['train', 'test', 'valid']) argparser.add_argument('--max-persona-time', type=int, default=360, help='max time to view persona') argparser.add_argument('--model-config', type=str, default=None, help='model config from standalone file') def inject_override(opt, override_dict): opt['override'] = override_dict for k, v in override_dict.items(): opt[k] = v def get_logger(opt): logger = logging.getLogger() logger.setLevel(logging.INFO) fmt = logging.Formatter('%(asctime)s: [ %(message)s ]', '%m/%d/%Y %I:%M:%S %p') console = logging.StreamHandler() console.setFormatter(fmt) logger.addHandler(console) if 'mturk_log' in opt: logfile = logging.FileHandler(opt['mturk_log'], 'a') logfile.setFormatter(fmt) logger.addHandler(logfile) logger.info('COMMAND: %s' % ' '.join(sys.argv)) logger.info('-' * 100) logger.info('CONFIG:\n%s' % json.dumps(opt, indent=4, sort_keys=True)) return logger PREV_MODEL_QUALS = [] start_opt = argparser.parse_args() # Get model config from config file config = getattr(mcf, start_opt['model_config']) #config = mcf.mtbeam1_noiter_noexpand_model_config inject_override(start_opt, config) if not start_opt.get('human_eval'): bot = create_agent(start_opt) shared_bot_params = bot.share() else: shared_bot_params = None if not start_opt['human_eval']: get_logger(bot.opt) else: get_logger(start_opt) if start_opt['human_eval']: folder_name = 'human_eval-{}'.format(start_time) else: folder_name = '{}-{}'.format(start_opt['model_config'], start_time) start_opt['task'] = os.path.basename( os.path.dirname(os.path.abspath(__file__))) start_opt['save_data_path'] = os.path.join(os.getcwd(), 'data', 'convAI2_eval', folder_name) start_opt.update(task_config) if not start_opt.get('human_eval'): mturk_agent_ids = ['PERSON_1'] else: mturk_agent_ids = ['PERSON_1', 'PERSON_2'] # QUALIFICATION STUFF if start_opt['limit_workers'] > 0: start_opt['unique_qual_name'] = config['chat_qual_max'] start_opt['max_hits_per_worker'] = start_opt['limit_workers'] mturk_manager = MTurkManager(opt=start_opt, mturk_agent_ids=mturk_agent_ids) personas_generator = PersonasGenerator(start_opt) directory_path = os.path.dirname(os.path.abspath(__file__)) mturk_manager.setup_server(task_directory_path=directory_path) try: mturk_manager.start_new_run() agent_qualifications = [] if not start_opt['is_sandbox']: # assign qualifications if start_opt['only_masters']: agent_qualifications.append(MASTER_QUALIF) if start_opt['unique_workers']: qual_name = 'ChatEval' qual_desc = ( 'Qualification to ensure each worker completes a maximum ' 'of one of these chat/eval HITs') qualification_id = \ mturk_utils.find_or_create_qualification(qual_name, qual_desc, False) print('Created qualification: ', qualification_id) UNIQUE_QUALIF = { 'QualificationTypeId': qualification_id, 'Comparator': 'DoesNotExist', 'RequiredToPreview': True } start_opt['unique_qualif_id'] = qualification_id agent_qualifications.append(UNIQUE_QUALIF) elif start_opt['create_model_qualif']: qual_name = config['chat_qual_name'] qual_desc = ( 'Qualification to ensure workers complete only a certain' 'number of these HITs') qualification_id = \ mturk_utils.find_or_create_qualification(qual_name, qual_desc, False) print('Created qualification: ', qualification_id) start_opt['unique_qualif_id'] = qualification_id for qual_name in PREV_MODEL_QUALS: qualification_id = \ mturk_utils.find_or_create_qualification( qual_name, qual_desc, False ) QUALIF = { 'QualificationTypeId': qualification_id, 'Comparator': 'DoesNotExist', 'RequiredToPreview': True } agent_qualifications.append(QUALIF) mturk_manager.create_hits(qualifications=agent_qualifications) #with open('/home/kulikov/code/mt-beam-parlai/mturk/mtdont.txt', 'r') as f: # MTDONT_LIST = [i.rstrip() for i in f.readlines()] #if not start_opt['is_sandbox']: # # ADD BLOCKED WORKERS HERE # blocked_worker_list = MTDONT_LIST # for w in blocked_worker_list: # try: # print('Soft Blocking {}\n'.format(w)) # mturk_manager.soft_block_worker(w) # except: # print('Did not soft block worker:', w) # time.sleep(0.1) def run_onboard(worker): worker.personas_generator = personas_generator world = PersonaAssignWorld(start_opt, worker) world.parley() world.shutdown() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def run_conversation(mturk_manager, opt, workers): conv_idx = mturk_manager.conversation_index world = ConvAI2Eval( opt=start_opt, agents=workers, range_turn=[ int(s) for s in start_opt['range_turn'].split(',') ], max_turn=start_opt['max_turns'], max_resp_time=start_opt['max_resp_time'], model_agent_opt=shared_bot_params, world_tag='conversation t_{}'.format(conv_idx), agent_timeout_shutdown=opt['ag_shutdown_time'], ) world.reset_random() while not world.episode_done(): world.parley() world.save_data() world.shutdown() gc.collect() mturk_manager.start_task(eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of an MTurk agent evaluating a chit-chat model. They are asked to chat to the model adopting a specific persona. After their conversation, they are asked to evaluate their partner on several metrics. """ argparser = ParlaiParser(False, add_model_args=True) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('-mt', '--max-turns', default=10, type=int, help='maximal number of chat turns') argparser.add_argument('--max-resp-time', default=180, type=int, help='time limit for entering a dialog message') argparser.add_argument('--max-persona-time', type=int, default=300, help='time limit for turker' 'entering the persona') argparser.add_argument('--ag-shutdown-time', default=120, type=int, help='time limit for entering a dialog message') argparser.add_argument('--persona-type', default='both', type=str, choices=['both', 'self', 'other'], help='Which personas to load from personachat') argparser.add_argument('--revised', default=False, type='bool', help='Whether to use revised personas') argparser.add_argument('-rt', '--range-turn', default='5,6', help='sample range of number of turns') argparser.add_argument('--auto-approve-delay', type=int, default=3600 * 24 * 1, help='how long to wait for \ auto approval') # ADD MODEL ARGS HERE (KVMEMNN ADDED AS AN EXAMPLE) argparser.set_defaults( model='projects.personachat.kvmemnn.kvmemnn:Kvmemnn', model_file='models:convai2/kvmemnn/model', ) opt = argparser.parse_args() # add additional model args opt['no_cuda'] = True opt['override'] = ['interactive_mode'] opt['interactive_mode'] = True bot = create_agent(opt) shared_bot_params = bot.share() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) if 'data_path' not in opt: opt['data_path'] = os.getcwd() + '/data/' + opt['task'] opt.update(task_config) mturk_agent_ids = ['PERSON_1'] mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids) persona_generator = PersonasGenerator(opt) mturk_manager.setup_server() try: mturk_manager.start_new_run() mturk_manager.create_hits() if not opt['is_sandbox']: # ADD BLOCKED WORKERS HERE blocked_worker_list = [] for w in blocked_worker_list: mturk_manager.block_worker( w, 'We found that you have unexpected behaviors in our \ previous HITs. For more questions please email us.') def run_onboard(worker): worker.persona_generator = persona_generator world = PersonaProfileWorld(opt, worker) world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): agents = workers[0] conv_idx = mturk_manager.conversation_index world = Convai2EvalWorld( opt=opt, agents=[agents], range_turn=[int(s) for s in opt['range_turn'].split(',')], max_turn=opt['max_turns'], max_resp_time=opt['max_resp_time'], model_agent_opt=shared_bot_params, world_tag='conversation t_{}'.format(conv_idx), agent_timeout_shutdown=opt['ag_shutdown_time'], ) world.reset_random() while not world.episode_done(): world.parley() world.save_data() world.shutdown() world.review_work() mturk_manager.start_task(eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()