def main(): """This task consists of an MTurk agent evaluating a wizard model. They are assigned a topic and asked to chat. """ start_time = datetime.datetime.today().strftime('%Y-%m-%d-%H-%M') argparser = ParlaiParser(False, add_model_args=True) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('-mt', '--max-turns', default=10, type=int, help='maximal number of chat turns') argparser.add_argument('--max-resp-time', default=240, type=int, help='time limit for entering a dialog message') argparser.add_argument('--max-choice-time', type=int, default=300, help='time limit for turker' 'choosing the topic') argparser.add_argument('--ag-shutdown-time', default=120, type=int, help='time limit for entering a dialog message') argparser.add_argument('-rt', '--range-turn', default='3,5', help='sample range of number of turns') argparser.add_argument('--human-eval', type='bool', default=False, help='human vs human eval, no models involved') argparser.add_argument('--auto-approve-delay', type=int, default=3600 * 24 * 1, help='how long to wait for auto approval') argparser.add_argument('--only-masters', type='bool', default=False, help='Set to true to use only master turks for ' 'this test eval') argparser.add_argument('--unique-workers', type='bool', default=False, help='Each worker must be unique') argparser.add_argument('--mturk-log', type=str, default='data/mturklogs/{}.log'.format(start_time)) def inject_override(opt, override_dict): opt['override'] = override_dict for k, v in override_dict.items(): opt[k] = v def get_logger(opt): logger = logging.getLogger() logger.setLevel(logging.INFO) fmt = logging.Formatter('%(asctime)s: [ %(message)s ]', '%m/%d/%Y %I:%M:%S %p') console = logging.StreamHandler() console.setFormatter(fmt) logger.addHandler(console) if 'mturk_log' in opt: logfile = logging.FileHandler(opt['mturk_log'], 'a') logfile.setFormatter(fmt) logger.addHandler(logfile) logger.info('COMMAND: %s' % ' '.join(sys.argv)) logger.info('-' * 100) logger.info('CONFIG:\n%s' % json.dumps(opt, indent=4, sort_keys=True)) return logger # MODEL CONFIG # NOTE: please edit this to test your own models config = { 'model': 'projects:wizard_of_wikipedia:interactive_retrieval', 'retriever_model_file': 'models:wikipedia_full/tfidf_retriever/model', 'responder_model_file': 'models:wizard_of_wikipedia/full_dialogue_retrieval_model/model', } argparser.add_model_subargs(config['model']) # add model args to opt start_opt = argparser.parse_args() inject_override(start_opt, config) if not start_opt.get('human_eval'): bot = create_agent(start_opt) shared_bot_params = bot.share() else: shared_bot_params = None if not start_opt['human_eval']: get_logger(bot.opt) else: get_logger(start_opt) if start_opt['human_eval']: folder_name = 'human_eval-{}'.format(start_time) else: folder_name = '{}-{}'.format(start_opt['model'], start_time) start_opt['task'] = os.path.basename( os.path.dirname(os.path.abspath(__file__))) if 'data_path' not in start_opt: start_opt['data_path'] = os.path.join(os.getcwd(), 'data', 'wizard_eval', folder_name) start_opt.update(task_config) if not start_opt.get('human_eval'): mturk_agent_ids = ['PERSON_1'] else: mturk_agent_ids = ['PERSON_1', 'PERSON_2'] mturk_manager = MTurkManager(opt=start_opt, mturk_agent_ids=mturk_agent_ids) topics_generator = TopicsGenerator(start_opt) directory_path = os.path.dirname(os.path.abspath(__file__)) mturk_manager.setup_server(task_directory_path=directory_path) worker_roles = {} connect_counter = AttrDict(value=0) try: mturk_manager.start_new_run() agent_qualifications = [] if not start_opt['is_sandbox']: # assign qualifications if start_opt['only_masters']: agent_qualifications.append(MASTER_QUALIF) if start_opt['unique_workers']: qual_name = 'UniqueChatEval' qual_desc = ( 'Qualification to ensure each worker completes a maximum ' 'of one of these chat/eval HITs') qualification_id = \ mturk_utils.find_or_create_qualification(qual_name, qual_desc, False) print('Created qualification: ', qualification_id) UNIQUE_QUALIF = { 'QualificationTypeId': qualification_id, 'Comparator': 'DoesNotExist', 'RequiredToPreview': True } start_opt['unique_qualif_id'] = qualification_id agent_qualifications.append(UNIQUE_QUALIF) mturk_manager.create_hits(qualifications=agent_qualifications) def run_onboard(worker): if start_opt['human_eval']: role = mturk_agent_ids[connect_counter.value % len(mturk_agent_ids)] connect_counter.value += 1 worker_roles[worker.worker_id] = role else: role = 'PERSON_1' worker.topics_generator = topics_generator world = TopicChooseWorld(start_opt, worker, role=role) world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_single_worker_eligibility(worker): return True def check_multiple_workers_eligibility(workers): valid_workers = {} for worker in workers: worker_id = worker.worker_id if worker_id not in worker_roles: print('Something went wrong') continue role = worker_roles[worker_id] if role not in valid_workers: valid_workers[role] = worker if len(valid_workers) == 2: break return valid_workers.values() if len(valid_workers) == 2 else [] if not start_opt['human_eval']: eligibility_function = { 'func': check_single_worker_eligibility, 'multiple': False, } else: eligibility_function = { 'func': check_multiple_workers_eligibility, 'multiple': True, } def assign_worker_roles(workers): if start_opt['human_eval']: for worker in workers: worker.id = worker_roles[worker.worker_id] else: for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def run_conversation(mturk_manager, opt, workers): conv_idx = mturk_manager.conversation_index world = WizardEval( opt=start_opt, agents=workers, range_turn=[ int(s) for s in start_opt['range_turn'].split(',') ], max_turn=start_opt['max_turns'], max_resp_time=start_opt['max_resp_time'], model_agent_opt=shared_bot_params, world_tag='conversation t_{}'.format(conv_idx), agent_timeout_shutdown=opt['ag_shutdown_time'], ) while not world.episode_done(): world.parley() world.save_data() world.shutdown() gc.collect() mturk_manager.start_task(eligibility_function=eligibility_function, assign_role_function=assign_worker_roles, task_function=run_conversation) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of an MTurk agent evaluating a Controllable Dialog model. """ start_time = datetime.datetime.today().strftime('%Y-%m-%d-%H-%M') argparser = ParlaiParser(False, add_model_args=True) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument( '--max-resp-time', default=240, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '--max-choice-time', type=int, default=300, help='time limit for turker' 'choosing the topic', ) argparser.add_argument( '--ag-shutdown-time', default=120, type=int, help='time limit for entering a dialog message', ) argparser.add_argument( '--num-turns', default=6, type=int, help='number of turns of dialogue' ) argparser.add_argument( '--human-eval', type='bool', default=False, help='human vs human eval, no models involved', ) argparser.add_argument( '--auto-approve-delay', type=int, default=3600 * 24 * 2, help='how long to wait for auto approval', ) argparser.add_argument( '--only-masters', type='bool', default=False, help='Set to true to use only master turks for ' 'this test eval', ) argparser.add_argument( '--create-model-qualif', type='bool', default=True, help='Create model qualif so unique eval between' 'models.', ) argparser.add_argument( '--limit-workers', type=int, default=len(SETTINGS_TO_RUN), help='max HITs a worker can complete', ) argparser.add_argument( '--mturk-log', type=str, default=('data/mturklogs/controllable/{}.log'.format(start_time)), ) argparser.add_argument( '--short-eval', type='bool', default=True, help='Only ask engagingness question and persona' 'question.', ) # persona specific arguments argparser.add_argument( '--persona-type', type=str, default='self', choices=['self', 'other', 'none'] ) argparser.add_argument( '--persona-datatype', type=str, default='valid', choices=['train', 'test', 'valid'], ) argparser.add_argument( '--max-persona-time', type=int, default=360, help='max time to view persona' ) def get_logger(opt): fmt = '%(asctime)s: [ %(message)s ]' logfn = None if 'mturk_log' in opt: logfn = opt['mturk_log'] if not os.path.isdir(os.path.dirname(logfn)): os.makedirs(os.path.dirname(logfn), exist_ok=True) logger = ParlaiLogger( name="mturk_controllable", console_level=INFO, file_level=INFO, console_format=fmt, file_format=fmt, filename=logfn, ) logger.info('COMMAND: %s' % ' '.join(sys.argv)) logger.info('-' * 100) logger.info('CONFIG:\n%s' % json.dumps(opt, indent=4, sort_keys=True)) return logger start_opt = argparser.parse_args() task_config['task_description'] = task_config['task_description'].format( start_opt['reward'] ) # set options start_opt['limit_workers'] = len(SETTINGS_TO_RUN) start_opt['allowed_conversations'] = 1 start_opt['max_hits_per_worker'] = start_opt['limit_workers'] start_opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) start_opt.update(task_config) logger = get_logger(start_opt) model_share_params = {} worker_models_seen = {} model_opts = {} model_counts = {} lock = Lock() for setup in SETTINGS_TO_RUN: assert 'human' not in setup model_counts[setup] = 0 agent_config = getattr(mcf, setup) combined_config = copy.deepcopy(start_opt) for k, v in agent_config.items(): combined_config[k] = v combined_config['override'][k] = v folder_name = '{}-{}'.format(setup, start_time) combined_config['save_data_path'] = os.path.join( start_opt['datapath'], 'local_controllable_dialogue', folder_name ) model_opts[setup] = combined_config bot = create_agent(combined_config, True) model_share_params[setup] = bot.share() if not start_opt.get('human_eval'): mturk_agent_ids = ['PERSON_1'] else: mturk_agent_ids = ['PERSON_1', 'PERSON_2'] mturk_manager = MTurkManager(opt=start_opt, mturk_agent_ids=mturk_agent_ids) personas_generator = PersonasGenerator(start_opt) directory_path = os.path.dirname(os.path.abspath(__file__)) mturk_manager.setup_server(task_directory_path=directory_path) try: mturk_manager.start_new_run() agent_qualifications = [] # assign qualifications if start_opt['create_model_qualif']: qual_name = 'ControlEvalRound2' qual_desc = ( 'Qualification to ensure workers complete only a certain' 'number of these HITs' ) qualification_id = mturk_utils.find_or_create_qualification( qual_name, qual_desc, False ) print('Created qualification: ', qualification_id) start_opt['unique_qualif_id'] = qualification_id def run_onboard(worker): worker.personas_generator = personas_generator world = PersonaAssignWorld(start_opt, worker) world.parley() world.shutdown() def check_worker_eligibility(worker): worker_id = worker.worker_id lock.acquire() retval = len(worker_models_seen.get(worker_id, [])) < len(SETTINGS_TO_RUN) lock.release() return retval def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() mturk_manager.create_hits(qualifications=agent_qualifications) def run_conversation(mturk_manager, opt, workers): conv_idx = mturk_manager.conversation_index # gotta find a bot this worker hasn't seen yet assert len(workers) == 1 worker_id = workers[0].worker_id lock.acquire() if worker_id not in worker_models_seen: worker_models_seen[worker_id] = set() print("MODELCOUNTS:") print(pprint.pformat(model_counts)) logger.info("MODELCOUNTS\n" + pprint.pformat(model_counts)) model_options = [ (model_counts[setup_name] + 10 * random.random(), setup_name) for setup_name in SETTINGS_TO_RUN if setup_name not in worker_models_seen[worker_id] ] if not model_options: lock.release() logger.error( "Worker {} already finished all settings! Returning none".format( worker_id ) ) return None _, model_choice = min(model_options) worker_models_seen[worker_id].add(model_choice) model_counts[model_choice] += 1 lock.release() world = ControllableDialogEval( opt=model_opts[model_choice], agents=workers, num_turns=start_opt['num_turns'], max_resp_time=start_opt['max_resp_time'], model_agent_opt=model_share_params[model_choice], world_tag='conversation t_{}'.format(conv_idx), agent_timeout_shutdown=opt['ag_shutdown_time'], model_config=model_choice, ) world.reset_random() while not world.episode_done(): world.parley() world.save_data() lock.acquire() if not world.convo_finished: model_counts[model_choice] -= 1 worker_models_seen[worker_id].remove(model_choice) lock.release() world.shutdown() gc.collect() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation, ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): completed_workers = [] argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) mturk_agent_id = 'Worker' mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=[mturk_agent_id]) mturk_manager.setup_server() qual_name = 'ParlAIExcludeQual{}t{}'.format(random.randint(10000, 99999), random.randint(10000, 99999)) qual_desc = ( 'Qualification for a worker not correctly completing the ' 'first iteration of a task. Used to filter to different task pools.') qualification_id = \ mturk_utils.find_or_create_qualification(qual_name, qual_desc, opt['is_sandbox']) print('Created qualification: ', qualification_id) def run_onboard(worker): world = QualificationFlowOnboardWorld(opt, worker) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) try: mturk_manager.start_new_run() agent_qualifications = [{ 'QualificationTypeId': qualification_id, 'Comparator': 'DoesNotExist', 'RequiredToPreview': True }] mturk_manager.create_hits(qualifications=agent_qualifications) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(worker): worker[0].id = mturk_agent_id global run_conversation def run_conversation(mturk_manager, opt, workers): mturk_agent = workers[0] world = QualificationFlowSoloWorld( opt=opt, mturk_agent=mturk_agent, qualification_id=qualification_id, firstTime=(mturk_agent.worker_id not in completed_workers), ) while not world.episode_done(): world.parley() completed_workers.append(mturk_agent.worker_id) world.shutdown() world.review_work() mturk_manager.start_task(eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation) except BaseException: raise finally: mturk_utils.delete_qualification(qualification_id, opt['is_sandbox']) mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """ Main function for the DMG pilot data collection task :return: Nothing. """ global available_games argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('--two_mturk_agents', dest='two_mturk_agents', action='store_true', help='data collection mode ' 'with converations between two MTurk agents') opt = argparser.parse_args() opt['task'] = 'dmg_pilot_dev' opt['datatype'] = 'dmg_pilot_data_1' opt.update(task_config) is_sandbox = True if opt['is_sandbox'] else False if is_sandbox: print("\n- - - Running in Sandbox Mode - - -") local_agent_1_id = 'local_1' mturk_agent_ids = ['mturk_agent_1'] if opt['two_mturk_agents']: mturk_agent_ids.append('mturk_agent_2') mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids) mturk_manager.setup_server() # Remove qualification blocks # mturk_utils.delete_qualification('33DUYNETVPNVNFT4X22Z6F584M1WCM', is_sandbox) qual_name = 'DMG Pilot: Max Games Reached v1' qual_desc = 'Qualification for a worker who completed the maximum number of games in the DMG Pilot' qualification_id = mturk_utils.find_or_create_qualification( qual_name, qual_desc, is_sandbox, True) print('Created qualification: {}'.format(qualification_id)) available_games = len(DMGMultiRoundTeacher(opt=opt).episodes) print("Available games: {}".format(available_games)) try: def load_records(): global worker_record global worker_bans try: with open('records/worker_record.dill', 'rb') as infile: worker_record = dill.load(infile) print("Loaded worker records.") except FileNotFoundError: pass try: with open('records/worker_bans.json', 'r') as infile: worker_bans = json.load(infile) except FileNotFoundError: pass # Exclude players who are blocked by completing the maximum number of games in a previous HIT for worker_id in worker_bans: print("Excluded banned worker {}".format(worker_id)) mturk_utils.give_worker_qualification(worker_id, qualification_id) def save_records(): global worker_record global worker_bans if VERBOSE: print("Writing worker records to file") if not os.path.exists("records"): os.makedirs("records") with open('records/worker_record.json', 'w') as f: json.dump(worker_record, f) with open('records/worker_record.dill', 'wb') as f: dill.dump(worker_record, f) with open('records/worker_bans.json', 'w') as f: json.dump(worker_bans, f) def select_random_game(exceptions=None): """ Returns a random game ID with exception of those specified in exceptions :param exceptions: list of game IDs that cannot be selected :return: a random game ID with exception of those specified in exceptions """ global available_games while True: game_id = randint(0, available_games) # if VERBOSE: print("Game ID is {}. Exceptions are {}".format(game_id, exceptions)) if exceptions is not None: if game_id not in exceptions: break else: break # if VERBOSE: print("Selected random game {}".format(game_id)) return game_id def game_is_blocked(id, player_id): """ Returns True if the given game ID is blocked for the given player, False otherwise :param id: ID of a given game :param player_id: ID of a given player :return: True if the given game ID is blocked for the given player, False otherwise """ if player_id in worker_record: if worker_record[player_id]["games"].count(id) == 2: return True return False def update_records(players, played_game_id): """ Updates the HITs worker records :param players: players paired for a game :param played_game_id: game ID :return: Nothing. """ print("UPDATING RECORDS!!!") # Add game ID to the worker record and to the blocked list if it is the second occurance. # If a player has played 10 games, he or she gets banned from the game. update_worker_record(players[0], players[1], played_game_id) update_worker_record(players[1], players[0], played_game_id) def update_worker_record(worker, partner, played_game_id): """ Updates the record for a specific worker - partner pairing with a given game ID :param worker: worker agent :param partner: partner agent :param played_game_id: assigned game ID :return: Nothing. """ global worker_record global worker_bans player_id = worker.worker_id partner_id = partner.worker_id worker_record[player_id]["games"].append(played_game_id) worker_record[player_id]["partners"].append(partner_id) if len(worker_record[player_id]["games"]) == 5: mturk_utils.give_worker_qualification(player_id, qualification_id) worker_bans.append(player_id) worker.block_worker( "Reached the maxiumum of 5 games in the DMG Pilot") def check_workers_eligibility(workers): """ Checks the list of available workers, pairs the first two players that pass the game criteria and sets the corresponding game id :param workers: a list of all available workers :return: a list of two workers that are paired for the next game """ global worker_record global game_id players = [] # Return an empty list if not enough workers are in queue if len(workers) < 2: return players if VERBOSE: print("{} workers available:".format(len(workers))) for idx, worker in enumerate(workers): worker_id = worker.worker_id if VERBOSE: print("Worker: {}".format(worker_id)) # Worker never played before. Pair him or her with the next queued worker who also never played before if worker_id not in worker_record: if VERBOSE: print("Worker has no recorded games") for partner in workers[idx + 1:]: partner_id = partner.worker_id if partner_id not in worker_record: if VERBOSE: print("Partner: {}".format(partner_id)) players.append(worker) players.append(partner) next_game_id = select_random_game() if VERBOSE: print( "Partner has no recorded games. Setting game ID randomly to {}" .format(next_game_id)) game_id = next_game_id return players # Nobody in the queue is new. Continue with the loop print("Nobody in the queue is new. Continue with the loop") continue # Worker played before. else: last_game_id = worker_record[worker_id]["games"][-1] if not game_is_blocked(last_game_id, worker_id): # Check if anybody in the queue has not played this game yet and didn't play with worker before for partner in workers[idx + 1:]: partner_id = partner.worker_id # If partner also played before, pair them if partner_id in worker_record and last_game_id not in worker_record[ partner_id][ "games"] and worker_id not in worker_record[ partner_id]["partners"]: if VERBOSE: print( "Partner has not played this game before." ) players.append(worker) players.append(partner) next_game_id = last_game_id game_id = next_game_id return players # No suitable partner was found to play worker's last game. # So pair worker with the next available player and check their games for partner in workers[idx + 1:]: partner_id = partner.worker_id # If partner also played before, but never with worker, pair them if partner_id in worker_record and worker_id not in worker_record[ partner_id]["partners"]: last_game_id = worker_record[partner_id]["games"][ -1] players.append(worker) players.append(partner) # Check if the partner's last game is not yet blocked and never played by the worker if not game_is_blocked( last_game_id, partner_id ) and last_game_id not in worker_record[worker_id][ "games"]: next_game_id = last_game_id if VERBOSE: print( "Partner has recorded games. Setting game ID to {}" .format(next_game_id)) game_id = next_game_id return players # Else select a random one that none of the two played before else: blocked = copy( worker_record[worker_id]["games"]) blocked.extend( worker_record[partner_id]["games"]) next_game_id = select_random_game( exceptions=blocked) if VERBOSE: print( "Selected game {} as it was not played by any of the players before" .format(next_game_id)) game_id = next_game_id return players # Nobody in the queue played before. Continue with the loop print( "Nobody in the queue played before. Continue with the loop" ) continue # No match could be made since the only workers available are from different categories print( "No match could be made since the only workers available are from different categories" ) return players def get_worker_names(players): """ Returns gender-neutral nicknames for the players based on how many games they played already :param players: List of player IDs :return: a list of gender-neutral nicknames for the players based on how many games they played already """ global worker_names player_names = [] for player in players: player_id = player.worker_id if player_id in worker_record: n = len(worker_record[player_id]) - 1 player_names.append(worker_names[n]) else: return ["Kelsey", "Robin"] return player_names def assign_worker_roles(workers): """ Assigns indexes to the assigned workers :param workers: list of workers :return: Nothing. """ for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] def pay_workers(agents, get_pay, time_bonus=None): if not os.path.exists("records"): os.makedirs("records") for agent in agents: if get_pay[agent.worker_id]: print("Paying worker {}".format(agent.worker_id)) if len(worker_record[agent.worker_id]["games"]) > 1: agent.pay_bonus( 0.25, reason="DMP Pilot: Bonus for multiple games") print("Paying bonus for multiple games!") with open('records/payments.txt', 'a') as f: f.write("{}; {}; {}; multiple_bonus\n".format( agent.worker_id, 0.25, agent.assignment_id)) if time_bonus: agent.pay_bonus(time_bonus, reason="DMG Pilot: Bonus for long HIT") print("Paying bonus for long HIT!") with open('records/payments.txt', 'a') as f: f.write("{}; {}; {}; long_bonus\n".format( agent.worker_id, time_bonus, agent.assignment_id)) agent.approve_work() with open('records/payments.txt', 'a') as f: f.write("{}; {}; {}; payment\n".format( agent.worker_id, 1.75, agent.assignment_id)) else: print( "Rejecting agent {}'s work as he or she disconnected (too early) or score is too low." .format(agent.worker_id)) agent.reject_work( reason= 'Disconnected before end of HIT or scored too low') def run_conversation(mturk_manager, opt, workers): """ Runs the conversation :param mturk_manager: MTurk manager :param opt: command line arguments :param workers: list of workers :return: Nothing. """ global game_id global worker_record conversation_start_time = time.time() # Copy workers into agents list agents = workers[:] # Get worker names names = get_worker_names(agents) print(names) # Create a local agent if not opt['two_mturk_agents']: if 'model' in opt: local_agent = create_agent(opt) else: local_agent = LocalHumanAgent(opt=None) local_agent.id = local_agent_1_id agents.append(local_agent) opt["batchindex"] = mturk_manager.started_conversations print("Loading game {}".format(game_id)) print(list(worker_record.keys())) print(agents[0].worker_id) print(agents[1].worker_id) # If the workers never played before, start with the warm-up round if (agents[0].worker_id not in worker_record) and (agents[1].worker_id not in worker_record): world = MTurkDMGDialogWarmupWorld( opt=opt, agents=agents, ) print("--- Starting Warming-Up Round ---") while not world.episode_done(): if world.parley(): break world = MTurkDMGDialogWorld(opt=opt, agents=agents, game_id=game_id, names=names) get_pay = {agents[0].worker_id: False, agents[1].worker_id: False} print("--- Starting Game ---") while not world.episode_done(): print("Parley!") world.parley() print("# # # DONE # # #") if world.disconnected: print("Game ended due to disconnect.") if world.round_nr > 1: for agent in agents: if not agent.disconnected: print("CHECK: Agent {} did NOT disconnect".format( agent.worker_id)) get_pay[agent.worker_id] = True else: print("CHECK: Agent {} DID disconnect".format( agent.worker_id)) else: # Only save records when game was complete print("Updating records") update_records(agents, game_id) save_records() if world.total_score > 24: print("Total score was above 24, paying both workers.") get_pay = { agents[0].worker_id: True, agents[1].worker_id: True } else: print("Score too low!") if world.end_time: conversation_end_time = world.end_time else: conversation_end_time = conversation_start_time world.shutdown() print("# # # Game ended # # #") duration = conversation_end_time - conversation_start_time duration_mins = duration / 60.0 time_bonus = None if duration_mins > 1: if duration_mins >= 25: time_bonus = 1.50 else: time_bonus = int(duration_mins - 10) * 0.10 time_bonus = round(time_bonus, 2) if time_bonus and time_bonus > 1.5: time_bonus = 1.5 if time_bonus and time_bonus < 0: time_bonus = None pay_workers(agents, get_pay, time_bonus) print("Conversation closed.") load_records() print("# # # Loaded records # # #") def run_onboard(worker): global worker_record if worker.worker_id not in worker_record: world = MTurkDMGDialogOnboardWorld(opt=opt, mturk_agent=worker) while not world.episode_done(): world.parley() world.shutdown() print("Onboarding done.") mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.start_new_run() agent_qualifications = [{ 'QualificationTypeId': qualification_id, 'Comparator': 'DoesNotExist', 'RequiredToPreview': True }] mturk_manager.create_hits(qualifications=agent_qualifications) # Increasing restart time mturk_manager.ready_to_accept_workers(timeout_seconds=120) eligibility_function = { 'func': check_workers_eligibility, 'multiple': True, } mturk_manager.start_task(eligibility_function=eligibility_function, assign_role_function=assign_worker_roles, task_function=run_conversation) print("HIT ended.") except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): completed_workers = [] argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__))) opt.update(task_config) mturk_agent_id = 'Worker' mturk_manager = MTurkManager( opt=opt, mturk_agent_ids=[mturk_agent_id] ) mturk_manager.setup_server() qual_name = 'ParlAIExcludeQual{}t{}'.format( random.randint(10000, 99999), random.randint(10000, 99999)) qual_desc = ( 'Qualification for a worker not correctly completing the ' 'first iteration of a task. Used to filter to different task pools.' ) qualification_id = \ mturk_utils.find_or_create_qualification(qual_name, qual_desc) print('Created qualification: ', qualification_id) def run_onboard(worker): world = QualificationFlowOnboardWorld(opt, worker) while not world.episode_done(): world.parley() world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) try: mturk_manager.start_new_run() agent_qualifications = [{ 'QualificationTypeId': qualification_id, 'Comparator': 'DoesNotExist', 'RequiredToPreview': True }] mturk_manager.create_hits(qualifications=agent_qualifications) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(worker): worker[0].id = mturk_agent_id global run_conversation def run_conversation(mturk_manager, opt, workers): mturk_agent = workers[0] world = QualificationFlowSoloWorld( opt=opt, mturk_agent=mturk_agent, qualification_id=qualification_id, firstTime=(mturk_agent.worker_id not in completed_workers), ) while not world.episode_done(): world.parley() completed_workers.append(mturk_agent.worker_id) world.shutdown() world.review_work() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation ) except BaseException: raise finally: mturk_utils.delete_qualification(qualification_id) mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
def main(): """This task consists of an MTurk agent evaluating a ConvAI2 model. """ start_time = datetime.datetime.today().strftime('%Y-%m-%d-%H-%M') argparser = ParlaiParser(False, add_model_args=True) argparser.add_parlai_data_path() argparser.add_mturk_args() argparser.add_argument('-mt', '--max-turns', default=10, type=int, help='maximal number of chat turns') argparser.add_argument('--max-resp-time', default=240, type=int, help='time limit for entering a dialog message') argparser.add_argument('--max-choice-time', type=int, default=300, help='time limit for turker' 'choosing the topic') argparser.add_argument('--ag-shutdown-time', default=120, type=int, help='time limit for entering a dialog message') argparser.add_argument('-rt', '--range-turn', default='3,5', help='sample range of number of turns') argparser.add_argument('--human-eval', type='bool', default=False, help='human vs human eval, no models involved') argparser.add_argument('--auto-approve-delay', type=int, default=3600 * 24 * 2, help='how long to wait for auto approval') argparser.add_argument('--only-masters', type='bool', default=False, help='Set to true to use only master turks for ' 'this test eval') argparser.add_argument('--unique-workers', type='bool', default=False, help='Each worker must be unique') argparser.add_argument('--create-model-qualif', type='bool', default=True, help='Create model qualif so unique eval between' 'models.') argparser.add_argument('--limit-workers', type=int, default=5, help='max HITs a worker can complete') argparser.add_argument( '--mturk-log', type=str, default='/home/kulikov/projects/mtbeam/human-eval/mturklogs/{}.log'. format(start_time)) argparser.add_argument('--short-eval', type='bool', default=True, help='Only ask engagingness question and persona' 'question.') # persona specific arguments argparser.add_argument('--persona-type', type=str, default='self', choices=['self', 'other', 'none']) argparser.add_argument('--persona-datatype', type=str, default='test', choices=['train', 'test', 'valid']) argparser.add_argument('--max-persona-time', type=int, default=360, help='max time to view persona') argparser.add_argument('--model-config', type=str, default=None, help='model config from standalone file') def inject_override(opt, override_dict): opt['override'] = override_dict for k, v in override_dict.items(): opt[k] = v def get_logger(opt): logger = logging.getLogger() logger.setLevel(logging.INFO) fmt = logging.Formatter('%(asctime)s: [ %(message)s ]', '%m/%d/%Y %I:%M:%S %p') console = logging.StreamHandler() console.setFormatter(fmt) logger.addHandler(console) if 'mturk_log' in opt: logfile = logging.FileHandler(opt['mturk_log'], 'a') logfile.setFormatter(fmt) logger.addHandler(logfile) logger.info('COMMAND: %s' % ' '.join(sys.argv)) logger.info('-' * 100) logger.info('CONFIG:\n%s' % json.dumps(opt, indent=4, sort_keys=True)) return logger PREV_MODEL_QUALS = [] start_opt = argparser.parse_args() # Get model config from config file config = getattr(mcf, start_opt['model_config']) #config = mcf.mtbeam1_noiter_noexpand_model_config inject_override(start_opt, config) if not start_opt.get('human_eval'): bot = create_agent(start_opt) shared_bot_params = bot.share() else: shared_bot_params = None if not start_opt['human_eval']: get_logger(bot.opt) else: get_logger(start_opt) if start_opt['human_eval']: folder_name = 'human_eval-{}'.format(start_time) else: folder_name = '{}-{}'.format(start_opt['model_config'], start_time) start_opt['task'] = os.path.basename( os.path.dirname(os.path.abspath(__file__))) start_opt['save_data_path'] = os.path.join(os.getcwd(), 'data', 'convAI2_eval', folder_name) start_opt.update(task_config) if not start_opt.get('human_eval'): mturk_agent_ids = ['PERSON_1'] else: mturk_agent_ids = ['PERSON_1', 'PERSON_2'] # QUALIFICATION STUFF if start_opt['limit_workers'] > 0: start_opt['unique_qual_name'] = config['chat_qual_max'] start_opt['max_hits_per_worker'] = start_opt['limit_workers'] mturk_manager = MTurkManager(opt=start_opt, mturk_agent_ids=mturk_agent_ids) personas_generator = PersonasGenerator(start_opt) directory_path = os.path.dirname(os.path.abspath(__file__)) mturk_manager.setup_server(task_directory_path=directory_path) try: mturk_manager.start_new_run() agent_qualifications = [] if not start_opt['is_sandbox']: # assign qualifications if start_opt['only_masters']: agent_qualifications.append(MASTER_QUALIF) if start_opt['unique_workers']: qual_name = 'ChatEval' qual_desc = ( 'Qualification to ensure each worker completes a maximum ' 'of one of these chat/eval HITs') qualification_id = \ mturk_utils.find_or_create_qualification(qual_name, qual_desc, False) print('Created qualification: ', qualification_id) UNIQUE_QUALIF = { 'QualificationTypeId': qualification_id, 'Comparator': 'DoesNotExist', 'RequiredToPreview': True } start_opt['unique_qualif_id'] = qualification_id agent_qualifications.append(UNIQUE_QUALIF) elif start_opt['create_model_qualif']: qual_name = config['chat_qual_name'] qual_desc = ( 'Qualification to ensure workers complete only a certain' 'number of these HITs') qualification_id = \ mturk_utils.find_or_create_qualification(qual_name, qual_desc, False) print('Created qualification: ', qualification_id) start_opt['unique_qualif_id'] = qualification_id for qual_name in PREV_MODEL_QUALS: qualification_id = \ mturk_utils.find_or_create_qualification( qual_name, qual_desc, False ) QUALIF = { 'QualificationTypeId': qualification_id, 'Comparator': 'DoesNotExist', 'RequiredToPreview': True } agent_qualifications.append(QUALIF) mturk_manager.create_hits(qualifications=agent_qualifications) #with open('/home/kulikov/code/mt-beam-parlai/mturk/mtdont.txt', 'r') as f: # MTDONT_LIST = [i.rstrip() for i in f.readlines()] #if not start_opt['is_sandbox']: # # ADD BLOCKED WORKERS HERE # blocked_worker_list = MTDONT_LIST # for w in blocked_worker_list: # try: # print('Soft Blocking {}\n'.format(w)) # mturk_manager.soft_block_worker(w) # except: # print('Did not soft block worker:', w) # time.sleep(0.1) def run_onboard(worker): worker.personas_generator = personas_generator world = PersonaAssignWorld(start_opt, worker) world.parley() world.shutdown() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): for index, worker in enumerate(workers): worker.id = mturk_agent_ids[index % len(mturk_agent_ids)] mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def run_conversation(mturk_manager, opt, workers): conv_idx = mturk_manager.conversation_index world = ConvAI2Eval( opt=start_opt, agents=workers, range_turn=[ int(s) for s in start_opt['range_turn'].split(',') ], max_turn=start_opt['max_turns'], max_resp_time=start_opt['max_resp_time'], model_agent_opt=shared_bot_params, world_tag='conversation t_{}'.format(conv_idx), agent_timeout_shutdown=opt['ag_shutdown_time'], ) world.reset_random() while not world.episode_done(): world.parley() world.save_data() world.shutdown() gc.collect() mturk_manager.start_task(eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()