def main():
"""This task consists of an MTurk agent evaluating a wizard model. They
are assigned a topic and asked to chat.
"""
start_time = datetime.datetime.today().strftime('%Y-%m-%d-%H-%M')
argparser = ParlaiParser(False, add_model_args=True)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('-mt',
'--max-turns',
default=10,
type=int,
help='maximal number of chat turns')
argparser.add_argument('--max-resp-time',
default=240,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('--max-choice-time',
type=int,
default=300,
help='time limit for turker'
'choosing the topic')
argparser.add_argument('--ag-shutdown-time',
default=120,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('-rt',
'--range-turn',
default='3,5',
help='sample range of number of turns')
argparser.add_argument('--human-eval',
type='bool',
default=False,
help='human vs human eval, no models involved')
argparser.add_argument('--auto-approve-delay',
type=int,
default=3600 * 24 * 1,
help='how long to wait for auto approval')
argparser.add_argument('--only-masters',
type='bool',
default=False,
help='Set to true to use only master turks for '
'this test eval')
argparser.add_argument('--unique-workers',
type='bool',
default=False,
help='Each worker must be unique')
argparser.add_argument('--mturk-log',
type=str,
default='data/mturklogs/{}.log'.format(start_time))
def inject_override(opt, override_dict):
opt['override'] = override_dict
for k, v in override_dict.items():
opt[k] = v
def get_logger(opt):
logger = logging.getLogger()
logger.setLevel(logging.INFO)
fmt = logging.Formatter('%(asctime)s: [ %(message)s ]',
'%m/%d/%Y %I:%M:%S %p')
console = logging.StreamHandler()
console.setFormatter(fmt)
logger.addHandler(console)
if 'mturk_log' in opt:
logfile = logging.FileHandler(opt['mturk_log'], 'a')
logfile.setFormatter(fmt)
logger.addHandler(logfile)
logger.info('COMMAND: %s' % ' '.join(sys.argv))
logger.info('-' * 100)
logger.info('CONFIG:\n%s' % json.dumps(opt, indent=4, sort_keys=True))
return logger
# MODEL CONFIG
# NOTE: please edit this to test your own models
config = {
'model':
'projects:wizard_of_wikipedia:interactive_retrieval',
'retriever_model_file':
'models:wikipedia_full/tfidf_retriever/model',
'responder_model_file':
'models:wizard_of_wikipedia/full_dialogue_retrieval_model/model',
}
argparser.add_model_subargs(config['model']) # add model args to opt
start_opt = argparser.parse_args()
inject_override(start_opt, config)
if not start_opt.get('human_eval'):
bot = create_agent(start_opt)
shared_bot_params = bot.share()
else:
shared_bot_params = None
if not start_opt['human_eval']:
get_logger(bot.opt)
else:
get_logger(start_opt)
if start_opt['human_eval']:
folder_name = 'human_eval-{}'.format(start_time)
else:
folder_name = '{}-{}'.format(start_opt['model'], start_time)
start_opt['task'] = os.path.basename(
os.path.dirname(os.path.abspath(__file__)))
if 'data_path' not in start_opt:
start_opt['data_path'] = os.path.join(os.getcwd(), 'data',
'wizard_eval', folder_name)
start_opt.update(task_config)
if not start_opt.get('human_eval'):
mturk_agent_ids = ['PERSON_1']
else:
mturk_agent_ids = ['PERSON_1', 'PERSON_2']
mturk_manager = MTurkManager(opt=start_opt,
mturk_agent_ids=mturk_agent_ids)
topics_generator = TopicsGenerator(start_opt)
directory_path = os.path.dirname(os.path.abspath(__file__))
mturk_manager.setup_server(task_directory_path=directory_path)
worker_roles = {}
connect_counter = AttrDict(value=0)
try:
mturk_manager.start_new_run()
agent_qualifications = []
if not start_opt['is_sandbox']:
# assign qualifications
if start_opt['only_masters']:
agent_qualifications.append(MASTER_QUALIF)
if start_opt['unique_workers']:
qual_name = 'UniqueChatEval'
qual_desc = (
'Qualification to ensure each worker completes a maximum '
'of one of these chat/eval HITs')
qualification_id = \
mturk_utils.find_or_create_qualification(qual_name, qual_desc,
False)
print('Created qualification: ', qualification_id)
UNIQUE_QUALIF = {
'QualificationTypeId': qualification_id,
'Comparator': 'DoesNotExist',
'RequiredToPreview': True
}
start_opt['unique_qualif_id'] = qualification_id
agent_qualifications.append(UNIQUE_QUALIF)
mturk_manager.create_hits(qualifications=agent_qualifications)
def run_onboard(worker):
if start_opt['human_eval']:
role = mturk_agent_ids[connect_counter.value %
len(mturk_agent_ids)]
connect_counter.value += 1
worker_roles[worker.worker_id] = role
else:
role = 'PERSON_1'
worker.topics_generator = topics_generator
world = TopicChooseWorld(start_opt, worker, role=role)
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_single_worker_eligibility(worker):
return True
def check_multiple_workers_eligibility(workers):
valid_workers = {}
for worker in workers:
worker_id = worker.worker_id
if worker_id not in worker_roles:
print('Something went wrong')
continue
role = worker_roles[worker_id]
if role not in valid_workers:
valid_workers[role] = worker
if len(valid_workers) == 2:
break
return valid_workers.values() if len(valid_workers) == 2 else []
if not start_opt['human_eval']:
eligibility_function = {
'func': check_single_worker_eligibility,
'multiple': False,
}
else:
eligibility_function = {
'func': check_multiple_workers_eligibility,
'multiple': True,
}
def assign_worker_roles(workers):
if start_opt['human_eval']:
for worker in workers:
worker.id = worker_roles[worker.worker_id]
else:
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
conv_idx = mturk_manager.conversation_index
world = WizardEval(
opt=start_opt,
agents=workers,
range_turn=[
int(s) for s in start_opt['range_turn'].split(',')
],
max_turn=start_opt['max_turns'],
max_resp_time=start_opt['max_resp_time'],
model_agent_opt=shared_bot_params,
world_tag='conversation t_{}'.format(conv_idx),
agent_timeout_shutdown=opt['ag_shutdown_time'],
)
while not world.episode_done():
world.parley()
world.save_data()
world.shutdown()
gc.collect()
mturk_manager.start_task(eligibility_function=eligibility_function,
assign_role_function=assign_worker_roles,
task_function=run_conversation)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""This task consists of an MTurk agent evaluating a Controllable Dialog model.
"""
start_time = datetime.datetime.today().strftime('%Y-%m-%d-%H-%M')
argparser = ParlaiParser(False, add_model_args=True)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument(
'--max-resp-time',
default=240,
type=int,
help='time limit for entering a dialog message',
)
argparser.add_argument(
'--max-choice-time',
type=int,
default=300,
help='time limit for turker' 'choosing the topic',
)
argparser.add_argument(
'--ag-shutdown-time',
default=120,
type=int,
help='time limit for entering a dialog message',
)
argparser.add_argument(
'--num-turns', default=6, type=int, help='number of turns of dialogue'
)
argparser.add_argument(
'--human-eval',
type='bool',
default=False,
help='human vs human eval, no models involved',
)
argparser.add_argument(
'--auto-approve-delay',
type=int,
default=3600 * 24 * 2,
help='how long to wait for auto approval',
)
argparser.add_argument(
'--only-masters',
type='bool',
default=False,
help='Set to true to use only master turks for ' 'this test eval',
)
argparser.add_argument(
'--create-model-qualif',
type='bool',
default=True,
help='Create model qualif so unique eval between' 'models.',
)
argparser.add_argument(
'--limit-workers',
type=int,
default=len(SETTINGS_TO_RUN),
help='max HITs a worker can complete',
)
argparser.add_argument(
'--mturk-log',
type=str,
default=('data/mturklogs/controllable/{}.log'.format(start_time)),
)
argparser.add_argument(
'--short-eval',
type='bool',
default=True,
help='Only ask engagingness question and persona' 'question.',
)
# persona specific arguments
argparser.add_argument(
'--persona-type', type=str, default='self', choices=['self', 'other', 'none']
)
argparser.add_argument(
'--persona-datatype',
type=str,
default='valid',
choices=['train', 'test', 'valid'],
)
argparser.add_argument(
'--max-persona-time', type=int, default=360, help='max time to view persona'
)
def get_logger(opt):
fmt = '%(asctime)s: [ %(message)s ]'
logfn = None
if 'mturk_log' in opt:
logfn = opt['mturk_log']
if not os.path.isdir(os.path.dirname(logfn)):
os.makedirs(os.path.dirname(logfn), exist_ok=True)
logger = ParlaiLogger(
name="mturk_controllable",
console_level=INFO,
file_level=INFO,
console_format=fmt,
file_format=fmt,
filename=logfn,
)
logger.info('COMMAND: %s' % ' '.join(sys.argv))
logger.info('-' * 100)
logger.info('CONFIG:\n%s' % json.dumps(opt, indent=4, sort_keys=True))
return logger
start_opt = argparser.parse_args()
task_config['task_description'] = task_config['task_description'].format(
start_opt['reward']
)
# set options
start_opt['limit_workers'] = len(SETTINGS_TO_RUN)
start_opt['allowed_conversations'] = 1
start_opt['max_hits_per_worker'] = start_opt['limit_workers']
start_opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
start_opt.update(task_config)
logger = get_logger(start_opt)
model_share_params = {}
worker_models_seen = {}
model_opts = {}
model_counts = {}
lock = Lock()
for setup in SETTINGS_TO_RUN:
assert 'human' not in setup
model_counts[setup] = 0
agent_config = getattr(mcf, setup)
combined_config = copy.deepcopy(start_opt)
for k, v in agent_config.items():
combined_config[k] = v
combined_config['override'][k] = v
folder_name = '{}-{}'.format(setup, start_time)
combined_config['save_data_path'] = os.path.join(
start_opt['datapath'], 'local_controllable_dialogue', folder_name
)
model_opts[setup] = combined_config
bot = create_agent(combined_config, True)
model_share_params[setup] = bot.share()
if not start_opt.get('human_eval'):
mturk_agent_ids = ['PERSON_1']
else:
mturk_agent_ids = ['PERSON_1', 'PERSON_2']
mturk_manager = MTurkManager(opt=start_opt, mturk_agent_ids=mturk_agent_ids)
personas_generator = PersonasGenerator(start_opt)
directory_path = os.path.dirname(os.path.abspath(__file__))
mturk_manager.setup_server(task_directory_path=directory_path)
try:
mturk_manager.start_new_run()
agent_qualifications = []
# assign qualifications
if start_opt['create_model_qualif']:
qual_name = 'ControlEvalRound2'
qual_desc = (
'Qualification to ensure workers complete only a certain'
'number of these HITs'
)
qualification_id = mturk_utils.find_or_create_qualification(
qual_name, qual_desc, False
)
print('Created qualification: ', qualification_id)
start_opt['unique_qualif_id'] = qualification_id
def run_onboard(worker):
worker.personas_generator = personas_generator
world = PersonaAssignWorld(start_opt, worker)
world.parley()
world.shutdown()
def check_worker_eligibility(worker):
worker_id = worker.worker_id
lock.acquire()
retval = len(worker_models_seen.get(worker_id, [])) < len(SETTINGS_TO_RUN)
lock.release()
return retval
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
mturk_manager.create_hits(qualifications=agent_qualifications)
def run_conversation(mturk_manager, opt, workers):
conv_idx = mturk_manager.conversation_index
# gotta find a bot this worker hasn't seen yet
assert len(workers) == 1
worker_id = workers[0].worker_id
lock.acquire()
if worker_id not in worker_models_seen:
worker_models_seen[worker_id] = set()
print("MODELCOUNTS:")
print(pprint.pformat(model_counts))
logger.info("MODELCOUNTS\n" + pprint.pformat(model_counts))
model_options = [
(model_counts[setup_name] + 10 * random.random(), setup_name)
for setup_name in SETTINGS_TO_RUN
if setup_name not in worker_models_seen[worker_id]
]
if not model_options:
lock.release()
logger.error(
"Worker {} already finished all settings! Returning none".format(
worker_id
)
)
return None
_, model_choice = min(model_options)
worker_models_seen[worker_id].add(model_choice)
model_counts[model_choice] += 1
lock.release()
world = ControllableDialogEval(
opt=model_opts[model_choice],
agents=workers,
num_turns=start_opt['num_turns'],
max_resp_time=start_opt['max_resp_time'],
model_agent_opt=model_share_params[model_choice],
world_tag='conversation t_{}'.format(conv_idx),
agent_timeout_shutdown=opt['ag_shutdown_time'],
model_config=model_choice,
)
world.reset_random()
while not world.episode_done():
world.parley()
world.save_data()
lock.acquire()
if not world.convo_finished:
model_counts[model_choice] -= 1
worker_models_seen[worker_id].remove(model_choice)
lock.release()
world.shutdown()
gc.collect()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation,
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
completed_workers = []
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=[mturk_agent_id])
mturk_manager.setup_server()
qual_name = 'ParlAIExcludeQual{}t{}'.format(random.randint(10000, 99999),
random.randint(10000, 99999))
qual_desc = (
'Qualification for a worker not correctly completing the '
'first iteration of a task. Used to filter to different task pools.')
qualification_id = \
mturk_utils.find_or_create_qualification(qual_name, qual_desc,
opt['is_sandbox'])
print('Created qualification: ', qualification_id)
def run_onboard(worker):
world = QualificationFlowOnboardWorld(opt, worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
try:
mturk_manager.start_new_run()
agent_qualifications = [{
'QualificationTypeId': qualification_id,
'Comparator': 'DoesNotExist',
'RequiredToPreview': True
}]
mturk_manager.create_hits(qualifications=agent_qualifications)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
world = QualificationFlowSoloWorld(
opt=opt,
mturk_agent=mturk_agent,
qualification_id=qualification_id,
firstTime=(mturk_agent.worker_id not in completed_workers),
)
while not world.episode_done():
world.parley()
completed_workers.append(mturk_agent.worker_id)
world.shutdown()
world.review_work()
mturk_manager.start_task(eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation)
except BaseException:
raise
finally:
mturk_utils.delete_qualification(qualification_id, opt['is_sandbox'])
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""
Main function for the DMG pilot data collection task
:return: Nothing.
"""
global available_games
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('--two_mturk_agents',
dest='two_mturk_agents',
action='store_true',
help='data collection mode '
'with converations between two MTurk agents')
opt = argparser.parse_args()
opt['task'] = 'dmg_pilot_dev'
opt['datatype'] = 'dmg_pilot_data_1'
opt.update(task_config)
is_sandbox = True if opt['is_sandbox'] else False
if is_sandbox: print("\n- - - Running in Sandbox Mode - - -")
local_agent_1_id = 'local_1'
mturk_agent_ids = ['mturk_agent_1']
if opt['two_mturk_agents']:
mturk_agent_ids.append('mturk_agent_2')
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
mturk_manager.setup_server()
# Remove qualification blocks
# mturk_utils.delete_qualification('33DUYNETVPNVNFT4X22Z6F584M1WCM', is_sandbox)
qual_name = 'DMG Pilot: Max Games Reached v1'
qual_desc = 'Qualification for a worker who completed the maximum number of games in the DMG Pilot'
qualification_id = mturk_utils.find_or_create_qualification(
qual_name, qual_desc, is_sandbox, True)
print('Created qualification: {}'.format(qualification_id))
available_games = len(DMGMultiRoundTeacher(opt=opt).episodes)
print("Available games: {}".format(available_games))
try:
def load_records():
global worker_record
global worker_bans
try:
with open('records/worker_record.dill', 'rb') as infile:
worker_record = dill.load(infile)
print("Loaded worker records.")
except FileNotFoundError:
pass
try:
with open('records/worker_bans.json', 'r') as infile:
worker_bans = json.load(infile)
except FileNotFoundError:
pass
# Exclude players who are blocked by completing the maximum number of games in a previous HIT
for worker_id in worker_bans:
print("Excluded banned worker {}".format(worker_id))
mturk_utils.give_worker_qualification(worker_id,
qualification_id)
def save_records():
global worker_record
global worker_bans
if VERBOSE: print("Writing worker records to file")
if not os.path.exists("records"):
os.makedirs("records")
with open('records/worker_record.json', 'w') as f:
json.dump(worker_record, f)
with open('records/worker_record.dill', 'wb') as f:
dill.dump(worker_record, f)
with open('records/worker_bans.json', 'w') as f:
json.dump(worker_bans, f)
def select_random_game(exceptions=None):
"""
Returns a random game ID with exception of those specified in exceptions
:param exceptions: list of game IDs that cannot be selected
:return: a random game ID with exception of those specified in exceptions
"""
global available_games
while True:
game_id = randint(0, available_games)
# if VERBOSE: print("Game ID is {}. Exceptions are {}".format(game_id, exceptions))
if exceptions is not None:
if game_id not in exceptions:
break
else:
break
# if VERBOSE: print("Selected random game {}".format(game_id))
return game_id
def game_is_blocked(id, player_id):
"""
Returns True if the given game ID is blocked for the given player, False otherwise
:param id: ID of a given game
:param player_id: ID of a given player
:return: True if the given game ID is blocked for the given player, False otherwise
"""
if player_id in worker_record:
if worker_record[player_id]["games"].count(id) == 2:
return True
return False
def update_records(players, played_game_id):
"""
Updates the HITs worker records
:param players: players paired for a game
:param played_game_id: game ID
:return: Nothing.
"""
print("UPDATING RECORDS!!!")
# Add game ID to the worker record and to the blocked list if it is the second occurance.
# If a player has played 10 games, he or she gets banned from the game.
update_worker_record(players[0], players[1], played_game_id)
update_worker_record(players[1], players[0], played_game_id)
def update_worker_record(worker, partner, played_game_id):
"""
Updates the record for a specific worker - partner pairing with a given game ID
:param worker: worker agent
:param partner: partner agent
:param played_game_id: assigned game ID
:return: Nothing.
"""
global worker_record
global worker_bans
player_id = worker.worker_id
partner_id = partner.worker_id
worker_record[player_id]["games"].append(played_game_id)
worker_record[player_id]["partners"].append(partner_id)
if len(worker_record[player_id]["games"]) == 5:
mturk_utils.give_worker_qualification(player_id,
qualification_id)
worker_bans.append(player_id)
worker.block_worker(
"Reached the maxiumum of 5 games in the DMG Pilot")
def check_workers_eligibility(workers):
"""
Checks the list of available workers, pairs the first two players that pass
the game criteria and sets the corresponding game id
:param workers: a list of all available workers
:return: a list of two workers that are paired for the next game
"""
global worker_record
global game_id
players = []
# Return an empty list if not enough workers are in queue
if len(workers) < 2:
return players
if VERBOSE: print("{} workers available:".format(len(workers)))
for idx, worker in enumerate(workers):
worker_id = worker.worker_id
if VERBOSE: print("Worker: {}".format(worker_id))
# Worker never played before. Pair him or her with the next queued worker who also never played before
if worker_id not in worker_record:
if VERBOSE: print("Worker has no recorded games")
for partner in workers[idx + 1:]:
partner_id = partner.worker_id
if partner_id not in worker_record:
if VERBOSE: print("Partner: {}".format(partner_id))
players.append(worker)
players.append(partner)
next_game_id = select_random_game()
if VERBOSE:
print(
"Partner has no recorded games. Setting game ID randomly to {}"
.format(next_game_id))
game_id = next_game_id
return players
# Nobody in the queue is new. Continue with the loop
print("Nobody in the queue is new. Continue with the loop")
continue
# Worker played before.
else:
last_game_id = worker_record[worker_id]["games"][-1]
if not game_is_blocked(last_game_id, worker_id):
# Check if anybody in the queue has not played this game yet and didn't play with worker before
for partner in workers[idx + 1:]:
partner_id = partner.worker_id
# If partner also played before, pair them
if partner_id in worker_record and last_game_id not in worker_record[
partner_id][
"games"] and worker_id not in worker_record[
partner_id]["partners"]:
if VERBOSE:
print(
"Partner has not played this game before."
)
players.append(worker)
players.append(partner)
next_game_id = last_game_id
game_id = next_game_id
return players
# No suitable partner was found to play worker's last game.
# So pair worker with the next available player and check their games
for partner in workers[idx + 1:]:
partner_id = partner.worker_id
# If partner also played before, but never with worker, pair them
if partner_id in worker_record and worker_id not in worker_record[
partner_id]["partners"]:
last_game_id = worker_record[partner_id]["games"][
-1]
players.append(worker)
players.append(partner)
# Check if the partner's last game is not yet blocked and never played by the worker
if not game_is_blocked(
last_game_id, partner_id
) and last_game_id not in worker_record[worker_id][
"games"]:
next_game_id = last_game_id
if VERBOSE:
print(
"Partner has recorded games. Setting game ID to {}"
.format(next_game_id))
game_id = next_game_id
return players
# Else select a random one that none of the two played before
else:
blocked = copy(
worker_record[worker_id]["games"])
blocked.extend(
worker_record[partner_id]["games"])
next_game_id = select_random_game(
exceptions=blocked)
if VERBOSE:
print(
"Selected game {} as it was not played by any of the players before"
.format(next_game_id))
game_id = next_game_id
return players
# Nobody in the queue played before. Continue with the loop
print(
"Nobody in the queue played before. Continue with the loop"
)
continue
# No match could be made since the only workers available are from different categories
print(
"No match could be made since the only workers available are from different categories"
)
return players
def get_worker_names(players):
"""
Returns gender-neutral nicknames for the players based on how many games they played already
:param players: List of player IDs
:return: a list of gender-neutral nicknames for the players based on how many games they played already
"""
global worker_names
player_names = []
for player in players:
player_id = player.worker_id
if player_id in worker_record:
n = len(worker_record[player_id]) - 1
player_names.append(worker_names[n])
else:
return ["Kelsey", "Robin"]
return player_names
def assign_worker_roles(workers):
"""
Assigns indexes to the assigned workers
:param workers: list of workers
:return: Nothing.
"""
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def pay_workers(agents, get_pay, time_bonus=None):
if not os.path.exists("records"):
os.makedirs("records")
for agent in agents:
if get_pay[agent.worker_id]:
print("Paying worker {}".format(agent.worker_id))
if len(worker_record[agent.worker_id]["games"]) > 1:
agent.pay_bonus(
0.25, reason="DMP Pilot: Bonus for multiple games")
print("Paying bonus for multiple games!")
with open('records/payments.txt', 'a') as f:
f.write("{}; {}; {}; multiple_bonus\n".format(
agent.worker_id, 0.25, agent.assignment_id))
if time_bonus:
agent.pay_bonus(time_bonus,
reason="DMG Pilot: Bonus for long HIT")
print("Paying bonus for long HIT!")
with open('records/payments.txt', 'a') as f:
f.write("{}; {}; {}; long_bonus\n".format(
agent.worker_id, time_bonus,
agent.assignment_id))
agent.approve_work()
with open('records/payments.txt', 'a') as f:
f.write("{}; {}; {}; payment\n".format(
agent.worker_id, 1.75, agent.assignment_id))
else:
print(
"Rejecting agent {}'s work as he or she disconnected (too early) or score is too low."
.format(agent.worker_id))
agent.reject_work(
reason=
'Disconnected before end of HIT or scored too low')
def run_conversation(mturk_manager, opt, workers):
"""
Runs the conversation
:param mturk_manager: MTurk manager
:param opt: command line arguments
:param workers: list of workers
:return: Nothing.
"""
global game_id
global worker_record
conversation_start_time = time.time()
# Copy workers into agents list
agents = workers[:]
# Get worker names
names = get_worker_names(agents)
print(names)
# Create a local agent
if not opt['two_mturk_agents']:
if 'model' in opt:
local_agent = create_agent(opt)
else:
local_agent = LocalHumanAgent(opt=None)
local_agent.id = local_agent_1_id
agents.append(local_agent)
opt["batchindex"] = mturk_manager.started_conversations
print("Loading game {}".format(game_id))
print(list(worker_record.keys()))
print(agents[0].worker_id)
print(agents[1].worker_id)
# If the workers never played before, start with the warm-up round
if (agents[0].worker_id
not in worker_record) and (agents[1].worker_id
not in worker_record):
world = MTurkDMGDialogWarmupWorld(
opt=opt,
agents=agents,
)
print("--- Starting Warming-Up Round ---")
while not world.episode_done():
if world.parley():
break
world = MTurkDMGDialogWorld(opt=opt,
agents=agents,
game_id=game_id,
names=names)
get_pay = {agents[0].worker_id: False, agents[1].worker_id: False}
print("--- Starting Game ---")
while not world.episode_done():
print("Parley!")
world.parley()
print("# # # DONE # # #")
if world.disconnected:
print("Game ended due to disconnect.")
if world.round_nr > 1:
for agent in agents:
if not agent.disconnected:
print("CHECK: Agent {} did NOT disconnect".format(
agent.worker_id))
get_pay[agent.worker_id] = True
else:
print("CHECK: Agent {} DID disconnect".format(
agent.worker_id))
else:
# Only save records when game was complete
print("Updating records")
update_records(agents, game_id)
save_records()
if world.total_score > 24:
print("Total score was above 24, paying both workers.")
get_pay = {
agents[0].worker_id: True,
agents[1].worker_id: True
}
else:
print("Score too low!")
if world.end_time:
conversation_end_time = world.end_time
else:
conversation_end_time = conversation_start_time
world.shutdown()
print("# # # Game ended # # #")
duration = conversation_end_time - conversation_start_time
duration_mins = duration / 60.0
time_bonus = None
if duration_mins > 1:
if duration_mins >= 25:
time_bonus = 1.50
else:
time_bonus = int(duration_mins - 10) * 0.10
time_bonus = round(time_bonus, 2)
if time_bonus and time_bonus > 1.5:
time_bonus = 1.5
if time_bonus and time_bonus < 0:
time_bonus = None
pay_workers(agents, get_pay, time_bonus)
print("Conversation closed.")
load_records()
print("# # # Loaded records # # #")
def run_onboard(worker):
global worker_record
if worker.worker_id not in worker_record:
world = MTurkDMGDialogOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
print("Onboarding done.")
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.start_new_run()
agent_qualifications = [{
'QualificationTypeId': qualification_id,
'Comparator': 'DoesNotExist',
'RequiredToPreview': True
}]
mturk_manager.create_hits(qualifications=agent_qualifications)
# Increasing restart time
mturk_manager.ready_to_accept_workers(timeout_seconds=120)
eligibility_function = {
'func': check_workers_eligibility,
'multiple': True,
}
mturk_manager.start_task(eligibility_function=eligibility_function,
assign_role_function=assign_worker_roles,
task_function=run_conversation)
print("HIT ended.")
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
completed_workers = []
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_id]
)
mturk_manager.setup_server()
qual_name = 'ParlAIExcludeQual{}t{}'.format(
random.randint(10000, 99999), random.randint(10000, 99999))
qual_desc = (
'Qualification for a worker not correctly completing the '
'first iteration of a task. Used to filter to different task pools.'
)
qualification_id = \
mturk_utils.find_or_create_qualification(qual_name, qual_desc)
print('Created qualification: ', qualification_id)
def run_onboard(worker):
world = QualificationFlowOnboardWorld(opt, worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
try:
mturk_manager.start_new_run()
agent_qualifications = [{
'QualificationTypeId': qualification_id,
'Comparator': 'DoesNotExist',
'RequiredToPreview': True
}]
mturk_manager.create_hits(qualifications=agent_qualifications)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
world = QualificationFlowSoloWorld(
opt=opt,
mturk_agent=mturk_agent,
qualification_id=qualification_id,
firstTime=(mturk_agent.worker_id not in completed_workers),
)
while not world.episode_done():
world.parley()
completed_workers.append(mturk_agent.worker_id)
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_utils.delete_qualification(qualification_id)
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""This task consists of an MTurk agent evaluating a ConvAI2 model.
"""
start_time = datetime.datetime.today().strftime('%Y-%m-%d-%H-%M')
argparser = ParlaiParser(False, add_model_args=True)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('-mt',
'--max-turns',
default=10,
type=int,
help='maximal number of chat turns')
argparser.add_argument('--max-resp-time',
default=240,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('--max-choice-time',
type=int,
default=300,
help='time limit for turker'
'choosing the topic')
argparser.add_argument('--ag-shutdown-time',
default=120,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('-rt',
'--range-turn',
default='3,5',
help='sample range of number of turns')
argparser.add_argument('--human-eval',
type='bool',
default=False,
help='human vs human eval, no models involved')
argparser.add_argument('--auto-approve-delay',
type=int,
default=3600 * 24 * 2,
help='how long to wait for auto approval')
argparser.add_argument('--only-masters',
type='bool',
default=False,
help='Set to true to use only master turks for '
'this test eval')
argparser.add_argument('--unique-workers',
type='bool',
default=False,
help='Each worker must be unique')
argparser.add_argument('--create-model-qualif',
type='bool',
default=True,
help='Create model qualif so unique eval between'
'models.')
argparser.add_argument('--limit-workers',
type=int,
default=5,
help='max HITs a worker can complete')
argparser.add_argument(
'--mturk-log',
type=str,
default='/home/kulikov/projects/mtbeam/human-eval/mturklogs/{}.log'.
format(start_time))
argparser.add_argument('--short-eval',
type='bool',
default=True,
help='Only ask engagingness question and persona'
'question.')
# persona specific arguments
argparser.add_argument('--persona-type',
type=str,
default='self',
choices=['self', 'other', 'none'])
argparser.add_argument('--persona-datatype',
type=str,
default='test',
choices=['train', 'test', 'valid'])
argparser.add_argument('--max-persona-time',
type=int,
default=360,
help='max time to view persona')
argparser.add_argument('--model-config',
type=str,
default=None,
help='model config from standalone file')
def inject_override(opt, override_dict):
opt['override'] = override_dict
for k, v in override_dict.items():
opt[k] = v
def get_logger(opt):
logger = logging.getLogger()
logger.setLevel(logging.INFO)
fmt = logging.Formatter('%(asctime)s: [ %(message)s ]',
'%m/%d/%Y %I:%M:%S %p')
console = logging.StreamHandler()
console.setFormatter(fmt)
logger.addHandler(console)
if 'mturk_log' in opt:
logfile = logging.FileHandler(opt['mturk_log'], 'a')
logfile.setFormatter(fmt)
logger.addHandler(logfile)
logger.info('COMMAND: %s' % ' '.join(sys.argv))
logger.info('-' * 100)
logger.info('CONFIG:\n%s' % json.dumps(opt, indent=4, sort_keys=True))
return logger
PREV_MODEL_QUALS = []
start_opt = argparser.parse_args()
# Get model config from config file
config = getattr(mcf, start_opt['model_config'])
#config = mcf.mtbeam1_noiter_noexpand_model_config
inject_override(start_opt, config)
if not start_opt.get('human_eval'):
bot = create_agent(start_opt)
shared_bot_params = bot.share()
else:
shared_bot_params = None
if not start_opt['human_eval']:
get_logger(bot.opt)
else:
get_logger(start_opt)
if start_opt['human_eval']:
folder_name = 'human_eval-{}'.format(start_time)
else:
folder_name = '{}-{}'.format(start_opt['model_config'], start_time)
start_opt['task'] = os.path.basename(
os.path.dirname(os.path.abspath(__file__)))
start_opt['save_data_path'] = os.path.join(os.getcwd(), 'data',
'convAI2_eval', folder_name)
start_opt.update(task_config)
if not start_opt.get('human_eval'):
mturk_agent_ids = ['PERSON_1']
else:
mturk_agent_ids = ['PERSON_1', 'PERSON_2']
# QUALIFICATION STUFF
if start_opt['limit_workers'] > 0:
start_opt['unique_qual_name'] = config['chat_qual_max']
start_opt['max_hits_per_worker'] = start_opt['limit_workers']
mturk_manager = MTurkManager(opt=start_opt,
mturk_agent_ids=mturk_agent_ids)
personas_generator = PersonasGenerator(start_opt)
directory_path = os.path.dirname(os.path.abspath(__file__))
mturk_manager.setup_server(task_directory_path=directory_path)
try:
mturk_manager.start_new_run()
agent_qualifications = []
if not start_opt['is_sandbox']:
# assign qualifications
if start_opt['only_masters']:
agent_qualifications.append(MASTER_QUALIF)
if start_opt['unique_workers']:
qual_name = 'ChatEval'
qual_desc = (
'Qualification to ensure each worker completes a maximum '
'of one of these chat/eval HITs')
qualification_id = \
mturk_utils.find_or_create_qualification(qual_name, qual_desc,
False)
print('Created qualification: ', qualification_id)
UNIQUE_QUALIF = {
'QualificationTypeId': qualification_id,
'Comparator': 'DoesNotExist',
'RequiredToPreview': True
}
start_opt['unique_qualif_id'] = qualification_id
agent_qualifications.append(UNIQUE_QUALIF)
elif start_opt['create_model_qualif']:
qual_name = config['chat_qual_name']
qual_desc = (
'Qualification to ensure workers complete only a certain'
'number of these HITs')
qualification_id = \
mturk_utils.find_or_create_qualification(qual_name, qual_desc,
False)
print('Created qualification: ', qualification_id)
start_opt['unique_qualif_id'] = qualification_id
for qual_name in PREV_MODEL_QUALS:
qualification_id = \
mturk_utils.find_or_create_qualification(
qual_name,
qual_desc,
False
)
QUALIF = {
'QualificationTypeId': qualification_id,
'Comparator': 'DoesNotExist',
'RequiredToPreview': True
}
agent_qualifications.append(QUALIF)
mturk_manager.create_hits(qualifications=agent_qualifications)
#with open('/home/kulikov/code/mt-beam-parlai/mturk/mtdont.txt', 'r') as f:
# MTDONT_LIST = [i.rstrip() for i in f.readlines()]
#if not start_opt['is_sandbox']:
# # ADD BLOCKED WORKERS HERE
# blocked_worker_list = MTDONT_LIST
# for w in blocked_worker_list:
# try:
# print('Soft Blocking {}\n'.format(w))
# mturk_manager.soft_block_worker(w)
# except:
# print('Did not soft block worker:', w)
# time.sleep(0.1)
def run_onboard(worker):
worker.personas_generator = personas_generator
world = PersonaAssignWorld(start_opt, worker)
world.parley()
world.shutdown()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def run_conversation(mturk_manager, opt, workers):
conv_idx = mturk_manager.conversation_index
world = ConvAI2Eval(
opt=start_opt,
agents=workers,
range_turn=[
int(s) for s in start_opt['range_turn'].split(',')
],
max_turn=start_opt['max_turns'],
max_resp_time=start_opt['max_resp_time'],
model_agent_opt=shared_bot_params,
world_tag='conversation t_{}'.format(conv_idx),
agent_timeout_shutdown=opt['ag_shutdown_time'],
)
world.reset_random()
while not world.episode_done():
world.parley()
world.save_data()
world.shutdown()
gc.collect()
mturk_manager.start_task(eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()