def main():
"""
Wizard of Wikipedia Data Collection Task.
The task involves two people holding a conversation. One dialog partner
chooses a topic to discuss, and then dialog proceeds.
One partner is the Wizard, who has access to retrieved external
information conditioned on the last two utterances, as well as
information regarding the chosen topic.
The other partner is the Apprentice, who assumes the role of someone
eager to learn about the chosen topic.
"""
argparser = ParlaiParser(False, False)
DictionaryAgent.add_cmdline_args(argparser)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument(
'-min_t', '--min_turns', default=3, type=int, help='minimum number of turns'
)
argparser.add_argument(
'-max_t',
'--max_turns',
default=5,
type=int,
help='maximal number of chat turns',
)
argparser.add_argument(
'-mx_rsp_time',
'--max_resp_time',
default=120,
type=int,
help='time limit for entering a dialog message',
)
argparser.add_argument(
'-mx_onb_time',
'--max_onboard_time',
type=int,
default=300,
help='time limit for turker' 'in onboarding',
)
argparser.add_argument(
'--persona-type',
default='both',
type=str,
choices=['both', 'self', 'other'],
help='Which personas to load from personachat',
)
argparser.add_argument(
'--auto-approve-delay',
type=int,
default=3600 * 24 * 1,
help='how long to wait for \
auto approval',
)
argparser.add_argument(
'--word-overlap-threshold',
type=int,
default=2,
help='How much word overlap we want between message \
and checked sentence',
)
argparser.add_argument(
'--num-good-sentence-threshold',
type=int,
default=2,
help='How many good sentences with sufficient overlap \
are necessary for turker to be considered good.',
)
argparser.add_argument(
'--num-passages-retrieved',
type=int,
default=7,
help='How many passages to retrieve per dialog \
message',
)
opt = argparser.parse_args()
directory_path = os.path.dirname(os.path.abspath(__file__))
opt['task'] = os.path.basename(directory_path)
if 'data_path' not in opt:
opt['data_path'] = os.getcwd() + '/data/' + opt['task']
opt['current_working_dir'] = os.getcwd()
opt.update(task_config)
mturk_agent_ids = [APPRENTICE, WIZARD]
opt['min_messages'] = 2
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
setup_personas_with_wiki_links(opt)
ir_agent, task = setup_retriever(opt)
persona_generator = PersonasGenerator(opt)
wiki_title_to_passage = setup_title_to_passage(opt)
mturk_manager.setup_server(task_directory_path=directory_path)
worker_roles = {}
connect_counter = AttrDict(value=0)
try:
mturk_manager.start_new_run()
if not opt['is_sandbox']:
with open(os.path.join(opt['current_working_dir'], 'mtdont.txt')) as f:
lines = [l.replace('\n', '') for l in f.readlines()]
for w in lines:
mturk_manager.soft_block_worker(w)
def run_onboard(worker):
role = mturk_agent_ids[connect_counter.value % len(mturk_agent_ids)]
connect_counter.value += 1
worker_roles[worker.worker_id] = role
worker.persona_generator = persona_generator
world = RoleOnboardWorld(opt, worker, role)
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
mturk_manager.create_hits()
def check_workers_eligibility(workers):
if opt['is_sandbox']:
return workers
valid_workers = {}
for worker in workers:
worker_id = worker.worker_id
if worker_id not in worker_roles:
"""
Something went wrong...
"""
continue
role = worker_roles[worker_id]
if role not in valid_workers:
valid_workers[role] = worker
if len(valid_workers) == 2:
break
return valid_workers.values() if len(valid_workers) == 2 else []
eligibility_function = {'func': check_workers_eligibility, 'multiple': True}
def assign_worker_roles(workers):
if opt['is_sandbox']:
for i, worker in enumerate(workers):
worker.id = mturk_agent_ids[i % len(mturk_agent_ids)]
else:
for worker in workers:
worker.id = worker_roles[worker.worker_id]
def run_conversation(mturk_manager, opt, workers):
agents = workers[:]
if not opt['is_sandbox']:
for agent in agents:
worker_roles.pop(agent.worker_id)
conv_idx = mturk_manager.conversation_index
world = MTurkWizardOfWikipediaWorld(
opt,
agents=agents,
world_tag='conversation t_{}'.format(conv_idx),
ir_agent=ir_agent,
wiki_title_to_passage=wiki_title_to_passage,
task=task,
)
world.reset_random()
while not world.episode_done():
world.parley()
world.save_data()
if world.convo_finished and not world.good_wiz and not opt['is_sandbox']:
mturk_manager.soft_block_worker(world.wizard_worker)
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=eligibility_function,
assign_role_function=assign_worker_roles,
task_function=run_conversation,
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
"""This task consists of an MTurk agent evaluating a chit-chat model. They
are asked to chat to the model adopting a specific persona. After their
conversation, they are asked to evaluate their partner on several metrics.
"""
argparser = ParlaiParser(False, add_model_args=True)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('-mt', '--max-turns', default=10, type=int,
help='maximal number of chat turns')
argparser.add_argument('--max-resp-time', default=240,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('--max-persona-time', type=int,
default=300, help='time limit for turker'
'entering the persona')
argparser.add_argument('--ag-shutdown-time', default=120,
type=int,
help='time limit for entering a dialog message')
argparser.add_argument('--persona-type', default='both', type=str,
choices=['both', 'self', 'other'],
help='Which personas to load from personachat')
argparser.add_argument('--revised', default=False, type='bool',
help='Whether to use revised personas')
argparser.add_argument('-rt', '--range-turn', default='5,6',
help='sample range of number of turns')
argparser.add_argument('--auto-approve-delay', type=int,
default=3600 * 24 * 1,
help='how long to wait for auto approval')
argparser.add_argument('--only-masters', type='bool', default=False,
help='Set to True to use only master turks for this' +
' test eval, default is %(default)s')
# ADD MODEL ARGS HERE, UNCOMMENT TO USE KVMEMNN MODEL AS AN EXAMPLE
# argparser.set_defaults(
# model='projects.personachat.kvmemnn.kvmemnn:Kvmemnn',
# model_file='models:convai2/kvmemnn/model',
# )
opt = argparser.parse_args()
# add additional model args
opt['override'] = {
'no_cuda': True,
'interactive_mode': True,
'tensorboard_log': False
}
bot = create_agent(opt)
shared_bot_params = bot.share()
print(
'=== Actual bot opt === :\n {}'.format(
'\n'.join(["[{}] : {}".format(k, v) for k, v in bot.opt.items()])
)
)
folder_name = (
'master_{}_YOURCOMMENT__'.format(opt['only_masters']) +
'__'.join(['{}_{}'.format(k, v) for k, v in opt['override'].items()])
)
# this is mturk task, not convai2 task from ParlAI
opt['task'] = 'convai2:self'
if 'data_path' not in opt:
opt['data_path'] = os.getcwd() + '/data/' + folder_name
opt.update(task_config)
mturk_agent_ids = ['PERSON_1']
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=mturk_agent_ids
)
persona_generator = PersonasGenerator(opt)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
agent_qualifications = []
if opt['only_masters']:
if opt['is_sandbox']:
agent_qualifications.append(MASTER_QUALIF_SDBOX)
else:
agent_qualifications.append(MASTER_QUALIF)
mturk_manager.create_hits(qualifications=agent_qualifications)
if not opt['is_sandbox']:
# ADD SOFT-BLOCKED WORKERS HERE
# NOTE: blocking qual *must be* specified
blocked_worker_list = []
for w in blocked_worker_list:
print('Soft Blocking {}\n'.format(w))
mturk_manager.soft_block_worker(w)
time.sleep(0.1) # do the sleep to prevent amazon query drop
def run_onboard(worker):
worker.persona_generator = persona_generator
world = PersonaProfileWorld(opt, worker)
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for index, worker in enumerate(workers):
worker.id = mturk_agent_ids[index % len(mturk_agent_ids)]
def run_conversation(mturk_manager, opt, workers):
agents = workers[0]
conv_idx = mturk_manager.conversation_index
world = Convai2EvalWorld(
opt=opt,
agents=[agents],
range_turn=[int(s) for s in opt['range_turn'].split(',')],
max_turn=opt['max_turns'],
max_resp_time=opt['max_resp_time'],
model_agent_opt=shared_bot_params,
world_tag='conversation t_{}'.format(conv_idx),
agent_timeout_shutdown=opt['ag_shutdown_time'],
)
world.reset_random()
while not world.episode_done():
world.parley()
world.save_data()
world.shutdown()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def run_task(override_opt: Optional[dict] = None):
"""
This task consists of an MTurk worker talking to a model and MTurker also evaluates
each utterance of the bot for various buckets (see constants).
"""
config_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'task_config')
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
default_task_folder = os.path.join(argparser.parlai_home, 'data',
'turn_annotations')
argparser.add_mturk_args()
argparser.add_argument('-num_t',
'--num_turns',
default=6,
type=int,
help='minimum number of turns')
argparser.add_argument(
'--conversations-needed',
dest='conversations_needed_string',
default=None,
type=str,
help=
'Number of convos needed for each model. For example: "modelA:50,modelB:20"',
)
argparser.add_argument(
'--task-model-parallel',
default=True,
type=bool,
help='Whether to load models to be used with model_parallel True.',
)
argparser.add_argument(
'--auto-approve-delay',
dest='auto_approve_delay',
type=int,
default=3600 * 24 * 5,
help='how long to wait for auto approval',
)
argparser.add_argument(
'--max-resp-time',
type=int,
default=180,
help='time limit for entering a dialog message',
)
argparser.add_argument(
'--max-onboard-time',
type=int,
default=300,
help='time limit accepting onboarding',
)
argparser.add_argument(
'--base-save-folder',
default=default_task_folder,
type=str,
help='base folder for saving all crowdsourcing results',
)
argparser.add_argument(
'--base-model-folder',
default=None,
type=str,
help='base folder for loading model files from',
)
argparser.add_argument(
'--onboard-worker-answer-folder',
default=os.path.join(default_task_folder, 'onboard_answers'),
type=str,
help=
'base folder for saving all worker answer results during onboarding',
)
argparser.add_argument(
'--worker-blocklist-paths',
default=None,
type=str,
help=
'Path(s) to a list of IDs of workers to soft-block, separated by newlines. Use commas to indicate multiple lists',
)
argparser.add_argument(
'--check-acceptability',
default=False,
type=bool,
help=
"Check worker's responses against several metrics of acceptability",
)
argparser.add_argument('--include-persona',
default=False,
type=bool,
help="Show persona to the bot")
argparser.add_argument(
'--conversation-start-mode',
default='hi',
type=str,
choices=['hi', 'bst'],
help=
'Whether to show "Hi!" or two previous utterances (as in BlendedSkillTalk) at the beginning of the conversation',
)
argparser.add_argument(
'--context-seed',
default=None,
type=int,
help="Set seed for pulling the context info (for testing)",
)
argparser.add_argument(
'--hit-config-path',
default=os.path.join(config_folder, 'hit_config.json'),
type=str,
help=
'Path to file of parameters describing how MTurk will describe the HIT to the workers',
)
argparser.add_argument(
'--task-description-path',
default=os.path.join(config_folder, 'task_description.html'),
type=str,
help='Path to file of HTML to show on the task-description page',
)
argparser.add_argument(
'--left-pane-text-path',
default=os.path.join(config_folder, 'left_pane_text.html'),
type=str,
help=
'Path to file of HTML to show on the left-hand pane of the chat window',
)
argparser.add_argument(
'--annotations-intro',
default=
'Does this comment from your partner have any of the following attributes? (Check all that apply)',
type=str,
help='Text shown to worker before they fill out annotation form',
)
argparser.add_argument(
'--annotations-config-path',
default=os.path.join(config_folder, 'annotations_config.json'),
type=str,
help='Path to JSON of annotation categories',
)
argparser.add_argument(
'--onboard-task-data-path',
default=os.path.join(config_folder, 'onboard_task_data.json'),
type=str,
help='Path to JSON containing settings for running onboarding',
)
argparser.add_argument(
'--final-rating-question',
default='Please rate your partner on a scale of 1-5.',
type=str,
help='Text to show when asking worker to make their final rating',
)
# NOTE: you have to set all three of these opts to enforce the MTurk core
# param max_hits_per_worker.
# - Without unique_qual_name, MTurkManager creates different qualification
# for each run (so a worker could do N hits per run) Also, the
# worker has to get to N HITs in at least one run or they won't be given
# the qualification.
# - allowed_conversations is like max concurrent conversations
# allowed_conversations needs to be 1 or the actual max would be N +
# allowed_conversations. Worker gets notified via frontend message that
# they aren't eligible (second description screen), UNLESS the frontend
# overwrites that functionality.
# There's also still a race condition where the worker might be able to open
# 1 extra task
argparser.set_defaults(
unique_qual_name='turn_annotations_max_submissions',
max_hits_per_worker=10,
allowed_conversations=3,
)
if override_opt is not None:
argparser.set_params(**override_opt)
opt = argparser.parse_args([])
else:
opt = argparser.parse_args()
directory_path = os.path.dirname(os.path.abspath(__file__))
opt['task'] = os.path.basename(directory_path)
# Set the number of conversations needed
if opt.get('conversations_needed_string') is not None:
parts = opt['conversations_needed_string'].split(',')
conversations_needed = {}
for part in parts:
model_name, num_string = part.split(':')
conversations_needed[model_name] = int(num_string)
opt['conversations_needed'] = conversations_needed
# Read in workers to soft-block
if opt.get('worker_blocklist_paths') is not None:
blocklist_paths = opt['worker_blocklist_paths'].split(',')
worker_blocklist = set()
for path in blocklist_paths:
with open(path) as f:
worker_blocklist |= set(f.read().strip().split('\n'))
opt['worker_blocklist'] = worker_blocklist
# Read in and define text shown to users
if opt.get('hit_config') is None:
with open(opt['hit_config_path']) as f:
opt['hit_config'] = json.load(f)
opt.update(opt['hit_config'])
# Add all of the settings in hit_config into the base opt
if opt.get('task_description') is None:
with open(opt['task_description_path']) as f:
opt['task_description'] = f.readlines()
if opt.get('left_pane_text') is None:
with open(opt['left_pane_text_path']) as f:
opt['left_pane_text'] = f.readlines()
if opt.get('annotations_config') is None:
with open(opt['annotations_config_path']) as f:
opt['annotations_config'] = json.load(f)
if opt.get('onboard_task_data') is None:
with open(opt['onboard_task_data_path']) as f:
opt['onboard_task_data'] = json.load(f)
# Limits the number of models that can generate at once
max_concurrent_responses = 1
semaphore = threading.Semaphore(max_concurrent_responses)
run_statistics = copy.deepcopy(opt['conversations_needed'])
run_statistics = {r: 0 for (r, v) in run_statistics.items()}
onboard_statistics = {}
save_folder = 'sandbox' if opt['is_sandbox'] else 'live'
opt['save_folder'] = os.path.join(opt['base_save_folder'], save_folder,
time.strftime("%Y_%m_%d"))
os.makedirs(opt['save_folder'], exist_ok=True)
print(
f'Going to start collecting {opt["num_conversations"]} conversations, max_hits_per_worker: {opt["max_hits_per_worker"]}, reward: {opt["reward"]}, is_sandbox: {opt["is_sandbox"]}.'
)
# Create the models before it launches Heroku backend b/c takes a while
models_needed = list(opt['conversations_needed'].keys())
active_models = [
m for m in models_needed if opt['conversations_needed'][m] > 0
]
shared_bot_agents = TurkLikeAgent.get_bot_agents(opt, active_models)
mturk_agent_ids = [AGENT_0]
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
mturk_manager.setup_server(task_directory_path=directory_path)
if opt['include_persona'] or opt['conversation_start_mode'] == 'bst':
context_generator = ContextGenerator(opt, datatype='test', seed=0)
# We pull from the test set so that the model can't regurgitate
# memorized conversations
else:
context_generator = None
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
if not opt['is_sandbox']:
# Soft-block all chosen workers
if len(opt['worker_blocklist']) > 0:
print(
f"About to soft-block {len(opt['worker_blocklist'])} workers."
)
for w in set(opt['worker_blocklist']):
try:
print('Soft Blocking {}\n'.format(w))
mturk_manager.soft_block_worker(w)
except Exception as e:
print(f'Did not soft block worker {w}: {e}')
time.sleep(0.1)
else:
print(
'WARNING: We are in live mode, but a list of workers to soft-block '
'has not been passed in.')
def run_onboard(worker):
world = TurnAnnotationsOnboardWorld(opt, worker)
status = world.parley()
if status not in onboard_statistics:
onboard_statistics[status] = 0
onboard_statistics[status] += 1
print(
f'After onboard world parley. About to shutdown onboard world for {worker.worker_id}, status was: {status}. Total onboard statistics for this run are: {onboard_statistics}.'
)
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
workers[0].id = mturk_agent_ids[0]
def run_conversation(mturk_manager, opt, workers):
remaining_counts_needed = [
(m, c - run_statistics[m])
for (m, c) in opt['conversations_needed'].items()
]
remaining_counts_needed.sort(reverse=True, key=lambda x: x[1])
model_name = remaining_counts_needed[0][0]
print(
f'Remaining conversation counts needed: {remaining_counts_needed}'
)
# Get a bot and add it to the list of "workers"
print(f'Choosing the "{model_name}" model for the bot.')
agent = create_agent_from_shared(shared_bot_agents[model_name])
bot_worker = TurkLikeAgent(
opt,
model_name=model_name,
model_agent=agent,
num_turns=opt['num_turns'],
semaphore=semaphore,
)
workers_including_bot = workers + [bot_worker]
assert len(workers_including_bot) == 2
# Get context: personas, previous utterances, etc.
if context_generator is not None:
context_info = context_generator.get_context()
else:
context_info = None
conv_idx = mturk_manager.conversation_index
world = TurnAnnotationsChatWorld(
opt=opt,
agents=workers_including_bot,
num_turns=opt['num_turns'],
max_resp_time=opt['max_resp_time'],
tag='conversation t_{}'.format(conv_idx),
context_info=context_info,
)
while not world.episode_done():
print('About to parley')
world.parley()
model_nickname, worker_is_unacceptable, convo_finished = world.save_data(
)
if worker_is_unacceptable:
print(f'Soft-blocking worker {workers[0].worker_id}')
mturk_manager.soft_block_worker(workers[0].worker_id)
time.sleep(0.1)
if not worker_is_unacceptable and convo_finished:
run_statistics[model_nickname] += 1
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation,
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
