def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_manager = MTurkManager()
mturk_manager.init_aws(opt=opt)
mturk_agent_id = 'Worker'
mturk_manager.mturk_agent_ids = [mturk_agent_id]
mturk_manager.all_agent_ids = [
ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id
] # In speaking order
global run_hit
def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id,
manager=mturk_manager,
conversation_id=conversation_id,
opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, opt, task_opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.review_hits()
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids = [mturk_agent_id],
all_agent_ids = [ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id] # In speaking order
)
mturk_manager.init_aws(opt=opt)
global run_hit
def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.create_hits(opt=opt)
results = Parallel(n_jobs=opt['num_hits'] * opt['num_assignments'], backend='threading') \
(delayed(run_hit)(hit_index, assignment_index, opt, task_opt, mturk_manager) \
for hit_index, assignment_index in product(range(1, opt['num_hits']+1), range(1, opt['num_assignments']+1)))
mturk_manager.shutdown()
def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def run_conversation(opt, workers):
mturk_agent = workers[0]
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
model_agent = IrBaselineAgent(opt=opt)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
# Create the MTurk agent which provides a chat interface to the Turker
opt.update(task_config)
mturk_agent_id = 'Worker'
opt['agent_id'] = mturk_agent_id
opt['mturk_agent_ids'] = [mturk_agent_id]
opt['all_agent_ids'] = [
ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id
]
opt['conversation_id'] = str(int(time.time()))
mturk_agent = MTurkAgent(opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
model_agent = IrBaselineAgent(opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
def run_hit(i, opt, task_opt, mturk_manager):
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id='Worker',
manager=mturk_manager,
conversation_id=i,
opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=[mturk_agent_id])
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = ModelEvaluatorOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
model_agent = IrBaselineAgent(opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.start_task(eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation)
except:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
opt.update(task_config)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
mturk_agent_id = 'Worker'
mturk_manager = MTurkManager(
opt=opt,
mturk_agent_ids=[mturk_agent_id]
)
mturk_manager.setup_server()
try:
mturk_manager.start_new_run()
mturk_manager.create_hits()
def run_onboard(worker):
world = ModelEvaluatorOnboardWorld(opt=opt, mturk_agent=worker)
while not world.episode_done():
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(worker):
worker[0].id = mturk_agent_id
global run_conversation
def run_conversation(mturk_manager, opt, workers):
mturk_agent = workers[0]
model_agent = IrBaselineAgent(opt=opt)
world = ModelEvaluatorWorld(
opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent
)
while not world.episode_done():
world.parley()
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()
