def run_hit(hit_index, assignment_index, opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
# Create mturk agents
mturk_agent_1 = MTurkAgent(id=mturk_agent_1_id,
manager=mturk_manager,
conversation_id=conversation_id,
opt=opt)
mturk_agent_2 = MTurkAgent(id=mturk_agent_2_id,
manager=mturk_manager,
conversation_id=conversation_id,
opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = human_agent_2_id
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
while not world.episode_done():
world.parley()
world.shutdown()
def test_worker_state_agent_management(self):
"""
Test public state management methods of worker_state.
"""
agent_1 = MTurkAgent(
self.opt,
self.mturk_manager,
TEST_HIT_ID_1,
TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1,
)
agent_2 = MTurkAgent(
self.opt,
self.mturk_manager,
TEST_HIT_ID_2,
TEST_ASSIGNMENT_ID_2,
TEST_WORKER_ID_1,
)
agent_3 = MTurkAgent(
self.opt,
self.mturk_manager,
TEST_HIT_ID_3,
TEST_ASSIGNMENT_ID_3,
TEST_WORKER_ID_3,
)
self.assertEqual(self.work_state_1.active_conversation_count(), 0)
self.work_state_1.add_agent(agent_1)
self.assertEqual(self.work_state_1.active_conversation_count(), 1)
self.work_state_1.add_agent(agent_2)
self.assertEqual(self.work_state_1.active_conversation_count(), 2)
with self.assertRaises(AssertionError):
self.work_state_1.add_agent(agent_3)
self.assertEqual(self.work_state_1.active_conversation_count(), 2)
self.assertEqual(self.work_state_1.completed_assignments(), 0)
self.assertTrue(self.work_state_1.has_assignment(
agent_1.assignment_id))
self.assertTrue(self.work_state_1.has_assignment(
agent_2.assignment_id))
self.assertFalse(
self.work_state_1.has_assignment(agent_3.assignment_id))
self.assertEqual(
agent_1,
self.work_state_1.get_agent_for_assignment(agent_1.assignment_id))
self.assertEqual(
agent_2,
self.work_state_1.get_agent_for_assignment(agent_2.assignment_id))
self.assertIsNone(
self.work_state_1.get_agent_for_assignment(agent_3.assignment_id))
agent_1.set_status(AssignState.STATUS_DONE)
self.assertEqual(self.work_state_1.active_conversation_count(), 1)
self.assertEqual(self.work_state_1.completed_assignments(), 1)
agent_2.set_status(AssignState.STATUS_DISCONNECT)
self.assertEqual(self.work_state_1.active_conversation_count(), 0)
self.assertEqual(self.work_state_1.completed_assignments(), 1)
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
# Initialize a SQuAD teacher agent, which we will later get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
task = task_class(task_opt)
# Create the MTurk agent which provides a chat interface to the Turker
opt.update(task_config)
mturk_agent_id = 'Worker'
opt['agent_id'] = mturk_agent_id
opt['mturk_agent_ids'] = [mturk_agent_id]
opt['all_agent_ids'] = [
QADataCollectionWorld.collector_agent_id, mturk_agent_id
]
opt['conversation_id'] = str(int(time.time()))
mturk_agent = MTurkAgent(opt=opt)
world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def run_hit(i, task_class, task_opt, opt, mturk_manager):
task = task_class(task_opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id='Worker', manager=mturk_manager, conversation_id=i, opt=opt)
world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def _create_agent(self, hit_id, assignment_id, worker_id):
"""Initialize an agent and add it to the map"""
agent = MTurkAgent(self.opt, self, hit_id, assignment_id, worker_id)
if (worker_id in self.mturk_agents):
self.mturk_agents[worker_id][assignment_id] = agent
else:
self.mturk_agents[worker_id] = {}
self.mturk_agents[worker_id][assignment_id] = agent
def run_hit(i, task_class, task_opt, opt):
task = task_class(task_opt)
opt['conversation_id'] = str(i)
mturk_agent = MTurkAgent(opt=opt)
world = QADataCollectionWorld(opt=opt,
task=task,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def run_hit(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
task = task_class(task_opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
human_agent_2_id = 'human_2'
# Create the MTurk agents
opt.update(task_config)
opt['conversation_id'] = str(int(time.time()))
opt['mturk_agent_ids'] = [mturk_agent_1_id, mturk_agent_2_id]
opt['all_agent_ids'] = [
human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id
]
opt['agent_id'] = mturk_agent_1_id
mturk_agent_1 = MTurkAgent(opt=opt)
opt['agent_id'] = mturk_agent_2_id
mturk_agent_2 = MTurkAgent(opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = human_agent_2_id
world = MultiAgentDialogWorld(
opt=opt,
agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
while not world.episode_done():
world.parley()
world.shutdown()
def run_hit(i, opt):
opt['conversation_id'] = str(i)
opt['agent_id'] = opt['mturk_agent_1_id']
mturk_agent_1 = MTurkAgent(opt=opt)
opt['agent_id'] = opt['mturk_agent_2_id']
mturk_agent_2 = MTurkAgent(opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = opt['human_agent_1_id']
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = opt['human_agent_2_id']
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
while not world.episode_done():
world.parley()
world.shutdown()
def run_hit(i, opt, task_opt, mturk_manager):
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id='Worker',
manager=mturk_manager,
conversation_id=i,
opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def setUp(self):
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args(print_args=False)
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt.copy(),
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
self.turk_agent = MTurkAgent(self.opt.copy(), self.mturk_manager,
TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1)
def run_hit(i, opt, mturk_manager):
# Create mturk agents
mturk_agent_1 = MTurkAgent(id='mturk_agent_1',
manager=mturk_manager,
conversation_id=i,
opt=opt)
mturk_agent_2 = MTurkAgent(id='mturk_agent_2',
manager=mturk_manager,
conversation_id=i,
opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = 'human_1'
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = 'human_2'
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
# Since we are using the regular MultiAgentDialogWorld, we do the following outside of the world instead.
mturk_agent_ids = [mturk_agent_1.id, mturk_agent_2.id]
all_agent_ids = [human_agent_1.id, human_agent_2.id] + mturk_agent_ids
mturk_agent_1.mturk_agent_ids = mturk_agent_ids
mturk_agent_1.all_agent_ids = all_agent_ids
mturk_agent_2.mturk_agent_ids = mturk_agent_ids
mturk_agent_2.all_agent_ids = all_agent_ids
mturk_agent_1.create_hit()
mturk_agent_2.create_hit()
while not world.episode_done():
world.parley()
world.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
model_agent = IrBaselineAgent(opt=opt)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
# Create the MTurk agent which provides a chat interface to the Turker
opt.update(task_config)
mturk_agent_id = 'Worker'
opt['agent_id'] = mturk_agent_id
opt['mturk_agent_ids'] = [mturk_agent_id]
opt['all_agent_ids'] = [
ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id
]
opt['conversation_id'] = str(int(time.time()))
mturk_agent = MTurkAgent(opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def _create_agent(self, hit_id, assignment_id, worker_id):
"""Initialize an agent and return it"""
return MTurkAgent(self.opt, self.mturk_manager, hit_id, assignment_id,
worker_id)
