def run_hit(hit_index, assignment_index, opt, mturk_manager): conversation_id = str(hit_index) + '_' + str(assignment_index) # Create mturk agents mturk_agent_1 = MTurkAgent(id=mturk_agent_1_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) mturk_agent_2 = MTurkAgent(id=mturk_agent_2_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = human_agent_1_id human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = human_agent_2_id world = MultiAgentDialogWorld(opt=opt, agents=[ human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2 ]) while not world.episode_done(): world.parley() world.shutdown()
def test_worker_state_agent_management(self): """ Test public state management methods of worker_state. """ agent_1 = MTurkAgent( self.opt, self.mturk_manager, TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1, TEST_WORKER_ID_1, ) agent_2 = MTurkAgent( self.opt, self.mturk_manager, TEST_HIT_ID_2, TEST_ASSIGNMENT_ID_2, TEST_WORKER_ID_1, ) agent_3 = MTurkAgent( self.opt, self.mturk_manager, TEST_HIT_ID_3, TEST_ASSIGNMENT_ID_3, TEST_WORKER_ID_3, ) self.assertEqual(self.work_state_1.active_conversation_count(), 0) self.work_state_1.add_agent(agent_1) self.assertEqual(self.work_state_1.active_conversation_count(), 1) self.work_state_1.add_agent(agent_2) self.assertEqual(self.work_state_1.active_conversation_count(), 2) with self.assertRaises(AssertionError): self.work_state_1.add_agent(agent_3) self.assertEqual(self.work_state_1.active_conversation_count(), 2) self.assertEqual(self.work_state_1.completed_assignments(), 0) self.assertTrue(self.work_state_1.has_assignment( agent_1.assignment_id)) self.assertTrue(self.work_state_1.has_assignment( agent_2.assignment_id)) self.assertFalse( self.work_state_1.has_assignment(agent_3.assignment_id)) self.assertEqual( agent_1, self.work_state_1.get_agent_for_assignment(agent_1.assignment_id)) self.assertEqual( agent_2, self.work_state_1.get_agent_for_assignment(agent_2.assignment_id)) self.assertIsNone( self.work_state_1.get_agent_for_assignment(agent_3.assignment_id)) agent_1.set_status(AssignState.STATUS_DONE) self.assertEqual(self.work_state_1.active_conversation_count(), 1) self.assertEqual(self.work_state_1.completed_assignments(), 1) agent_2.set_status(AssignState.STATUS_DISCONNECT) self.assertEqual(self.work_state_1.active_conversation_count(), 0) self.assertEqual(self.work_state_1.completed_assignments(), 1)
def setUp(self): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() self.opt = argparser.parse_args(print_args=False) self.opt['task'] = 'unittest' self.opt['assignment_duration_in_seconds'] = 6 mturk_agent_ids = ['mturk_agent_1'] self.mturk_manager = MTurkManager(opt=self.opt.copy(), mturk_agent_ids=mturk_agent_ids) self.worker_manager = self.mturk_manager.worker_manager self.turk_agent = MTurkAgent(self.opt.copy(), self.mturk_manager, TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1, TEST_WORKER_ID_1)
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) # Initialize a SQuAD teacher agent, which we will later get context from module_name = 'parlai.tasks.squad.agents' class_name = 'DefaultTeacher' my_module = importlib.import_module(module_name) task_class = getattr(my_module, class_name) task_opt = {} task_opt['datatype'] = 'train' task_opt['datapath'] = opt['datapath'] task = task_class(task_opt) # Create the MTurk agent which provides a chat interface to the Turker opt.update(task_config) mturk_agent_id = 'Worker' opt['agent_id'] = mturk_agent_id opt['mturk_agent_ids'] = [mturk_agent_id] opt['all_agent_ids'] = [ QADataCollectionWorld.collector_agent_id, mturk_agent_id ] opt['conversation_id'] = str(int(time.time())) mturk_agent = MTurkAgent(opt=opt) world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown()
def main(): global run_hit argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) opt['mturk_agent_1_id'] = 'mturk_agent_1' opt['mturk_agent_2_id'] = 'mturk_agent_2' opt['human_agent_1_id'] = 'human_1' opt['human_agent_2_id'] = 'human_2' opt.update(task_config) opt['mturk_agent_ids'] = [opt['mturk_agent_1_id'], opt['mturk_agent_2_id']] opt['all_agent_ids'] = [opt['human_agent_1_id'], opt['human_agent_2_id'] ] + opt['mturk_agent_ids'] opt['run_id'] = str(int(time.time())) def run_hit(i, opt): opt['conversation_id'] = str(i) opt['agent_id'] = opt['mturk_agent_1_id'] mturk_agent_1 = MTurkAgent(opt=opt) opt['agent_id'] = opt['mturk_agent_2_id'] mturk_agent_2 = MTurkAgent(opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = opt['human_agent_1_id'] human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = opt['human_agent_2_id'] world = MultiAgentDialogWorld(opt=opt, agents=[ human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2 ]) while not world.episode_done(): world.parley() world.shutdown() MTurkAgent.init_aws(opt) results = Parallel(n_jobs=opt['num_hits'], backend='threading')( delayed(run_hit)(i, copy.deepcopy(opt)) for i in range(1, opt['num_hits'] + 1)) MTurkAgent.review_hits(opt=opt)
def _create_agent(self, hit_id, assignment_id, worker_id): """Initialize an agent and add it to the map""" agent = MTurkAgent(self.opt, self, hit_id, assignment_id, worker_id) if (worker_id in self.mturk_agents): self.mturk_agents[worker_id][assignment_id] = agent else: self.mturk_agents[worker_id] = {} self.mturk_agents[worker_id][assignment_id] = agent
def run_hit(i, task_class, task_opt, opt, mturk_manager): task = task_class(task_opt) # Create the MTurk agent which provides a chat interface to the Turker mturk_agent = MTurkAgent(id='Worker', manager=mturk_manager, conversation_id=i, opt=opt) world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown()
def main(): global run_hit argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) # Initialize a SQuAD teacher agent, which we will later get context from module_name = 'parlai.tasks.squad.agents' class_name = 'DefaultTeacher' my_module = importlib.import_module(module_name) task_class = getattr(my_module, class_name) task_opt = {} task_opt['datatype'] = 'train' task_opt['datapath'] = opt['datapath'] # Create the MTurk agent which provides a chat interface to the Turker opt.update(task_config) mturk_agent_id = 'Worker' opt['agent_id'] = mturk_agent_id opt['mturk_agent_ids'] = [mturk_agent_id] opt['all_agent_ids'] = [ QADataCollectionWorld.collector_agent_id, mturk_agent_id ] opt['run_id'] = str(int(time.time())) def run_hit(i, task_class, task_opt, opt): task = task_class(task_opt) opt['conversation_id'] = str(i) mturk_agent = MTurkAgent(opt=opt) world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown() MTurkAgent.init_aws(opt) results = Parallel(n_jobs=opt['num_hits'], backend='threading')( delayed(run_hit)(i, task_class, task_opt, copy.deepcopy(opt)) for i in range(1, opt['num_hits'] + 1)) MTurkAgent.review_hits(opt=opt)
def run_hit(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager): conversation_id = str(hit_index) + '_' + str(assignment_index) task = task_class(task_opt) # Create the MTurk agent which provides a chat interface to the Turker mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown()
def run_hit(i, task_class, task_opt, opt): task = task_class(task_opt) opt['conversation_id'] = str(i) mturk_agent = MTurkAgent(opt=opt) world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown()
def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager): conversation_id = str(hit_index) + '_' + str(assignment_index) model_agent = IrBaselineAgent(opt=opt) # Create the MTurk agent which provides a chat interface to the Turker mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt) world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) mturk_agent_1_id = 'mturk_agent_1' mturk_agent_2_id = 'mturk_agent_2' human_agent_1_id = 'human_1' human_agent_2_id = 'human_2' # Create the MTurk agents opt.update(task_config) opt['conversation_id'] = str(int(time.time())) opt['mturk_agent_ids'] = [mturk_agent_1_id, mturk_agent_2_id] opt['all_agent_ids'] = [ human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id ] opt['agent_id'] = mturk_agent_1_id mturk_agent_1 = MTurkAgent(opt=opt) opt['agent_id'] = mturk_agent_2_id mturk_agent_2 = MTurkAgent(opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = human_agent_1_id human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = human_agent_2_id world = MultiAgentDialogWorld( opt=opt, agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2]) while not world.episode_done(): world.parley() world.shutdown()
def run_hit(i, opt): opt['conversation_id'] = str(i) opt['agent_id'] = opt['mturk_agent_1_id'] mturk_agent_1 = MTurkAgent(opt=opt) opt['agent_id'] = opt['mturk_agent_2_id'] mturk_agent_2 = MTurkAgent(opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = opt['human_agent_1_id'] human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = opt['human_agent_2_id'] world = MultiAgentDialogWorld(opt=opt, agents=[ human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2 ]) while not world.episode_done(): world.parley() world.shutdown()
def run_hit(i, opt, task_opt, mturk_manager): model_agent = IrBaselineAgent(opt=opt) # Create the MTurk agent which provides a chat interface to the Turker mturk_agent = MTurkAgent(id='Worker', manager=mturk_manager, conversation_id=i, opt=opt) world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown()
def main(): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() # The dialog model we want to evaluate from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent IrBaselineAgent.add_cmdline_args(argparser) opt = argparser.parse_args() opt['task'] = os.path.basename(os.getcwd()) model_agent = IrBaselineAgent(opt=opt) # The task that we will evaluate the dialog model on task_opt = {} task_opt['datatype'] = 'test' task_opt['datapath'] = opt['datapath'] task_opt['task'] = '#MovieDD-Reddit' # Create the MTurk agent which provides a chat interface to the Turker opt.update(task_config) mturk_agent_id = 'Worker' opt['agent_id'] = mturk_agent_id opt['mturk_agent_ids'] = [mturk_agent_id] opt['all_agent_ids'] = [ ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id ] opt['conversation_id'] = str(int(time.time())) mturk_agent = MTurkAgent(opt=opt) world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent) while not world.episode_done(): world.parley() world.shutdown()
def _create_agent(self, hit_id, assignment_id, worker_id): """Initialize an agent and return it""" return MTurkAgent(self.opt, self.mturk_manager, hit_id, assignment_id, worker_id)
def run_hit(i, opt, mturk_manager): # Create mturk agents mturk_agent_1 = MTurkAgent(id='mturk_agent_1', manager=mturk_manager, conversation_id=i, opt=opt) mturk_agent_2 = MTurkAgent(id='mturk_agent_2', manager=mturk_manager, conversation_id=i, opt=opt) # Create the local human agents human_agent_1 = LocalHumanAgent(opt=None) human_agent_1.id = 'human_1' human_agent_2 = LocalHumanAgent(opt=None) human_agent_2.id = 'human_2' world = MultiAgentDialogWorld(opt=opt, agents=[ human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2 ]) # Since we are using the regular MultiAgentDialogWorld, we do the following outside of the world instead. mturk_agent_ids = [mturk_agent_1.id, mturk_agent_2.id] all_agent_ids = [human_agent_1.id, human_agent_2.id] + mturk_agent_ids mturk_agent_1.mturk_agent_ids = mturk_agent_ids mturk_agent_1.all_agent_ids = all_agent_ids mturk_agent_2.mturk_agent_ids = mturk_agent_ids mturk_agent_2.all_agent_ids = all_agent_ids mturk_agent_1.create_hit() mturk_agent_2.create_hit() while not world.episode_done(): world.parley() world.shutdown()
class TestMTurkAgent(unittest.TestCase): """ Various unit tests for the MTurkAgent class. """ def setUp(self): argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() argparser.add_mturk_args() self.opt = argparser.parse_args(print_args=False) self.opt['task'] = 'unittest' self.opt['assignment_duration_in_seconds'] = 6 mturk_agent_ids = ['mturk_agent_1'] self.mturk_manager = MTurkManager(opt=self.opt.copy(), mturk_agent_ids=mturk_agent_ids) self.worker_manager = self.mturk_manager.worker_manager self.turk_agent = MTurkAgent( self.opt.copy(), self.mturk_manager, TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1, TEST_WORKER_ID_1, ) def tearDown(self): self.mturk_manager.shutdown() disconnect_path = os.path.join(parent_dir, 'disconnect-test.pickle') if os.path.exists(disconnect_path): os.remove(disconnect_path) def test_init(self): """ Test initialization of an agent. """ self.assertIsNotNone(self.turk_agent.creation_time) self.assertIsNone(self.turk_agent.id) self.assertIsNone(self.turk_agent.message_request_time) self.assertIsNone(self.turk_agent.conversation_id) self.assertFalse(self.turk_agent.some_agent_disconnected) self.assertFalse(self.turk_agent.hit_is_expired) self.assertFalse(self.turk_agent.hit_is_abandoned) self.assertFalse(self.turk_agent.hit_is_returned) self.assertFalse(self.turk_agent.hit_is_complete) self.assertFalse(self.turk_agent.disconnected) self.assertTrue(self.turk_agent.alived) def test_state_wrappers(self): """ Test the mturk agent wrappers around its state. """ for status in statuses: self.turk_agent.set_status(status) self.assertEqual(self.turk_agent.get_status(), status) for status in [ AssignState.STATUS_DONE, AssignState.STATUS_PARTNER_DISCONNECT ]: self.turk_agent.set_status(status) self.assertTrue(self.turk_agent.submitted_hit()) for status in active_statuses: self.turk_agent.set_status(status) self.assertFalse(self.turk_agent.is_final()) for status in complete_statuses: self.turk_agent.set_status(status) self.assertTrue(self.turk_agent.is_final()) self.turk_agent.append_message(MESSAGE_1) self.assertEqual(len(self.turk_agent.get_messages()), 1) self.turk_agent.append_message(MESSAGE_2) self.assertEqual(len(self.turk_agent.get_messages()), 2) self.turk_agent.append_message(MESSAGE_1) self.assertEqual(len(self.turk_agent.get_messages()), 2) self.assertIn(MESSAGE_1, self.turk_agent.get_messages()) self.assertIn(MESSAGE_2, self.turk_agent.get_messages()) # Ensure command interactions work as expected self.turk_agent.set_last_command(COMMAND_1) self.assertEqual(self.turk_agent.get_last_command(), COMMAND_1) self.turk_agent.clear_messages() self.assertEqual(len(self.turk_agent.get_messages()), 0) # In task checks self.turk_agent.conversation_id = 't_12345' self.assertTrue(self.turk_agent.is_in_task()) self.turk_agent.conversation_id = 'b_12345' self.assertFalse(self.turk_agent.is_in_task()) def test_connection_id(self): """ Ensure the connection_id hasn't changed. """ connection_id = "{}_{}".format(self.turk_agent.worker_id, self.turk_agent.assignment_id) self.assertEqual(self.turk_agent.get_connection_id(), connection_id) def test_inactive_data(self): """ Ensure data packet generated for inactive commands is valid. """ for status in complete_statuses: self.turk_agent.set_status(status) data = self.turk_agent.get_inactive_command_data() self.assertIsNotNone(data['text']) self.assertIsNotNone(data['inactive_text']) self.assertEqual(data['conversation_id'], self.turk_agent.conversation_id) self.assertEqual(data['agent_id'], TEST_WORKER_ID_1) def test_status_change(self): has_changed = False self.turk_agent.set_status(AssignState.STATUS_ONBOARDING) def wait_for_status_wrap(): nonlocal has_changed # noqa 999 we don't use python2 self.turk_agent.wait_for_status(AssignState.STATUS_WAITING) has_changed = True t = threading.Thread(target=wait_for_status_wrap, daemon=True) t.start() self.assertFalse(has_changed) time.sleep(0.07) self.assertFalse(has_changed) self.turk_agent.set_status(AssignState.STATUS_WAITING) time.sleep(0.07) self.assertTrue(has_changed) def test_message_queue(self): """ Ensure observations and acts work as expected. """ self.mturk_manager.send_message = mock.MagicMock() self.turk_agent.observe(ACT_1) self.mturk_manager.send_message.assert_called_with( TEST_WORKER_ID_1, TEST_ASSIGNMENT_ID_1, ACT_1) # First act comes through the queue and returns properly self.assertTrue(self.turk_agent.msg_queue.empty()) self.turk_agent.id = AGENT_ID self.turk_agent.put_data(MESSAGE_ID_1, ACT_1) self.assertTrue(self.turk_agent.recieved_packets[MESSAGE_ID_1]) self.assertFalse(self.turk_agent.msg_queue.empty()) returned_act = self.turk_agent.get_new_act_message() self.assertEqual(returned_act, ACT_1) # Repeat act is ignored self.turk_agent.put_data(MESSAGE_ID_1, ACT_1) self.assertTrue(self.turk_agent.msg_queue.empty()) for i in range(100): self.turk_agent.put_data(str(i), ACT_1) self.assertEqual(self.turk_agent.msg_queue.qsize(), 100) self.turk_agent.flush_msg_queue() self.assertTrue(self.turk_agent.msg_queue.empty()) # Test non-act messages blank_message = self.turk_agent.get_new_act_message() self.assertIsNone(blank_message) self.turk_agent.disconnected = True disconnect_message = self.turk_agent.get_new_act_message() self.turk_agent.disconnected = False self.assertEqual(disconnect_message['text'], self.turk_agent.MTURK_DISCONNECT_MESSAGE) self.turk_agent.hit_is_returned = True return_message = self.turk_agent.get_new_act_message() self.assertEqual(return_message['text'], self.turk_agent.RETURN_MESSAGE) self.turk_agent.hit_is_returned = False # Reduce state self.turk_agent.reduce_state() self.assertIsNone(self.turk_agent.msg_queue) self.assertIsNone(self.turk_agent.recieved_packets) def test_message_acts(self): self.mturk_manager.send_command = mock.MagicMock() self.mturk_manager.handle_turker_timeout = mock.MagicMock() # non-Blocking check self.assertIsNone(self.turk_agent.message_request_time) returned_act = self.turk_agent.act(blocking=False) self.assertIsNotNone(self.turk_agent.message_request_time) self.assertIsNone(returned_act) self.turk_agent.id = AGENT_ID self.turk_agent.put_data(MESSAGE_ID_1, ACT_1) returned_act = self.turk_agent.act(blocking=False) self.assertIsNone(self.turk_agent.message_request_time) self.assertEqual(returned_act, ACT_1) self.mturk_manager.send_command.assert_called_once() # non-Blocking timeout check self.mturk_manager.send_command = mock.MagicMock() returned_act = self.turk_agent.act(timeout=0.07, blocking=False) self.assertIsNotNone(self.turk_agent.message_request_time) self.assertIsNone(returned_act) while returned_act is None: returned_act = self.turk_agent.act(timeout=0.07, blocking=False) self.mturk_manager.send_command.assert_called_once() self.mturk_manager.handle_turker_timeout.assert_called_once() self.assertEqual(returned_act['text'], self.turk_agent.TIMEOUT_MESSAGE) # Blocking timeout check self.mturk_manager.send_command = mock.MagicMock() self.mturk_manager.handle_turker_timeout = mock.MagicMock() returned_act = self.turk_agent.act(timeout=0.07) self.mturk_manager.send_command.assert_called_once() self.mturk_manager.handle_turker_timeout.assert_called_once() self.assertEqual(returned_act['text'], self.turk_agent.TIMEOUT_MESSAGE)