def setUp(self):
self.agent_state1 = AssignState()
self.agent_state2 = AssignState(status=AssignState.STATUS_IN_TASK)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt,
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
def setUp(self):
self.work_state_1 = WorkerState(TEST_WORKER_ID_1, 10)
self.work_state_2 = WorkerState(TEST_WORKER_ID_2)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt, mturk_agent_ids=mturk_agent_ids)
self.mturk_manager.send_message = mock.MagicMock()
self.mturk_manager.send_state_change = mock.MagicMock()
self.mturk_manager.send_command = mock.MagicMock()
self.worker_manager = WorkerManager(self.mturk_manager, self.opt)
def setUp(self):
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt.copy(),
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
self.mturk_manager.send_message = mock.MagicMock()
self.mturk_manager.send_state_change = mock.MagicMock()
self.mturk_manager.send_command = mock.MagicMock()
self.turk_agent = MTurkAgent(
self.opt.copy(),
self.mturk_manager,
TEST_HIT_ID_1,
TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1,
)
def setUp(self):
disconnect_path = os.path.join(parent_dir, 'disconnect-test.pickle')
if os.path.exists(disconnect_path):
os.remove(disconnect_path)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(
opt=self.opt.copy(), mturk_agent_ids=mturk_agent_ids
)
self.worker_manager = self.mturk_manager.worker_manager
self.mturk_manager.send_message = mock.MagicMock()
self.mturk_manager.send_state_change = mock.MagicMock()
self.mturk_manager.send_command = mock.MagicMock()
self.worker_state_1 = self.worker_manager.worker_alive(TEST_WORKER_ID_1)
self.worker_state_2 = self.worker_manager.worker_alive(TEST_WORKER_ID_2)
self.worker_state_3 = self.worker_manager.worker_alive(TEST_WORKER_ID_3)
class TestAssignState(unittest.TestCase):
"""
Various unit tests for the AssignState class.
"""
def setUp(self):
self.agent_state1 = AssignState()
self.agent_state2 = AssignState(status=AssignState.STATUS_IN_TASK)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt,
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
def tearDown(self):
self.mturk_manager.shutdown()
def test_assign_state_init(self):
"""
Test proper initialization of assignment states.
"""
self.assertEqual(self.agent_state1.status, AssignState.STATUS_NONE)
self.assertEqual(len(self.agent_state1.messages), 0)
self.assertEqual(len(self.agent_state1.message_ids), 0)
self.assertEqual(self.agent_state2.status, AssignState.STATUS_IN_TASK)
self.assertEqual(len(self.agent_state1.messages), 0)
self.assertEqual(len(self.agent_state1.message_ids), 0)
def test_message_management(self):
"""
Test message management in an AssignState.
"""
# Ensure message appends succeed and are idempotent
self.agent_state1.append_message(MESSAGE_1)
self.assertEqual(len(self.agent_state1.get_messages()), 1)
self.agent_state1.append_message(MESSAGE_2)
self.assertEqual(len(self.agent_state1.get_messages()), 2)
self.agent_state1.append_message(MESSAGE_1)
self.assertEqual(len(self.agent_state1.get_messages()), 2)
self.assertEqual(len(self.agent_state2.get_messages()), 0)
self.assertIn(MESSAGE_1, self.agent_state1.get_messages())
self.assertIn(MESSAGE_2, self.agent_state1.get_messages())
self.assertEqual(len(self.agent_state1.message_ids), 2)
self.agent_state2.append_message(MESSAGE_1)
self.assertEqual(len(self.agent_state2.message_ids), 1)
# Ensure clearing messages acts as intended and doesn't clear agent2
self.agent_state1.clear_messages()
self.assertEqual(len(self.agent_state1.messages), 0)
self.assertEqual(len(self.agent_state1.message_ids), 0)
self.assertEqual(len(self.agent_state2.message_ids), 1)
def test_state_handles_status(self):
"""
Ensures status updates and is_final are valid.
"""
for status in statuses:
self.agent_state1.set_status(status)
self.assertEqual(self.agent_state1.get_status(), status)
for status in active_statuses:
self.agent_state1.set_status(status)
self.assertFalse(self.agent_state1.is_final())
for status in complete_statuses:
self.agent_state1.set_status(status)
self.assertTrue(self.agent_state1.is_final())
class TestMTurkAgent(unittest.TestCase):
"""
Various unit tests for the MTurkAgent class.
"""
def setUp(self):
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt.copy(),
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
self.mturk_manager.send_message = mock.MagicMock()
self.mturk_manager.send_state_change = mock.MagicMock()
self.mturk_manager.send_command = mock.MagicMock()
self.turk_agent = MTurkAgent(
self.opt.copy(),
self.mturk_manager,
TEST_HIT_ID_1,
TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1,
)
def tearDown(self):
self.mturk_manager.shutdown()
disconnect_path = os.path.join(parent_dir, 'disconnect-test.pickle')
if os.path.exists(disconnect_path):
os.remove(disconnect_path)
def test_init(self):
"""
Test initialization of an agent.
"""
self.assertIsNotNone(self.turk_agent.creation_time)
self.assertIsNone(self.turk_agent.id)
self.assertIsNone(self.turk_agent.message_request_time)
self.assertIsNone(self.turk_agent.conversation_id)
self.assertFalse(self.turk_agent.some_agent_disconnected)
self.assertFalse(self.turk_agent.hit_is_expired)
self.assertFalse(self.turk_agent.hit_is_abandoned)
self.assertFalse(self.turk_agent.hit_is_returned)
self.assertFalse(self.turk_agent.hit_is_complete)
self.assertFalse(self.turk_agent.disconnected)
def test_state_wrappers(self):
"""
Test the mturk agent wrappers around its state.
"""
for status in statuses:
self.turk_agent.set_status(status)
self.assertEqual(self.turk_agent.get_status(), status)
for status in [
AssignState.STATUS_DONE, AssignState.STATUS_PARTNER_DISCONNECT
]:
self.turk_agent.set_status(status)
self.assertTrue(self.turk_agent.submitted_hit())
for status in active_statuses:
self.turk_agent.set_status(status)
self.assertFalse(self.turk_agent.is_final())
for status in complete_statuses:
self.turk_agent.set_status(status)
self.assertTrue(self.turk_agent.is_final())
self.turk_agent.state.append_message(MESSAGE_1)
self.assertEqual(len(self.turk_agent.get_messages()), 1)
self.turk_agent.state.append_message(MESSAGE_2)
self.assertEqual(len(self.turk_agent.get_messages()), 2)
self.turk_agent.state.append_message(MESSAGE_1)
self.assertEqual(len(self.turk_agent.get_messages()), 2)
self.assertIn(MESSAGE_1, self.turk_agent.get_messages())
self.assertIn(MESSAGE_2, self.turk_agent.get_messages())
self.turk_agent.clear_messages()
self.assertEqual(len(self.turk_agent.get_messages()), 0)
def test_connection_id(self):
"""
Ensure the connection_id hasn't changed.
"""
connection_id = "{}_{}".format(self.turk_agent.worker_id,
self.turk_agent.assignment_id)
self.assertEqual(self.turk_agent.get_connection_id(), connection_id)
def test_status_change(self):
self.turk_agent.set_status(AssignState.STATUS_ONBOARDING)
time.sleep(0.07)
self.assertEqual(self.turk_agent.get_status(),
AssignState.STATUS_ONBOARDING)
self.turk_agent.set_status(AssignState.STATUS_WAITING)
time.sleep(0.07)
self.assertEqual(self.turk_agent.get_status(),
AssignState.STATUS_WAITING)
def test_message_queue(self):
"""
Ensure observations and acts work as expected.
"""
self.turk_agent.observe(ACT_1)
self.mturk_manager.send_message.assert_called_with(
TEST_WORKER_ID_1, TEST_ASSIGNMENT_ID_1, ACT_1)
# First act comes through the queue and returns properly
self.assertTrue(self.turk_agent.msg_queue.empty())
self.turk_agent.id = AGENT_ID
self.turk_agent.put_data(MESSAGE_ID_1, ACT_1)
self.assertTrue(self.turk_agent.recieved_packets[MESSAGE_ID_1])
self.assertFalse(self.turk_agent.msg_queue.empty())
returned_act = self.turk_agent.get_new_act_message()
self.assertEqual(returned_act, ACT_1)
# Repeat act is ignored
self.turk_agent.put_data(MESSAGE_ID_1, ACT_1)
self.assertTrue(self.turk_agent.msg_queue.empty())
for i in range(100):
self.turk_agent.put_data(str(i), ACT_1)
self.assertEqual(self.turk_agent.msg_queue.qsize(), 100)
self.turk_agent.flush_msg_queue()
self.assertTrue(self.turk_agent.msg_queue.empty())
# Test non-act messages
blank_message = self.turk_agent.get_new_act_message()
self.assertIsNone(blank_message)
self.turk_agent.disconnected = True
with self.assertRaises(AgentDisconnectedError):
# Expect this to be a disconnect message
self.turk_agent.get_new_act_message()
self.turk_agent.disconnected = False
self.turk_agent.hit_is_returned = True
with self.assertRaises(AgentReturnedError):
# Expect this to be a returned message
self.turk_agent.get_new_act_message()
self.turk_agent.hit_is_returned = False
# Reduce state
self.turk_agent.reduce_state()
self.assertIsNone(self.turk_agent.msg_queue)
self.assertIsNone(self.turk_agent.recieved_packets)
def test_message_acts(self):
self.mturk_manager.handle_turker_timeout = mock.MagicMock()
# non-Blocking check
self.assertIsNone(self.turk_agent.message_request_time)
returned_act = self.turk_agent.act(blocking=False)
self.assertIsNotNone(self.turk_agent.message_request_time)
self.assertIsNone(returned_act)
self.turk_agent.id = AGENT_ID
self.turk_agent.put_data(MESSAGE_ID_1, ACT_1)
returned_act = self.turk_agent.act(blocking=False)
self.assertIsNone(self.turk_agent.message_request_time)
self.assertEqual(returned_act, ACT_1)
self.mturk_manager.send_command.assert_called_once()
# non-Blocking timeout check
with self.assertRaises(AgentTimeoutError):
self.mturk_manager.send_command = mock.MagicMock()
returned_act = self.turk_agent.act(timeout=0.07, blocking=False)
self.assertIsNotNone(self.turk_agent.message_request_time)
self.assertIsNone(returned_act)
while returned_act is None:
returned_act = self.turk_agent.act(timeout=0.07,
blocking=False)
# Blocking timeout check
with self.assertRaises(AgentTimeoutError):
returned_act = self.turk_agent.act(timeout=0.07)
class TestWorkerState(unittest.TestCase):
"""
Various unit tests for the WorkerState class.
"""
def setUp(self):
self.work_state_1 = WorkerState(TEST_WORKER_ID_1, 10)
self.work_state_2 = WorkerState(TEST_WORKER_ID_2)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt, mturk_agent_ids=mturk_agent_ids)
self.mturk_manager.send_message = mock.MagicMock()
self.mturk_manager.send_state_change = mock.MagicMock()
self.mturk_manager.send_command = mock.MagicMock()
self.worker_manager = WorkerManager(self.mturk_manager, self.opt)
def tearDown(self):
self.mturk_manager.shutdown()
def test_worker_state_init(self):
"""
Test proper initialization of worker states.
"""
self.assertEqual(self.work_state_1.worker_id, TEST_WORKER_ID_1)
self.assertEqual(self.work_state_2.worker_id, TEST_WORKER_ID_2)
self.assertEqual(self.work_state_1.disconnects, 10)
self.assertEqual(self.work_state_2.disconnects, 0)
def test_worker_state_agent_management(self):
"""
Test public state management methods of worker_state.
"""
agent_1 = MTurkAgent(
self.opt,
self.mturk_manager,
TEST_HIT_ID_1,
TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1,
)
agent_2 = MTurkAgent(
self.opt,
self.mturk_manager,
TEST_HIT_ID_2,
TEST_ASSIGNMENT_ID_2,
TEST_WORKER_ID_1,
)
agent_3 = MTurkAgent(
self.opt,
self.mturk_manager,
TEST_HIT_ID_3,
TEST_ASSIGNMENT_ID_3,
TEST_WORKER_ID_3,
)
self.assertEqual(self.work_state_1.active_conversation_count(), 0)
self.work_state_1.add_agent(agent_1)
self.assertEqual(self.work_state_1.active_conversation_count(), 1)
self.work_state_1.add_agent(agent_2)
self.assertEqual(self.work_state_1.active_conversation_count(), 2)
with self.assertRaises(AssertionError):
self.work_state_1.add_agent(agent_3)
self.assertEqual(self.work_state_1.active_conversation_count(), 2)
self.assertEqual(self.work_state_1.completed_assignments(), 0)
self.assertTrue(self.work_state_1.has_assignment(agent_1.assignment_id))
self.assertTrue(self.work_state_1.has_assignment(agent_2.assignment_id))
self.assertFalse(self.work_state_1.has_assignment(agent_3.assignment_id))
self.assertEqual(
agent_1, self.work_state_1.get_agent_for_assignment(agent_1.assignment_id)
)
self.assertEqual(
agent_2, self.work_state_1.get_agent_for_assignment(agent_2.assignment_id)
)
self.assertIsNone(
self.work_state_1.get_agent_for_assignment(agent_3.assignment_id)
)
agent_1.set_status(AssignState.STATUS_DONE)
self.assertEqual(self.work_state_1.active_conversation_count(), 1)
self.assertEqual(self.work_state_1.completed_assignments(), 1)
agent_2.set_status(AssignState.STATUS_DISCONNECT)
self.assertEqual(self.work_state_1.active_conversation_count(), 0)
self.assertEqual(self.work_state_1.completed_assignments(), 1)
def test_manager_alive_makes_state(self):
test_worker = self.worker_manager.worker_alive(TEST_WORKER_ID_1)
self.assertIsInstance(test_worker, WorkerState)
self.assertEqual(test_worker.worker_id, TEST_WORKER_ID_1)
self.assertNotEqual(test_worker, self.work_state_1)
class TestWorkerManager(unittest.TestCase):
"""
Various unit tests for the WorkerManager class.
"""
def setUp(self):
disconnect_path = os.path.join(parent_dir, 'disconnect-test.pickle')
if os.path.exists(disconnect_path):
os.remove(disconnect_path)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(
opt=self.opt.copy(), mturk_agent_ids=mturk_agent_ids
)
self.worker_manager = self.mturk_manager.worker_manager
self.mturk_manager.send_message = mock.MagicMock()
self.mturk_manager.send_state_change = mock.MagicMock()
self.mturk_manager.send_command = mock.MagicMock()
self.worker_state_1 = self.worker_manager.worker_alive(TEST_WORKER_ID_1)
self.worker_state_2 = self.worker_manager.worker_alive(TEST_WORKER_ID_2)
self.worker_state_3 = self.worker_manager.worker_alive(TEST_WORKER_ID_3)
def tearDown(self):
self.mturk_manager.shutdown()
disconnect_path = os.path.join(parent_dir, 'disconnect-test.pickle')
if os.path.exists(disconnect_path):
os.remove(disconnect_path)
def test_private_create_agent(self):
"""
Check create agent method used internally in worker_manager.
"""
test_agent = self.worker_manager._create_agent(
TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1, TEST_WORKER_ID_1
)
self.assertIsInstance(test_agent, MTurkAgent)
self.assertEqual(test_agent.worker_id, TEST_WORKER_ID_1)
self.assertEqual(test_agent.hit_id, TEST_HIT_ID_1)
self.assertEqual(test_agent.assignment_id, TEST_ASSIGNMENT_ID_1)
def test_agent_task_management(self):
"""
Ensure agents and tasks have proper bookkeeping.
"""
self.worker_manager.assign_task_to_worker(
TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1, TEST_WORKER_ID_1
)
self.worker_manager.assign_task_to_worker(
TEST_HIT_ID_2, TEST_ASSIGNMENT_ID_2, TEST_WORKER_ID_2
)
self.worker_manager.assign_task_to_worker(
TEST_HIT_ID_3, TEST_ASSIGNMENT_ID_3, TEST_WORKER_ID_1
)
self.assertTrue(self.worker_state_1.has_assignment(TEST_ASSIGNMENT_ID_1))
self.assertTrue(self.worker_state_1.has_assignment(TEST_ASSIGNMENT_ID_3))
self.assertTrue(self.worker_state_2.has_assignment(TEST_ASSIGNMENT_ID_2))
assign_agent = self.worker_manager.get_agent_for_assignment(
TEST_ASSIGNMENT_ID_1
)
self.assertEqual(assign_agent.worker_id, TEST_WORKER_ID_1)
self.assertEqual(assign_agent.hit_id, TEST_HIT_ID_1)
self.assertEqual(assign_agent.assignment_id, TEST_ASSIGNMENT_ID_1)
no_such_agent = self.worker_manager.get_agent_for_assignment(FAKE_ID)
self.assertIsNone(no_such_agent)
# Ensure all agents are being maintained
checked_count = 0
filtered_count = 0
def check_is_worker_1(agent):
nonlocal checked_count # noqa E999 python 3 only
checked_count += 1
self.assertEqual(agent.worker_id, TEST_WORKER_ID_1)
def is_worker_1(agent):
nonlocal filtered_count
filtered_count += 1
return agent.worker_id == TEST_WORKER_ID_1
self.worker_manager.map_over_agents(check_is_worker_1, is_worker_1)
self.assertEqual(checked_count, 2)
self.assertEqual(filtered_count, 3)
# Ensuring _get_worker is accurate
self.assertEqual(
self.worker_manager._get_worker(TEST_WORKER_ID_1), self.worker_state_1
)
self.assertEqual(
self.worker_manager._get_worker(TEST_WORKER_ID_2), self.worker_state_2
)
self.assertEqual(
self.worker_manager._get_worker(TEST_WORKER_ID_3), self.worker_state_3
)
self.assertIsNone(self.worker_manager._get_worker(FAKE_ID))
# Ensuring _get_agent is accurate
self.assertEqual(
self.worker_manager._get_agent(TEST_WORKER_ID_1, TEST_ASSIGNMENT_ID_1),
self.worker_manager.get_agent_for_assignment(TEST_ASSIGNMENT_ID_1),
)
self.assertNotEqual(
self.worker_manager._get_agent(TEST_WORKER_ID_1, TEST_ASSIGNMENT_ID_2),
self.worker_manager.get_agent_for_assignment(TEST_ASSIGNMENT_ID_2),
)
def test_shutdown(self):
"""
Ensure shutdown clears required resources.
"""
self.worker_manager.save_disconnects = mock.MagicMock()
self.worker_manager.un_time_block_workers = mock.MagicMock()
self.worker_manager.shutdown()
self.assertEqual(
len(self.worker_manager.save_disconnects.mock_calls),
1,
'save_disconnects must be called in worker manager shutdown',
)
self.assertEqual(
len(self.worker_manager.un_time_block_workers.mock_calls),
1,
'un_time_block_workers must be called in worker manager shutdown',
)
def test_time_blocks(self):
"""
Check to see if time blocking and clearing works.
"""
self.mturk_manager.soft_block_worker = mock.MagicMock()
self.mturk_manager.un_soft_block_worker = mock.MagicMock()
# No workers blocked, none should be unblocked
self.worker_manager.un_time_block_workers()
self.assertEqual(len(self.mturk_manager.un_soft_block_worker.mock_calls), 0)
# Block some workers, ensure state change and correct calls
self.assertEqual(len(self.worker_manager.time_blocked_workers), 0)
self.worker_manager.time_block_worker(TEST_WORKER_ID_1)
self.mturk_manager.soft_block_worker.assert_called_with(
TEST_WORKER_ID_1, 'max_time_qual'
)
self.assertEqual(len(self.worker_manager.time_blocked_workers), 1)
self.worker_manager.time_block_worker(TEST_WORKER_ID_2)
self.mturk_manager.soft_block_worker.assert_called_with(
TEST_WORKER_ID_2, 'max_time_qual'
)
self.assertEqual(len(self.worker_manager.time_blocked_workers), 2)
self.assertEqual(len(self.mturk_manager.soft_block_worker.mock_calls), 2)
# Unblock a worker passed in as a keyword arg, ensure state remains
self.worker_manager.un_time_block_workers([TEST_WORKER_ID_3])
self.mturk_manager.un_soft_block_worker.assert_called_with(
TEST_WORKER_ID_3, 'max_time_qual'
)
self.assertEqual(len(self.worker_manager.time_blocked_workers), 2)
# Unblock blocked workers, ensure proper calls and state change
self.worker_manager.un_time_block_workers()
self.assertEqual(len(self.worker_manager.time_blocked_workers), 0)
self.mturk_manager.un_soft_block_worker.assert_any_call(
TEST_WORKER_ID_1, 'max_time_qual'
)
self.mturk_manager.un_soft_block_worker.assert_any_call(
TEST_WORKER_ID_2, 'max_time_qual'
)
def test_disconnect_management(self):
self.worker_manager.load_disconnects()
self.worker_manager.is_sandbox = False
self.mturk_manager.block_worker = mock.MagicMock()
self.mturk_manager.soft_block_worker = mock.MagicMock()
self.assertEqual(len(self.worker_manager.disconnects), 0)
# Make one worker disconnect twice
self.worker_manager.handle_bad_disconnect(TEST_WORKER_ID_1)
self.assertEqual(len(self.worker_manager.disconnects), 1)
self.mturk_manager.block_worker.assert_not_called()
self.mturk_manager.soft_block_worker.assert_not_called()
self.worker_manager.handle_bad_disconnect(TEST_WORKER_ID_1)
self.assertEqual(len(self.worker_manager.disconnects), 2)
self.mturk_manager.block_worker.assert_not_called()
self.mturk_manager.soft_block_worker.assert_not_called()
# Ensure both disconnects recorded
self.assertEqual(
self.worker_manager.mturk_workers[TEST_WORKER_ID_1].disconnects, 2
)
# Make second worker disconnect
self.worker_manager.handle_bad_disconnect(TEST_WORKER_ID_2)
self.assertEqual(len(self.worker_manager.disconnects), 3)
self.mturk_manager.block_worker.assert_not_called()
self.mturk_manager.soft_block_worker.assert_not_called()
# Make us soft block workers on disconnect
self.worker_manager.opt['disconnect_qualification'] = 'test'
self.worker_manager.handle_bad_disconnect(TEST_WORKER_ID_1)
self.mturk_manager.block_worker.assert_not_called()
self.mturk_manager.soft_block_worker.assert_called_with(
TEST_WORKER_ID_1, 'disconnect_qualification'
)
self.mturk_manager.soft_block_worker.reset_mock()
# Make us now block workers on disconnect
self.worker_manager.opt['hard_block'] = True
self.worker_manager.handle_bad_disconnect(TEST_WORKER_ID_2)
self.mturk_manager.block_worker.assert_called_once()
self.mturk_manager.soft_block_worker.assert_not_called()
# Ensure we can save and reload disconnects
self.worker_manager.save_disconnects()
# Make a new worker manager
worker_manager2 = WorkerManager(self.mturk_manager, self.opt)
self.assertEqual(len(worker_manager2.disconnects), 5)
self.assertEqual(worker_manager2.mturk_workers[TEST_WORKER_ID_1].disconnects, 3)
self.assertEqual(worker_manager2.mturk_workers[TEST_WORKER_ID_2].disconnects, 2)
worker_manager2.shutdown()
def test_conversation_management(self):
"""
Tests handling conversation state, moving agents to the correct conversations,
and disconnecting one worker in an active convo.
"""
self.worker_manager.assign_task_to_worker(
TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1, TEST_WORKER_ID_1
)
self.worker_manager.assign_task_to_worker(
TEST_HIT_ID_2, TEST_ASSIGNMENT_ID_2, TEST_WORKER_ID_2
)
good_agent = self.worker_manager.get_agent_for_assignment(TEST_ASSIGNMENT_ID_1)
bad_agent = self.worker_manager.get_agent_for_assignment(TEST_ASSIGNMENT_ID_2)
def fake_command_send(worker_id, assignment_id, data, ack_func):
pkt = mock.MagicMock()
pkt.sender_id = worker_id
pkt.assignment_id = assignment_id
self.assertEqual(data['text'], data_model.COMMAND_CHANGE_CONVERSATION)
ack_func(pkt)
self.mturk_manager.send_command = fake_command_send
good_agent.set_status(
AssignState.STATUS_IN_TASK, conversation_id='t1', agent_id='good'
)
bad_agent.set_status(
AssignState.STATUS_IN_TASK, conversation_id='t1', agent_id='bad'
)
self.assertEqual(good_agent.id, 'good')
self.assertEqual(bad_agent.id, 'bad')
self.assertEqual(good_agent.conversation_id, 't1')
self.assertEqual(bad_agent.conversation_id, 't1')
self.assertIn('t1', self.worker_manager.conv_to_agent)
self.assertEqual(len(self.worker_manager.conv_to_agent['t1']), 2)
self.worker_manager.handle_bad_disconnect = mock.MagicMock()
checked_worker = False
def partner_callback(agent):
nonlocal checked_worker
checked_worker = True
self.assertEqual(agent.worker_id, good_agent.worker_id)
self.worker_manager.handle_agent_disconnect(
bad_agent.worker_id, bad_agent.assignment_id, partner_callback
)
self.assertTrue(checked_worker)
self.worker_manager.handle_bad_disconnect.assert_called_once_with(
bad_agent.worker_id
)
self.assertEqual(bad_agent.get_status(), AssignState.STATUS_DISCONNECT)
def setUp(self):
patcher = mock.patch('builtins.input', return_value='y')
self.addCleanup(patcher.stop)
patcher.start()
# Mock functions that hit external APIs and such
self.server_utils = MTurkManagerFile.server_utils
self.mturk_utils = MTurkManagerFile.mturk_utils
self.server_utils.setup_server = mock.MagicMock(
return_value='https://127.0.0.1')
self.server_utils.setup_legacy_server = mock.MagicMock(
return_value='https://127.0.0.1')
self.server_utils.delete_server = mock.MagicMock()
self.mturk_utils.setup_aws_credentials = mock.MagicMock()
self.mturk_utils.calculate_mturk_cost = mock.MagicMock(return_value=1)
self.mturk_utils.check_mturk_balance = mock.MagicMock(
return_value=True)
self.mturk_utils.create_hit_config = mock.MagicMock()
self.mturk_utils.setup_sns_topic = mock.MagicMock(
return_value=TOPIC_ARN)
self.mturk_utils.delete_sns_topic = mock.MagicMock()
self.mturk_utils.delete_qualification = mock.MagicMock()
self.mturk_utils.find_or_create_qualification = mock.MagicMock(
return_value=QUALIFICATION_ID)
self.mturk_utils.find_qualification = mock.MagicMock(
return_value=QUALIFICATION_ID)
self.mturk_utils.give_worker_qualification = mock.MagicMock()
self.mturk_utils.remove_worker_qualification = mock.MagicMock()
self.mturk_utils.create_hit_type = mock.MagicMock(
return_value=HIT_TYPE_ID)
self.mturk_utils.subscribe_to_hits = mock.MagicMock()
self.mturk_utils.create_hit_with_hit_type = mock.MagicMock(
return_value=(MTURK_PAGE_URL, FAKE_HIT_ID, 'MTURK_HIT_DATA'))
self.mturk_utils.get_mturk_client = mock.MagicMock(
return_value=mock.MagicMock())
self.onboarding_agents = {}
self.worlds_agents = {}
# Set up an MTurk Manager and get it ready for accepting workers
self.fake_socket = MockSocket()
time.sleep(0.1)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['frontend_version'] = 1
self.opt['assignment_duration_in_seconds'] = 1
self.opt['hit_title'] = 'test_hit_title'
self.opt['hit_description'] = 'test_hit_description'
self.opt['task_description'] = 'test_task_description'
self.opt['hit_keywords'] = 'test_hit_keywords'
self.opt['reward'] = 0.1
self.opt['is_debug'] = True
self.opt['log_level'] = 0
self.opt['num_conversations'] = 1
self.mturk_agent_ids = ['mturk_agent_1', 'mturk_agent_2']
self.mturk_manager = MTurkManager(opt=self.opt,
mturk_agent_ids=self.mturk_agent_ids,
is_test=True)
self.mturk_manager.port = self.fake_socket.port
self.mturk_manager.setup_server()
self.mturk_manager.start_new_run()
self.mturk_manager.ready_to_accept_workers()
self.mturk_manager.set_get_onboard_world(self.get_onboard_world)
self.mturk_manager.create_hits()
def assign_worker_roles(workers):
workers[0].id = 'mturk_agent_1'
workers[1].id = 'mturk_agent_2'
def run_task_wait():
self.mturk_manager.start_task(lambda w: True, assign_worker_roles,
self.get_task_world)
self.task_thread = threading.Thread(target=run_task_wait)
self.task_thread.start()
self.agent_1 = MockAgent(TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1, TASK_GROUP_ID_1)
self.agent_1_2 = MockAgent(TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_3,
TEST_WORKER_ID_1, TASK_GROUP_ID_1)
self.agent_2 = MockAgent(TEST_HIT_ID_2, TEST_ASSIGNMENT_ID_2,
TEST_WORKER_ID_2, TASK_GROUP_ID_1)
class TestMTurkManagerWorkflows(unittest.TestCase):
"""
Various test cases to replicate a whole mturk workflow.
"""
def setUp(self):
patcher = mock.patch('builtins.input', return_value='y')
self.addCleanup(patcher.stop)
patcher.start()
# Mock functions that hit external APIs and such
self.server_utils = MTurkManagerFile.server_utils
self.mturk_utils = MTurkManagerFile.mturk_utils
self.server_utils.setup_server = mock.MagicMock(
return_value='https://127.0.0.1')
self.server_utils.setup_legacy_server = mock.MagicMock(
return_value='https://127.0.0.1')
self.server_utils.delete_server = mock.MagicMock()
self.mturk_utils.setup_aws_credentials = mock.MagicMock()
self.mturk_utils.calculate_mturk_cost = mock.MagicMock(return_value=1)
self.mturk_utils.check_mturk_balance = mock.MagicMock(
return_value=True)
self.mturk_utils.create_hit_config = mock.MagicMock()
self.mturk_utils.setup_sns_topic = mock.MagicMock(
return_value=TOPIC_ARN)
self.mturk_utils.delete_sns_topic = mock.MagicMock()
self.mturk_utils.delete_qualification = mock.MagicMock()
self.mturk_utils.find_or_create_qualification = mock.MagicMock(
return_value=QUALIFICATION_ID)
self.mturk_utils.find_qualification = mock.MagicMock(
return_value=QUALIFICATION_ID)
self.mturk_utils.give_worker_qualification = mock.MagicMock()
self.mturk_utils.remove_worker_qualification = mock.MagicMock()
self.mturk_utils.create_hit_type = mock.MagicMock(
return_value=HIT_TYPE_ID)
self.mturk_utils.subscribe_to_hits = mock.MagicMock()
self.mturk_utils.create_hit_with_hit_type = mock.MagicMock(
return_value=(MTURK_PAGE_URL, FAKE_HIT_ID, 'MTURK_HIT_DATA'))
self.mturk_utils.get_mturk_client = mock.MagicMock(
return_value=mock.MagicMock())
self.onboarding_agents = {}
self.worlds_agents = {}
# Set up an MTurk Manager and get it ready for accepting workers
self.fake_socket = MockSocket()
time.sleep(0.1)
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args()
self.opt['task'] = 'unittest'
self.opt['frontend_version'] = 1
self.opt['assignment_duration_in_seconds'] = 1
self.opt['hit_title'] = 'test_hit_title'
self.opt['hit_description'] = 'test_hit_description'
self.opt['task_description'] = 'test_task_description'
self.opt['hit_keywords'] = 'test_hit_keywords'
self.opt['reward'] = 0.1
self.opt['is_debug'] = True
self.opt['log_level'] = 0
self.opt['num_conversations'] = 1
self.mturk_agent_ids = ['mturk_agent_1', 'mturk_agent_2']
self.mturk_manager = MTurkManager(opt=self.opt,
mturk_agent_ids=self.mturk_agent_ids,
is_test=True)
self.mturk_manager.port = self.fake_socket.port
self.mturk_manager.setup_server()
self.mturk_manager.start_new_run()
self.mturk_manager.ready_to_accept_workers()
self.mturk_manager.set_get_onboard_world(self.get_onboard_world)
self.mturk_manager.create_hits()
def assign_worker_roles(workers):
workers[0].id = 'mturk_agent_1'
workers[1].id = 'mturk_agent_2'
def run_task_wait():
self.mturk_manager.start_task(lambda w: True, assign_worker_roles,
self.get_task_world)
self.task_thread = threading.Thread(target=run_task_wait)
self.task_thread.start()
self.agent_1 = MockAgent(TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1, TASK_GROUP_ID_1)
self.agent_1_2 = MockAgent(TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_3,
TEST_WORKER_ID_1, TASK_GROUP_ID_1)
self.agent_2 = MockAgent(TEST_HIT_ID_2, TEST_ASSIGNMENT_ID_2,
TEST_WORKER_ID_2, TASK_GROUP_ID_1)
def tearDown(self):
for key in self.worlds_agents.keys():
self.worlds_agents[key] = True
self.mturk_manager.shutdown()
self.fake_socket.close()
if self.task_thread.isAlive():
self.task_thread.join()
def get_onboard_world(self, mturk_agent):
self.onboarding_agents[mturk_agent.worker_id] = False
def episode_done():
return not (
(mturk_agent.worker_id in self.onboarding_agents) and
(self.onboarding_agents[mturk_agent.worker_id] is False))
return TestMTurkOnboardWorld(mturk_agent, episode_done)
def get_task_world(self, mturk_manager, opt, workers):
for worker in workers:
self.worlds_agents[worker.worker_id] = False
def episode_done():
for worker in workers:
if self.worlds_agents[worker.worker_id] is False:
return False
return True
return TestMTurkWorld(workers, episode_done)
def alive_agent(self, agent):
agent.register_to_socket(self.fake_socket)
agent.send_alive()
time.sleep(0.3)
def test_successful_convo(self):
manager = self.mturk_manager
# Alive first agent
agent_1 = self.agent_1
self.alive_agent(agent_1)
assert_equal_by(lambda: agent_1.worker_id in self.onboarding_agents,
True, 2)
agent_1_object = manager.worker_manager.get_agent_for_assignment(
agent_1.assignment_id)
self.assertFalse(self.onboarding_agents[agent_1.worker_id])
self.assertEqual(agent_1_object.get_status(),
AssignState.STATUS_ONBOARDING)
self.onboarding_agents[agent_1.worker_id] = True
assert_equal_by(agent_1_object.get_status, AssignState.STATUS_WAITING,
2)
# Alive second agent
agent_2 = self.agent_2
self.alive_agent(agent_2)
assert_equal_by(lambda: agent_2.worker_id in self.onboarding_agents,
True, 2)
agent_2_object = manager.worker_manager.get_agent_for_assignment(
agent_2.assignment_id)
self.assertFalse(self.onboarding_agents[agent_2.worker_id])
self.assertEqual(agent_2_object.get_status(),
AssignState.STATUS_ONBOARDING)
self.onboarding_agents[agent_2.worker_id] = True
# Assert agents move to task
assert_equal_by(agent_2_object.get_status, AssignState.STATUS_IN_TASK,
2)
assert_equal_by(lambda: agent_2.worker_id in self.worlds_agents, True,
2)
self.assertIn(agent_1.worker_id, self.worlds_agents)
# Complete agents
self.worlds_agents[agent_1.worker_id] = True
self.worlds_agents[agent_2.worker_id] = True
agent_1_object.set_completed_act({})
agent_2_object.set_completed_act({})
assert_equal_by(agent_1_object.get_status, AssignState.STATUS_DONE, 2)
assert_equal_by(agent_2_object.get_status, AssignState.STATUS_DONE, 2)
# Assert conversation is complete for manager and agents
assert_equal_by(lambda: manager.completed_conversations, 1, 2)
def test_disconnect_end(self):
manager = self.mturk_manager
# Alive first agent
agent_1 = self.agent_1
self.alive_agent(agent_1)
assert_equal_by(lambda: agent_1.worker_id in self.onboarding_agents,
True, 2)
agent_1_object = manager.worker_manager.get_agent_for_assignment(
agent_1.assignment_id)
self.assertFalse(self.onboarding_agents[agent_1.worker_id])
self.assertEqual(agent_1_object.get_status(),
AssignState.STATUS_ONBOARDING)
self.onboarding_agents[agent_1.worker_id] = True
assert_equal_by(agent_1_object.get_status, AssignState.STATUS_WAITING,
2)
# Alive second agent
agent_2 = self.agent_2
self.alive_agent(agent_2)
assert_equal_by(lambda: agent_2.worker_id in self.onboarding_agents,
True, 2)
agent_2_object = manager.worker_manager.get_agent_for_assignment(
agent_2.assignment_id)
self.assertFalse(self.onboarding_agents[agent_2.worker_id])
self.assertEqual(agent_2_object.get_status(),
AssignState.STATUS_ONBOARDING)
self.onboarding_agents[agent_2.worker_id] = True
# Assert agents move to task
assert_equal_by(agent_2_object.get_status, AssignState.STATUS_IN_TASK,
2)
assert_equal_by(lambda: agent_2.worker_id in self.worlds_agents, True,
2)
self.assertIn(agent_1.worker_id, self.worlds_agents)
# Disconnect agent
agent_2.send_disconnect()
assert_equal_by(agent_1_object.get_status,
AssignState.STATUS_PARTNER_DISCONNECT, 3)
assert_equal_by(agent_2_object.get_status,
AssignState.STATUS_DISCONNECT, 3)
self.worlds_agents[agent_1.worker_id] = True
self.worlds_agents[agent_2.worker_id] = True
# assert conversation not marked as complete
self.assertEqual(manager.completed_conversations, 0)
agent_1_object.set_completed_act({})
def test_expire_onboarding(self):
manager = self.mturk_manager
# Alive first agent
agent_1 = self.agent_1
self.alive_agent(agent_1)
assert_equal_by(lambda: agent_1.worker_id in self.onboarding_agents,
True, 10)
agent_1_object = manager.worker_manager.get_agent_for_assignment(
agent_1.assignment_id)
self.assertFalse(self.onboarding_agents[agent_1.worker_id])
self.assertEqual(agent_1_object.get_status(),
AssignState.STATUS_ONBOARDING)
manager._expire_onboarding_pool()
self.onboarding_agents[agent_1.worker_id] = True
self.assertEqual(agent_1_object.get_status(),
AssignState.STATUS_EXPIRED)
def test_attempt_break_unique(self):
manager = self.mturk_manager
unique_worker_qual = 'is_unique_qual'
manager.is_unique = True
manager.opt['unique_qual_name'] = unique_worker_qual
manager.unique_qual_name = unique_worker_qual
# Alive first agent
agent_1 = self.agent_1
self.alive_agent(agent_1)
assert_equal_by(lambda: agent_1.worker_id in self.onboarding_agents,
True, 2)
agent_1_object = manager.worker_manager.get_agent_for_assignment(
agent_1.assignment_id)
self.assertFalse(self.onboarding_agents[agent_1.worker_id])
self.assertEqual(agent_1_object.get_status(),
AssignState.STATUS_ONBOARDING)
self.onboarding_agents[agent_1.worker_id] = True
assert_equal_by(agent_1_object.get_status, AssignState.STATUS_WAITING,
2)
# Alive second agent
agent_2 = self.agent_2
self.alive_agent(agent_2)
assert_equal_by(lambda: agent_2.worker_id in self.onboarding_agents,
True, 2)
agent_2_object = manager.worker_manager.get_agent_for_assignment(
agent_2.assignment_id)
self.assertFalse(self.onboarding_agents[agent_2.worker_id])
self.assertEqual(agent_2_object.get_status(),
AssignState.STATUS_ONBOARDING)
self.onboarding_agents[agent_2.worker_id] = True
# Assert agents move to task
assert_equal_by(agent_2_object.get_status, AssignState.STATUS_IN_TASK,
2)
assert_equal_by(lambda: agent_2.worker_id in self.worlds_agents, True,
2)
self.assertIn(agent_1.worker_id, self.worlds_agents)
# Complete agents
self.worlds_agents[agent_1.worker_id] = True
self.worlds_agents[agent_2.worker_id] = True
agent_1_object.set_completed_act({})
agent_2_object.set_completed_act({})
assert_equal_by(agent_1_object.get_status, AssignState.STATUS_DONE, 2)
assert_equal_by(agent_2_object.get_status, AssignState.STATUS_DONE, 2)
# Assert conversation is complete for manager and agents
assert_equal_by(lambda: manager.completed_conversations, 1, 2)
# ensure qualification was 'granted'
self.mturk_utils.find_qualification.assert_called_with(
unique_worker_qual, manager.is_sandbox)
self.mturk_utils.give_worker_qualification.assert_any_call(
agent_1.worker_id, QUALIFICATION_ID, None, manager.is_sandbox)
self.mturk_utils.give_worker_qualification.assert_any_call(
agent_2.worker_id, QUALIFICATION_ID, None, manager.is_sandbox)
# Try to alive with the first agent a second time
agent_1_2 = self.agent_1_2
self.alive_agent(agent_1_2)
assert_equal_by(lambda: agent_1_2.worker_id in self.onboarding_agents,
True, 2)
agent_1_2_object = manager.worker_manager.get_agent_for_assignment(
agent_1_2.assignment_id)
# No worker should be created for a unique task
self.assertIsNone(agent_1_2_object)
def test_break_multi_convo(self):
manager = self.mturk_manager
manager.opt['allowed_conversations'] = 1
# Alive first agent
agent_1 = self.agent_1
self.alive_agent(agent_1)
assert_equal_by(lambda: agent_1.worker_id in self.onboarding_agents,
True, 2)
agent_1_object = manager.worker_manager.get_agent_for_assignment(
agent_1.assignment_id)
self.assertFalse(self.onboarding_agents[agent_1.worker_id])
self.assertEqual(agent_1_object.get_status(),
AssignState.STATUS_ONBOARDING)
self.onboarding_agents[agent_1.worker_id] = True
assert_equal_by(agent_1_object.get_status, AssignState.STATUS_WAITING,
2)
# Alive second agent
agent_2 = self.agent_2
self.alive_agent(agent_2)
assert_equal_by(lambda: agent_2.worker_id in self.onboarding_agents,
True, 2)
agent_2_object = manager.worker_manager.get_agent_for_assignment(
agent_2.assignment_id)
self.assertFalse(self.onboarding_agents[agent_2.worker_id])
self.assertEqual(agent_2_object.get_status(),
AssignState.STATUS_ONBOARDING)
self.onboarding_agents[agent_2.worker_id] = True
# Assert agents move to task
assert_equal_by(agent_2_object.get_status, AssignState.STATUS_IN_TASK,
2)
assert_equal_by(lambda: agent_2.worker_id in self.worlds_agents, True,
2)
self.assertIn(agent_1.worker_id, self.worlds_agents)
# Attempt to start a new conversation with duplicate worker 1
agent_1_2 = self.agent_1_2
self.alive_agent(agent_1_2)
assert_equal_by(lambda: agent_1_2.worker_id in self.onboarding_agents,
True, 2)
agent_1_2_object = manager.worker_manager.get_agent_for_assignment(
agent_1_2.assignment_id)
# No worker should be created for a unique task
self.assertIsNone(agent_1_2_object)
# Complete agents
self.worlds_agents[agent_1.worker_id] = True
self.worlds_agents[agent_2.worker_id] = True
agent_1_object.set_completed_act({})
agent_2_object.set_completed_act({})
assert_equal_by(agent_1_object.get_status, AssignState.STATUS_DONE, 2)
assert_equal_by(agent_2_object.get_status, AssignState.STATUS_DONE, 2)
assert_equal_by(lambda: manager.completed_conversations, 1, 2)
def test_no_onboard_expire_waiting(self):
manager = self.mturk_manager
manager.set_get_onboard_world(None)
# Alive first agent
agent_1 = self.agent_1
self.alive_agent(agent_1)
agent_1_object = manager.worker_manager.get_agent_for_assignment(
agent_1.assignment_id)
assert_equal_by(agent_1_object.get_status, AssignState.STATUS_WAITING,
2)
manager._expire_agent_pool()
self.assertEqual(agent_1_object.get_status(),
AssignState.STATUS_EXPIRED)