def run_hit(hit_index, assignment_index, opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
# Create mturk agents
mturk_agent_1 = MTurkAgent(id=mturk_agent_1_id,
manager=mturk_manager,
conversation_id=conversation_id,
opt=opt)
mturk_agent_2 = MTurkAgent(id=mturk_agent_2_id,
manager=mturk_manager,
conversation_id=conversation_id,
opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = human_agent_2_id
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
while not world.episode_done():
world.parley()
world.shutdown()
def test_worker_state_agent_management(self):
"""
Test public state management methods of worker_state.
"""
agent_1 = MTurkAgent(
self.opt,
self.mturk_manager,
TEST_HIT_ID_1,
TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1,
)
agent_2 = MTurkAgent(
self.opt,
self.mturk_manager,
TEST_HIT_ID_2,
TEST_ASSIGNMENT_ID_2,
TEST_WORKER_ID_1,
)
agent_3 = MTurkAgent(
self.opt,
self.mturk_manager,
TEST_HIT_ID_3,
TEST_ASSIGNMENT_ID_3,
TEST_WORKER_ID_3,
)
self.assertEqual(self.work_state_1.active_conversation_count(), 0)
self.work_state_1.add_agent(agent_1)
self.assertEqual(self.work_state_1.active_conversation_count(), 1)
self.work_state_1.add_agent(agent_2)
self.assertEqual(self.work_state_1.active_conversation_count(), 2)
with self.assertRaises(AssertionError):
self.work_state_1.add_agent(agent_3)
self.assertEqual(self.work_state_1.active_conversation_count(), 2)
self.assertEqual(self.work_state_1.completed_assignments(), 0)
self.assertTrue(self.work_state_1.has_assignment(
agent_1.assignment_id))
self.assertTrue(self.work_state_1.has_assignment(
agent_2.assignment_id))
self.assertFalse(
self.work_state_1.has_assignment(agent_3.assignment_id))
self.assertEqual(
agent_1,
self.work_state_1.get_agent_for_assignment(agent_1.assignment_id))
self.assertEqual(
agent_2,
self.work_state_1.get_agent_for_assignment(agent_2.assignment_id))
self.assertIsNone(
self.work_state_1.get_agent_for_assignment(agent_3.assignment_id))
agent_1.set_status(AssignState.STATUS_DONE)
self.assertEqual(self.work_state_1.active_conversation_count(), 1)
self.assertEqual(self.work_state_1.completed_assignments(), 1)
agent_2.set_status(AssignState.STATUS_DISCONNECT)
self.assertEqual(self.work_state_1.active_conversation_count(), 0)
self.assertEqual(self.work_state_1.completed_assignments(), 1)
def setUp(self):
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args(print_args=False)
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt.copy(),
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
self.turk_agent = MTurkAgent(self.opt.copy(), self.mturk_manager,
TEST_HIT_ID_1, TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1)
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
# Initialize a SQuAD teacher agent, which we will later get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
task = task_class(task_opt)
# Create the MTurk agent which provides a chat interface to the Turker
opt.update(task_config)
mturk_agent_id = 'Worker'
opt['agent_id'] = mturk_agent_id
opt['mturk_agent_ids'] = [mturk_agent_id]
opt['all_agent_ids'] = [
QADataCollectionWorld.collector_agent_id, mturk_agent_id
]
opt['conversation_id'] = str(int(time.time()))
mturk_agent = MTurkAgent(opt=opt)
world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def main():
global run_hit
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
opt['mturk_agent_1_id'] = 'mturk_agent_1'
opt['mturk_agent_2_id'] = 'mturk_agent_2'
opt['human_agent_1_id'] = 'human_1'
opt['human_agent_2_id'] = 'human_2'
opt.update(task_config)
opt['mturk_agent_ids'] = [opt['mturk_agent_1_id'], opt['mturk_agent_2_id']]
opt['all_agent_ids'] = [opt['human_agent_1_id'], opt['human_agent_2_id']
] + opt['mturk_agent_ids']
opt['run_id'] = str(int(time.time()))
def run_hit(i, opt):
opt['conversation_id'] = str(i)
opt['agent_id'] = opt['mturk_agent_1_id']
mturk_agent_1 = MTurkAgent(opt=opt)
opt['agent_id'] = opt['mturk_agent_2_id']
mturk_agent_2 = MTurkAgent(opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = opt['human_agent_1_id']
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = opt['human_agent_2_id']
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
while not world.episode_done():
world.parley()
world.shutdown()
MTurkAgent.init_aws(opt)
results = Parallel(n_jobs=opt['num_hits'], backend='threading')(
delayed(run_hit)(i, copy.deepcopy(opt))
for i in range(1, opt['num_hits'] + 1))
MTurkAgent.review_hits(opt=opt)
def _create_agent(self, hit_id, assignment_id, worker_id):
"""Initialize an agent and add it to the map"""
agent = MTurkAgent(self.opt, self, hit_id, assignment_id, worker_id)
if (worker_id in self.mturk_agents):
self.mturk_agents[worker_id][assignment_id] = agent
else:
self.mturk_agents[worker_id] = {}
self.mturk_agents[worker_id][assignment_id] = agent
def run_hit(i, task_class, task_opt, opt, mturk_manager):
task = task_class(task_opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id='Worker', manager=mturk_manager, conversation_id=i, opt=opt)
world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def main():
global run_hit
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
# Initialize a SQuAD teacher agent, which we will later get context from
module_name = 'parlai.tasks.squad.agents'
class_name = 'DefaultTeacher'
my_module = importlib.import_module(module_name)
task_class = getattr(my_module, class_name)
task_opt = {}
task_opt['datatype'] = 'train'
task_opt['datapath'] = opt['datapath']
# Create the MTurk agent which provides a chat interface to the Turker
opt.update(task_config)
mturk_agent_id = 'Worker'
opt['agent_id'] = mturk_agent_id
opt['mturk_agent_ids'] = [mturk_agent_id]
opt['all_agent_ids'] = [
QADataCollectionWorld.collector_agent_id, mturk_agent_id
]
opt['run_id'] = str(int(time.time()))
def run_hit(i, task_class, task_opt, opt):
task = task_class(task_opt)
opt['conversation_id'] = str(i)
mturk_agent = MTurkAgent(opt=opt)
world = QADataCollectionWorld(opt=opt,
task=task,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
MTurkAgent.init_aws(opt)
results = Parallel(n_jobs=opt['num_hits'], backend='threading')(
delayed(run_hit)(i, task_class, task_opt, copy.deepcopy(opt))
for i in range(1, opt['num_hits'] + 1))
MTurkAgent.review_hits(opt=opt)
def run_hit(hit_index, assignment_index, task_class, task_opt, opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
task = task_class(task_opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = QADataCollectionWorld(opt=opt, task=task, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def run_hit(i, task_class, task_opt, opt):
task = task_class(task_opt)
opt['conversation_id'] = str(i)
mturk_agent = MTurkAgent(opt=opt)
world = QADataCollectionWorld(opt=opt,
task=task,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def run_hit(hit_index, assignment_index, opt, task_opt, mturk_manager):
conversation_id = str(hit_index) + '_' + str(assignment_index)
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id=mturk_agent_id, manager=mturk_manager, conversation_id=conversation_id, opt=opt)
world = ModelEvaluatorWorld(opt=opt, model_agent=model_agent, task_opt=task_opt, mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
mturk_agent_1_id = 'mturk_agent_1'
mturk_agent_2_id = 'mturk_agent_2'
human_agent_1_id = 'human_1'
human_agent_2_id = 'human_2'
# Create the MTurk agents
opt.update(task_config)
opt['conversation_id'] = str(int(time.time()))
opt['mturk_agent_ids'] = [mturk_agent_1_id, mturk_agent_2_id]
opt['all_agent_ids'] = [
human_agent_1_id, human_agent_2_id, mturk_agent_1_id, mturk_agent_2_id
]
opt['agent_id'] = mturk_agent_1_id
mturk_agent_1 = MTurkAgent(opt=opt)
opt['agent_id'] = mturk_agent_2_id
mturk_agent_2 = MTurkAgent(opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = human_agent_1_id
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = human_agent_2_id
world = MultiAgentDialogWorld(
opt=opt,
agents=[human_agent_1, human_agent_2, mturk_agent_1, mturk_agent_2])
while not world.episode_done():
world.parley()
world.shutdown()
def run_hit(i, opt):
opt['conversation_id'] = str(i)
opt['agent_id'] = opt['mturk_agent_1_id']
mturk_agent_1 = MTurkAgent(opt=opt)
opt['agent_id'] = opt['mturk_agent_2_id']
mturk_agent_2 = MTurkAgent(opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = opt['human_agent_1_id']
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = opt['human_agent_2_id']
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
while not world.episode_done():
world.parley()
world.shutdown()
def run_hit(i, opt, task_opt, mturk_manager):
model_agent = IrBaselineAgent(opt=opt)
# Create the MTurk agent which provides a chat interface to the Turker
mturk_agent = MTurkAgent(id='Worker',
manager=mturk_manager,
conversation_id=i,
opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def main():
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
# The dialog model we want to evaluate
from parlai.agents.ir_baseline.ir_baseline import IrBaselineAgent
IrBaselineAgent.add_cmdline_args(argparser)
opt = argparser.parse_args()
opt['task'] = os.path.basename(os.getcwd())
model_agent = IrBaselineAgent(opt=opt)
# The task that we will evaluate the dialog model on
task_opt = {}
task_opt['datatype'] = 'test'
task_opt['datapath'] = opt['datapath']
task_opt['task'] = '#MovieDD-Reddit'
# Create the MTurk agent which provides a chat interface to the Turker
opt.update(task_config)
mturk_agent_id = 'Worker'
opt['agent_id'] = mturk_agent_id
opt['mturk_agent_ids'] = [mturk_agent_id]
opt['all_agent_ids'] = [
ModelEvaluatorWorld.evaluator_agent_id, mturk_agent_id
]
opt['conversation_id'] = str(int(time.time()))
mturk_agent = MTurkAgent(opt=opt)
world = ModelEvaluatorWorld(opt=opt,
model_agent=model_agent,
task_opt=task_opt,
mturk_agent=mturk_agent)
while not world.episode_done():
world.parley()
world.shutdown()
def _create_agent(self, hit_id, assignment_id, worker_id):
"""Initialize an agent and return it"""
return MTurkAgent(self.opt, self.mturk_manager, hit_id, assignment_id,
worker_id)
def run_hit(i, opt, mturk_manager):
# Create mturk agents
mturk_agent_1 = MTurkAgent(id='mturk_agent_1',
manager=mturk_manager,
conversation_id=i,
opt=opt)
mturk_agent_2 = MTurkAgent(id='mturk_agent_2',
manager=mturk_manager,
conversation_id=i,
opt=opt)
# Create the local human agents
human_agent_1 = LocalHumanAgent(opt=None)
human_agent_1.id = 'human_1'
human_agent_2 = LocalHumanAgent(opt=None)
human_agent_2.id = 'human_2'
world = MultiAgentDialogWorld(opt=opt,
agents=[
human_agent_1, human_agent_2,
mturk_agent_1, mturk_agent_2
])
# Since we are using the regular MultiAgentDialogWorld, we do the following outside of the world instead.
mturk_agent_ids = [mturk_agent_1.id, mturk_agent_2.id]
all_agent_ids = [human_agent_1.id, human_agent_2.id] + mturk_agent_ids
mturk_agent_1.mturk_agent_ids = mturk_agent_ids
mturk_agent_1.all_agent_ids = all_agent_ids
mturk_agent_2.mturk_agent_ids = mturk_agent_ids
mturk_agent_2.all_agent_ids = all_agent_ids
mturk_agent_1.create_hit()
mturk_agent_2.create_hit()
while not world.episode_done():
world.parley()
world.shutdown()
class TestMTurkAgent(unittest.TestCase):
"""
Various unit tests for the MTurkAgent class.
"""
def setUp(self):
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
self.opt = argparser.parse_args(print_args=False)
self.opt['task'] = 'unittest'
self.opt['assignment_duration_in_seconds'] = 6
mturk_agent_ids = ['mturk_agent_1']
self.mturk_manager = MTurkManager(opt=self.opt.copy(),
mturk_agent_ids=mturk_agent_ids)
self.worker_manager = self.mturk_manager.worker_manager
self.turk_agent = MTurkAgent(
self.opt.copy(),
self.mturk_manager,
TEST_HIT_ID_1,
TEST_ASSIGNMENT_ID_1,
TEST_WORKER_ID_1,
)
def tearDown(self):
self.mturk_manager.shutdown()
disconnect_path = os.path.join(parent_dir, 'disconnect-test.pickle')
if os.path.exists(disconnect_path):
os.remove(disconnect_path)
def test_init(self):
"""
Test initialization of an agent.
"""
self.assertIsNotNone(self.turk_agent.creation_time)
self.assertIsNone(self.turk_agent.id)
self.assertIsNone(self.turk_agent.message_request_time)
self.assertIsNone(self.turk_agent.conversation_id)
self.assertFalse(self.turk_agent.some_agent_disconnected)
self.assertFalse(self.turk_agent.hit_is_expired)
self.assertFalse(self.turk_agent.hit_is_abandoned)
self.assertFalse(self.turk_agent.hit_is_returned)
self.assertFalse(self.turk_agent.hit_is_complete)
self.assertFalse(self.turk_agent.disconnected)
self.assertTrue(self.turk_agent.alived)
def test_state_wrappers(self):
"""
Test the mturk agent wrappers around its state.
"""
for status in statuses:
self.turk_agent.set_status(status)
self.assertEqual(self.turk_agent.get_status(), status)
for status in [
AssignState.STATUS_DONE, AssignState.STATUS_PARTNER_DISCONNECT
]:
self.turk_agent.set_status(status)
self.assertTrue(self.turk_agent.submitted_hit())
for status in active_statuses:
self.turk_agent.set_status(status)
self.assertFalse(self.turk_agent.is_final())
for status in complete_statuses:
self.turk_agent.set_status(status)
self.assertTrue(self.turk_agent.is_final())
self.turk_agent.append_message(MESSAGE_1)
self.assertEqual(len(self.turk_agent.get_messages()), 1)
self.turk_agent.append_message(MESSAGE_2)
self.assertEqual(len(self.turk_agent.get_messages()), 2)
self.turk_agent.append_message(MESSAGE_1)
self.assertEqual(len(self.turk_agent.get_messages()), 2)
self.assertIn(MESSAGE_1, self.turk_agent.get_messages())
self.assertIn(MESSAGE_2, self.turk_agent.get_messages())
# Ensure command interactions work as expected
self.turk_agent.set_last_command(COMMAND_1)
self.assertEqual(self.turk_agent.get_last_command(), COMMAND_1)
self.turk_agent.clear_messages()
self.assertEqual(len(self.turk_agent.get_messages()), 0)
# In task checks
self.turk_agent.conversation_id = 't_12345'
self.assertTrue(self.turk_agent.is_in_task())
self.turk_agent.conversation_id = 'b_12345'
self.assertFalse(self.turk_agent.is_in_task())
def test_connection_id(self):
"""
Ensure the connection_id hasn't changed.
"""
connection_id = "{}_{}".format(self.turk_agent.worker_id,
self.turk_agent.assignment_id)
self.assertEqual(self.turk_agent.get_connection_id(), connection_id)
def test_inactive_data(self):
"""
Ensure data packet generated for inactive commands is valid.
"""
for status in complete_statuses:
self.turk_agent.set_status(status)
data = self.turk_agent.get_inactive_command_data()
self.assertIsNotNone(data['text'])
self.assertIsNotNone(data['inactive_text'])
self.assertEqual(data['conversation_id'],
self.turk_agent.conversation_id)
self.assertEqual(data['agent_id'], TEST_WORKER_ID_1)
def test_status_change(self):
has_changed = False
self.turk_agent.set_status(AssignState.STATUS_ONBOARDING)
def wait_for_status_wrap():
nonlocal has_changed # noqa 999 we don't use python2
self.turk_agent.wait_for_status(AssignState.STATUS_WAITING)
has_changed = True
t = threading.Thread(target=wait_for_status_wrap, daemon=True)
t.start()
self.assertFalse(has_changed)
time.sleep(0.07)
self.assertFalse(has_changed)
self.turk_agent.set_status(AssignState.STATUS_WAITING)
time.sleep(0.07)
self.assertTrue(has_changed)
def test_message_queue(self):
"""
Ensure observations and acts work as expected.
"""
self.mturk_manager.send_message = mock.MagicMock()
self.turk_agent.observe(ACT_1)
self.mturk_manager.send_message.assert_called_with(
TEST_WORKER_ID_1, TEST_ASSIGNMENT_ID_1, ACT_1)
# First act comes through the queue and returns properly
self.assertTrue(self.turk_agent.msg_queue.empty())
self.turk_agent.id = AGENT_ID
self.turk_agent.put_data(MESSAGE_ID_1, ACT_1)
self.assertTrue(self.turk_agent.recieved_packets[MESSAGE_ID_1])
self.assertFalse(self.turk_agent.msg_queue.empty())
returned_act = self.turk_agent.get_new_act_message()
self.assertEqual(returned_act, ACT_1)
# Repeat act is ignored
self.turk_agent.put_data(MESSAGE_ID_1, ACT_1)
self.assertTrue(self.turk_agent.msg_queue.empty())
for i in range(100):
self.turk_agent.put_data(str(i), ACT_1)
self.assertEqual(self.turk_agent.msg_queue.qsize(), 100)
self.turk_agent.flush_msg_queue()
self.assertTrue(self.turk_agent.msg_queue.empty())
# Test non-act messages
blank_message = self.turk_agent.get_new_act_message()
self.assertIsNone(blank_message)
self.turk_agent.disconnected = True
disconnect_message = self.turk_agent.get_new_act_message()
self.turk_agent.disconnected = False
self.assertEqual(disconnect_message['text'],
self.turk_agent.MTURK_DISCONNECT_MESSAGE)
self.turk_agent.hit_is_returned = True
return_message = self.turk_agent.get_new_act_message()
self.assertEqual(return_message['text'],
self.turk_agent.RETURN_MESSAGE)
self.turk_agent.hit_is_returned = False
# Reduce state
self.turk_agent.reduce_state()
self.assertIsNone(self.turk_agent.msg_queue)
self.assertIsNone(self.turk_agent.recieved_packets)
def test_message_acts(self):
self.mturk_manager.send_command = mock.MagicMock()
self.mturk_manager.handle_turker_timeout = mock.MagicMock()
# non-Blocking check
self.assertIsNone(self.turk_agent.message_request_time)
returned_act = self.turk_agent.act(blocking=False)
self.assertIsNotNone(self.turk_agent.message_request_time)
self.assertIsNone(returned_act)
self.turk_agent.id = AGENT_ID
self.turk_agent.put_data(MESSAGE_ID_1, ACT_1)
returned_act = self.turk_agent.act(blocking=False)
self.assertIsNone(self.turk_agent.message_request_time)
self.assertEqual(returned_act, ACT_1)
self.mturk_manager.send_command.assert_called_once()
# non-Blocking timeout check
self.mturk_manager.send_command = mock.MagicMock()
returned_act = self.turk_agent.act(timeout=0.07, blocking=False)
self.assertIsNotNone(self.turk_agent.message_request_time)
self.assertIsNone(returned_act)
while returned_act is None:
returned_act = self.turk_agent.act(timeout=0.07, blocking=False)
self.mturk_manager.send_command.assert_called_once()
self.mturk_manager.handle_turker_timeout.assert_called_once()
self.assertEqual(returned_act['text'], self.turk_agent.TIMEOUT_MESSAGE)
# Blocking timeout check
self.mturk_manager.send_command = mock.MagicMock()
self.mturk_manager.handle_turker_timeout = mock.MagicMock()
returned_act = self.turk_agent.act(timeout=0.07)
self.mturk_manager.send_command.assert_called_once()
self.mturk_manager.handle_turker_timeout.assert_called_once()
self.assertEqual(returned_act['text'], self.turk_agent.TIMEOUT_MESSAGE)
