def test_interactive_mode(self): """Test if conversation history is destroyed in MTurk mode.""" # both manually setting bs to 1 and interactive mode true agent = get_agent(batchsize=1, interactive_mode=True) agent.observe(Message({'text': 'foo', 'episode_done': True})) response = agent.act() self.assertIn('Evaluating 0', response['text'], 'Incorrect output in single act()') shared = create_agent_from_shared(agent.share()) shared.observe(Message({'text': 'bar', 'episode_done': True})) response = shared.act() self.assertIn('Evaluating 0', response['text'], 'Incorrect output in single act()') # now just bs 1 agent = get_agent(batchsize=1, interactive_mode=False) agent.observe(Message({'text': 'foo', 'episode_done': True})) response = agent.act() self.assertIn('Evaluating 0', response['text'], 'Incorrect output in single act()') shared = create_agent_from_shared(agent.share()) shared.observe(Message({'text': 'bar', 'episode_done': True})) response = shared.act() self.assertIn('Evaluating 0', response['text'], 'Incorrect output in single act()') # now just interactive shared = create_agent_from_shared(agent.share()) agent.observe(Message({'text': 'foo', 'episode_done': True})) response = agent.act() self.assertIn('Evaluating 0', response['text'], 'Incorrect output in single act()') shared = create_agent_from_shared(agent.share()) shared.observe(Message({'text': 'bar', 'episode_done': True})) response = shared.act() self.assertIn('Evaluating 0', response['text'], 'Incorrect output in single act()') # finally, the expected failure with self.assertRaises(RuntimeError): agent = get_agent(batchsize=16, interactive_mode=False) agent.observe(Message({'text': 'foo', 'episode_done': True})) response = agent.act() shared = create_agent_from_shared(agent.share()) shared.observe(Message({'text': 'bar', 'episode_done': True})) response = shared.act()
def test_batch_act(self): """Make sure batch act calls the right step.""" agent = get_agent() obs_labs = [ Message({ 'text': "It's only a flesh wound.", 'labels': ['Yield!'], 'episode_done': True, }), Message({ 'text': 'The needs of the many outweigh...', 'labels': ['The needs of the few.'], 'episode_done': True, }), Message({ 'text': 'Hello there.', 'labels': ['General Kenobi.'], 'episode_done': True, }), ] obs_labs_vecs = [] for o in obs_labs: agent.history.reset() agent.history.update_history(o) obs_labs_vecs.append(agent.vectorize(o, agent.history)) reply = agent.batch_act(obs_labs_vecs) for i in range(len(obs_labs_vecs)): self.assertEqual(reply[i]['text'], 'Training {}!'.format(i)) obs_elabs = [ Message({ 'text': "It's only a flesh wound.", 'eval_labels': ['Yield!'], 'episode_done': True, }), Message({ 'text': 'The needs of the many outweigh...', 'eval_labels': ['The needs of the few.'], 'episode_done': True, }), Message({ 'text': 'Hello there.', 'eval_labels': ['General Kenobi.'], 'episode_done': True, }), ] obs_elabs_vecs = [] for o in obs_elabs: agent.history.reset() agent.history.update_history(o) obs_elabs_vecs.append(agent.vectorize(o, agent.history)) reply = agent.batch_act(obs_elabs_vecs) for i in range(len(obs_elabs_vecs)): self.assertIn('Evaluating {}'.format(i), reply[i]['text'])
def test_mturk_racehistory(self): """Emulate a setting where batch_act misappropriately handles mturk.""" agent = get_agent(batchsize=16, interactive_mode=True, echo=True) share1 = create_agent_from_shared(agent.share()) share1.observe(Message({ 'text': 'thread1-msg1', 'episode_done': False })) share2 = create_agent_from_shared(agent.share()) share2.observe(Message({ 'text': 'thread2-msg1', 'episode_done': False })) share1.act() share2.act() share1.observe(Message({ 'text': 'thread1-msg2', 'episode_done': False })) share2.observe(Message({ 'text': 'thread2-msg2', 'episode_done': False })) share2.act() share1.act() share2.observe(Message({ 'text': 'thread2-msg3', 'episode_done': False })) share1.observe(Message({ 'text': 'thread1-msg3', 'episode_done': False })) self.assertNotIn('thread1-msg1', share2.history.get_history_str()) self.assertNotIn('thread2-msg1', share1.history.get_history_str()) self.assertNotIn('thread1-msg2', share2.history.get_history_str()) self.assertNotIn('thread2-msg2', share1.history.get_history_str())
def test_batchify(self): """Make sure the batchify function sets up the right fields.""" agent = get_agent(rank_candidates=True) obs_labs = [ Message( { 'text': 'It\'s only a flesh wound.', 'labels': ['Yield!'], 'episode_done': True, } ), Message( { 'text': 'The needs of the many outweigh...', 'labels': ['The needs of the few.'], 'episode_done': True, } ), Message( { 'text': 'Hello there.', 'labels': ['General Kenobi.'], 'episode_done': True, } ), ] obs_elabs = [ Message( { 'text': 'It\'s only a flesh wound.', 'eval_labels': ['Yield!'], 'episode_done': True, } ), Message( { 'text': 'The needs of the many outweigh...', 'eval_labels': ['The needs of the few.'], 'episode_done': True, } ), Message( { 'text': 'Hello there.', 'eval_labels': ['General Kenobi.'], 'episode_done': True, } ), ] for obs_batch in (obs_labs, obs_elabs): lab_key = 'labels' if 'labels' in obs_batch[0] else 'eval_labels' # nothing has been vectorized yet so should be empty batch = agent.batchify(obs_batch) self.assertIsNone(batch.text_vec) self.assertIsNone(batch.text_lengths) self.assertIsNone(batch.label_vec) self.assertIsNone(batch.label_lengths) self.assertIsNone(batch.labels) self.assertIsNone(batch.valid_indices) self.assertIsNone(batch.candidates) self.assertIsNone(batch.candidate_vecs) self.assertIsNone(batch.image) obs_vecs = [] for o in obs_batch: agent.history.reset() agent.history.update_history(o) obs_vecs.append( agent.vectorize(o, agent.history, add_start=False, add_end=False) ) # is_valid should map to nothing def is_valid(obs): return False agent.is_valid = is_valid batch = agent.batchify(obs_batch) self.assertIsNone(batch.text_vec) self.assertIsNone(batch.text_lengths) self.assertIsNone(batch.label_vec) self.assertIsNone(batch.label_lengths) self.assertIsNone(batch.labels) self.assertIsNone(batch.valid_indices) self.assertIsNone(batch.candidates) self.assertIsNone(batch.candidate_vecs) self.assertIsNone(batch.image) # is_valid should check for text_vec def is_valid(obs): return 'text_vec' in obs agent.is_valid = is_valid batch = agent.batchify(obs_vecs) # which fields were filled vs should be empty? self.assertIsNotNone(batch.text_vec) self.assertIsNotNone(batch.text_lengths) self.assertIsNotNone(batch.label_vec) self.assertIsNotNone(batch.label_lengths) self.assertIsNotNone(batch.labels) self.assertIsNotNone(batch.valid_indices) self.assertIsNone(batch.candidates) self.assertIsNone(batch.candidate_vecs) self.assertIsNone(batch.image) # contents of certain fields: self.assertEqual( batch.text_vec.tolist(), [[1, 2, 3, 4, 5, 0], [1, 2, 3, 4, 5, 6], [1, 2, 0, 0, 0, 0]], ) self.assertEqual(batch.text_lengths, [5, 6, 2]) self.assertEqual( batch.label_vec.tolist(), [[1, 0, 0, 0, 0], [1, 2, 3, 4, 5], [1, 2, 0, 0, 0]], ) self.assertEqual(batch.label_lengths, [1, 5, 2]) self.assertEqual(batch.labels, [o[lab_key][0] for o in obs_batch]) self.assertEqual(list(batch.valid_indices), [0, 1, 2]) # now sort the batch, make sure fields are in sorted order batch = agent.batchify(obs_vecs, sort=True) self.assertEqual( batch.text_vec.tolist(), [[1, 2, 3, 4, 5, 6], [1, 2, 3, 4, 5, 0], [1, 2, 0, 0, 0, 0]], ) self.assertEqual(batch.text_lengths, [6, 5, 2]) self.assertEqual( batch.label_vec.tolist(), [[1, 2, 3, 4, 5], [1, 0, 0, 0, 0], [1, 2, 0, 0, 0]], ) self.assertEqual(batch.label_lengths, [5, 1, 2]) labs = [o[lab_key][0] for o in obs_batch] self.assertEqual(batch.labels, [labs[i] for i in [1, 0, 2]]) self.assertEqual(list(batch.valid_indices), [1, 0, 2]) # now sort just on ys new_vecs = [vecs.copy() for vecs in obs_vecs] for vec in new_vecs: vec.pop('text') vec.pop('text_vec') def is_valid(obs): return 'labels_vec' in obs or 'eval_labels_vec' in obs agent.is_valid = is_valid batch = agent.batchify(new_vecs, sort=True) self.assertIsNone(batch.text_vec) self.assertIsNone(batch.text_lengths) self.assertIsNotNone(batch.label_vec) self.assertIsNotNone(batch.label_lengths) self.assertEqual( batch.label_vec.tolist(), [[1, 2, 3, 4, 5], [1, 2, 0, 0, 0], [1, 0, 0, 0, 0]], ) self.assertEqual(batch.label_lengths, [5, 2, 1]) labs = [o[lab_key][0] for o in new_vecs] self.assertEqual(batch.labels, [labs[i] for i in [1, 2, 0]]) self.assertEqual(list(batch.valid_indices), [1, 2, 0]) # test is_valid def is_valid(obs): return 'text_vec' in obs and len(obs['text_vec']) < 3 agent.is_valid = is_valid batch = agent.batchify(obs_vecs) self.assertEqual(batch.text_vec.tolist(), [[1, 2]]) self.assertEqual(batch.text_lengths, [2]) self.assertEqual(batch.label_vec.tolist(), [[1, 2]]) self.assertEqual(batch.label_lengths, [2]) self.assertEqual(batch.labels, obs_batch[2][lab_key]) self.assertEqual(list(batch.valid_indices), [2]) agent.history.reset() obs_cands = [ agent.vectorize( Message({'label_candidates': ['A', 'B', 'C']}), agent.history ), agent.vectorize( Message({'label_candidates': ['1', '2', '5', '3', 'Sir']}), agent.history, ), agent.vectorize( Message({'label_candidates': ['Do', 'Re', 'Mi']}), agent.history ), agent.vectorize( Message({'label_candidates': ['Fa', 'So', 'La', 'Ti']}), agent.history ), ] # is_valid should check for label candidates vecs def is_valid(obs): return 'label_candidates_vecs' in obs agent.is_valid = is_valid batch = agent.batchify(obs_cands) self.assertTrue(agent.rank_candidates, 'Agent not set up to rank.') self.assertIsNone(batch.text_vec) self.assertIsNone(batch.text_lengths) self.assertIsNone(batch.label_vec) self.assertIsNone(batch.label_lengths) self.assertIsNone(batch.labels) self.assertIsNotNone(batch.valid_indices) self.assertIsNotNone(batch.candidates) self.assertIsNotNone(batch.candidate_vecs) self.assertEqual(list(batch.valid_indices), [0, 1, 2, 3]) self.assertEqual(batch.candidates, [o['label_candidates'] for o in obs_cands]) self.assertEqual(len(batch.candidate_vecs), len(obs_cands)) for i, cs in enumerate(batch.candidate_vecs): self.assertEqual(len(cs), len(obs_cands[i]['label_candidates']))
def test_vectorize(self): """Test the vectorization of observations. Make sure they do not recompute results, and respect the different param options. """ agent = get_agent() obs_labs = Message( {'text': 'No. Try not.', 'labels': ['Do.', 'Do not.'], 'episode_done': True} ) obs_elabs = Message( { 'text': 'No. Try not.', 'eval_labels': ['Do.', 'Do not.'], 'episode_done': True, } ) for obs in (obs_labs, obs_elabs): lab_key = 'labels' if 'labels' in obs else 'eval_labels' lab_vec = lab_key + '_vec' lab_chc = lab_key + '_choice' inp = obs.copy() # test add_start=True, add_end=True agent.history.reset() agent.history.update_history(inp) out = agent.vectorize(inp, agent.history, add_start=True, add_end=True) self.assertEqual(out['text_vec'].tolist(), [1, 2, 3]) # note that label could be either label above self.assertEqual(out[lab_vec][0].item(), MockDict.BEG_IDX) self.assertEqual(out[lab_vec][1].item(), 1) self.assertEqual(out[lab_vec][-1].item(), MockDict.END_IDX) self.assertEqual(out[lab_chc][:2], 'Do') # test add_start=True, add_end=False inp = obs.copy() out = agent.vectorize(inp, agent.history, add_start=True, add_end=False) self.assertEqual(out['text_vec'].tolist(), [1, 2, 3]) # note that label could be either label above self.assertEqual(out[lab_vec][0].item(), MockDict.BEG_IDX) self.assertNotEqual(out[lab_vec][-1].item(), MockDict.END_IDX) self.assertEqual(out[lab_chc][:2], 'Do') # test add_start=False, add_end=True inp = obs.copy() out = agent.vectorize(inp, agent.history, add_start=False, add_end=True) self.assertEqual(out['text_vec'].tolist(), [1, 2, 3]) # note that label could be either label above self.assertNotEqual(out[lab_vec][0].item(), MockDict.BEG_IDX) self.assertEqual(out[lab_vec][-1].item(), MockDict.END_IDX) self.assertEqual(out[lab_chc][:2], 'Do') # test add_start=False, add_end=False inp = obs.copy() out = agent.vectorize(inp, agent.history, add_start=False, add_end=False) self.assertEqual(out['text_vec'].tolist(), [1, 2, 3]) # note that label could be either label above self.assertNotEqual(out[lab_vec][0].item(), MockDict.BEG_IDX) self.assertNotEqual(out[lab_vec][-1].item(), MockDict.END_IDX) self.assertEqual(out[lab_chc][:2], 'Do') # test caching of tensors out_again = agent.vectorize(out, agent.history) # should have cached result from before self.assertIs(out['text_vec'], out_again['text_vec']) self.assertEqual(out['text_vec'].tolist(), [1, 2, 3]) # next: should truncate cached result prev_vec = out['text_vec'] out_again = agent.vectorize(out, agent.history, text_truncate=1) self.assertIsNot(prev_vec, out_again['text_vec']) self.assertEqual(out['text_vec'].tolist(), [3]) # test split_lines agent = get_agent(split_lines=True) obs = Message( { 'text': 'Hello.\nMy name is Inogo Montoya.\n' 'You killed my father.\nPrepare to die.', 'episode_done': True, } ) agent.history.update_history(obs) vecs = agent.history.get_history_vec_list() self.assertEqual(vecs, [[1], [1, 2, 3, 4, 5], [1, 2, 3, 4], [1, 2, 3]]) # check cache out_again = agent.vectorize(obs, agent.history) vecs = agent.history.get_history_vec_list() self.assertEqual(vecs, [[1], [1, 2, 3, 4, 5], [1, 2, 3, 4], [1, 2, 3]])