def test_dumped_example_visitor(self):
visitor = example_id_visitor.ExampleIdVisitor(self.kvstore, self.data_source, 0)
expected_index = 0
for (index, example) in visitor:
self.assertEqual(index, expected_index)
self.assertEqual('{}'.format(index).encode(), example.example_id)
self.assertEqual(150000000+index, example.event_time)
self.assertEqual(index, example.index)
expected_index += 1
self.assertEqual(0, expected_index)
self.assertRaises(StopIteration, visitor.seek, 200)
self.assertTrue(visitor.finished())
dumper = example_id_dumper.ExampleIdDumperManager(
self.kvstore, self.data_source, 0, self.example_id_dump_options
)
self.assertEqual(dumper.get_next_index(), 0)
self._dump_example_ids(dumper, 0, 10, 1024)
self.assertTrue(visitor.is_visitor_stale())
visitor.active_visitor()
for (index, example) in visitor:
self.assertEqual(index, expected_index)
self.assertEqual('{}'.format(index).encode(), example.example_id)
self.assertEqual(150000000+index, example.event_time)
self.assertEqual(index, example.index)
expected_index += 1
self.assertEqual(10240, expected_index)
self.assertTrue(visitor.finished())
visitor.seek(200)
expected_index = 200
self.assertEqual(expected_index, visitor.get_index())
self.assertEqual(expected_index, visitor.get_item().index)
self.assertEqual(150000000+expected_index, visitor.get_item().event_time)
dumper2 = example_id_dumper.ExampleIdDumperManager(
self.kvstore, self.data_source, 0, self.example_id_dump_options
)
self._dump_example_ids(dumper2, 10240, 10, 1024)
expected_index += 1
self.assertTrue(visitor.is_visitor_stale())
visitor.active_visitor()
for (index, example) in visitor:
self.assertEqual(index, expected_index)
self.assertEqual('{}'.format(index).encode(), example.example_id)
self.assertEqual(150000000+index, example.event_time)
self.assertEqual(index, example.index)
expected_index += 1
self.assertEqual(10240 * 2, expected_index)
visitor2 = example_id_visitor.ExampleIdVisitor(self.kvstore, self.data_source, 0)
visitor2.seek(886)
expected_index = 886
self.assertEqual(expected_index, visitor2.get_index())
self.assertEqual(expected_index, visitor2.get_item().index)
self.assertEqual(150000000+expected_index, visitor2.get_item().event_time)
expected_index += 1
for (index, example) in visitor2:
self.assertEqual(index, expected_index)
self.assertEqual('{}'.format(index).encode(), example.example_id)
self.assertEqual(150000000+index, example.event_time)
self.assertEqual(index, example.index)
expected_index += 1
self.assertEqual(10240 * 2, expected_index)
def test_dumped_example_visitor(self):
example_id_manager = example_id_visitor.ExampleIdManager(
self.data_source, 0)
visitor = example_id_visitor.ExampleIdVisitor(example_id_manager)
expected_index = 0
for (index, example) in visitor:
self.assertEqual(index, expected_index)
self.assertEqual('{}'.format(index).encode(), example.example_id)
self.assertEqual(150000000 + index, example.event_time)
self.assertEqual(index, example.index)
expected_index += 1
self.assertEqual(self.end_index, index)
try:
visitor.seek(1 << 30)
except StopIteration:
self.assertTrue(True)
self.assertTrue(visitor.finished())
else:
self.assertTrue(False)
index, example = visitor.seek(500)
self.assertFalse(visitor.finished())
expected_index = 500
self.assertEqual(index, expected_index)
self.assertEqual('{}'.format(index).encode(), example.example_id)
self.assertEqual(150000000 + index, example.event_time)
self.assertEqual(index, example.index)
expected_index = 501
for (index, example) in visitor:
self.assertEqual(index, expected_index)
self.assertEqual('{}'.format(index).encode(), example.example_id)
self.assertEqual(150000000 + index, example.event_time)
self.assertEqual(index, example.index)
expected_index += 1
self.assertEqual(self.end_index, index)