def testReadSplitEntries(self):
recordio = RecordIOShard.create("test", compressed=False)
recordio.insert(("a", STRING + "a"))
recordio.insert(("b", 0, 1, 1, STRING + "b"))
recordio.insert(("c", STRING + "c"))
recordio.insert(("d", 0, 2, 1, STRING + "1"))
recordio.insert(("d", 1, 2, 1, "2"))
recordio.insert(("e", 0, 3, 1, STRING + "1"))
recordio.insert(("e", 1, 3, 1, "2"))
recordio.insert(("e", 2, 3, 1, "3"))
recordio.insert(("f", STRING + "f"))
recordio.insert(("g", 0, 2, 2, STRING + "1"))
recordio.insert(("g", 1, 2, 1, "bad"))
recordio.insert(("g", 1, 2, 2, "2"))
recordio.insert(("g_missing_1", 0, 3, 1, STRING + "bad"))
recordio.insert(("g_missing_1", 1, 3, 1, "bad"))
recordio.insert(("g_missing_2", 1, 2, 1, "bad"))
recordio.insert(("h", STRING + "h"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("a", "a"),
("b", "b"),
("c", "c"),
("d", "12"),
("e", "123"),
("f", "f"),
("g", "12"),
("h", "h")], list(reader.read()))
self.assertEqual(["g_missing_1"], reader.get_not_read())
def testReadFromThreeShards(self):
recordio = RecordIOShard.create("test", hi=("1", ))
recordio.insert(("0", STRING + "a"))
recordio.commit()
recordio = RecordIOShard.create("test", lo=("1", ), hi=("3", ))
recordio.insert(("1", STRING + "b"))
recordio.insert(("2", STRING + "c"))
recordio.commit()
recordio = RecordIOShard.create("test", lo=("3", ))
recordio.insert(("3", STRING + "d"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(start_key="0")))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(end_key="4")))
self.assertEqual([("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(start_key="1")))
self.assertEqual([("2", "c"), ("3", "d")],
list(reader.read(start_key="2")))
self.assertEqual([("0", "a"), ("1", "b")],
list(reader.read(end_key="2")))
self.assertEqual([("1", "b"), ("2", "c")],
list(reader.read(start_key="1", end_key="3")))
self.assertEqual([("1", "b")],
list(reader.read(start_key="1", end_key="2")))
def testReadFromThreeShards(self):
recordio = RecordIOShard.create("test", hi=("1",))
recordio.insert(("0", STRING + "a"))
recordio.commit()
recordio = RecordIOShard.create("test", lo=("1",), hi=("3",))
recordio.insert(("1", STRING + "b"))
recordio.insert(("2", STRING + "c"))
recordio.commit()
recordio = RecordIOShard.create("test", lo=("3",))
recordio.insert(("3", STRING + "d"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(start_key="0")))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(end_key="4")))
self.assertEqual([("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(start_key="1")))
self.assertEqual([("2", "c"), ("3", "d")],
list(reader.read(start_key="2")))
self.assertEqual([("0", "a"), ("1", "b")],
list(reader.read(end_key="2")))
self.assertEqual([("1", "b"), ("2", "c")],
list(reader.read(start_key="1", end_key="3")))
self.assertEqual([("1", "b")],
list(reader.read(start_key="1", end_key="2")))
def readFromnOneShard(self, compressed):
recordio = RecordIOShard.create("test", compressed=compressed)
recordio.insert(("0", STRING + "a"))
recordio.insert(("1", STRING + "b"))
recordio.insert(("2", STRING + "c"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")], list(reader))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
list(reader.read(start_key="0")))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
list(reader.read(end_key="3")))
self.assertEqual([("1", "b"), ("2", "c")],
list(reader.read(start_key="1")))
self.assertEqual([("0", "a"), ("1", "b")],
list(reader.read(end_key="2")))
self.assertEqual([("1", "b")],
list(reader.read(start_key="1", end_key="2")))
self.assertTrue("0" in reader)
self.assertFalse("3" in reader)
self.assertEqual(reader["0"], "a")
def readFromnOneShard(self, compressed):
recordio = RecordIOShard.create("test", compressed=compressed)
recordio.insert(("0", STRING + "a"))
recordio.insert(("1", STRING + "b"))
recordio.insert(("2", STRING + "c"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")], list(reader))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
list(reader.read(start_key="0")))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
list(reader.read(end_key="3")))
self.assertEqual([("1", "b"), ("2", "c")],
list(reader.read(start_key="1")))
self.assertEqual([("0", "a"), ("1", "b")],
list(reader.read(end_key="2")))
self.assertEqual([("1", "b")],
list(reader.read(start_key="1", end_key="2")))
self.assertTrue("0" in reader)
self.assertFalse("3" in reader)
self.assertEqual(reader["0"], "a")
def testReadSplitEntries(self):
recordio = RecordIOShard.create("test", compressed=False)
recordio.insert(("a", STRING + "a"))
recordio.insert(("b", 0, 1, 1, STRING + "b"))
recordio.insert(("c", STRING + "c"))
recordio.insert(("d", 0, 2, 1, STRING + "1"))
recordio.insert(("d", 1, 2, 1, "2"))
recordio.insert(("e", 0, 3, 1, STRING + "1"))
recordio.insert(("e", 1, 3, 1, "2"))
recordio.insert(("e", 2, 3, 1, "3"))
recordio.insert(("f", STRING + "f"))
recordio.insert(("g", 0, 2, 2, STRING + "1"))
recordio.insert(("g", 1, 2, 1, "bad"))
recordio.insert(("g", 1, 2, 2, "2"))
recordio.insert(("g_missing_1", 0, 3, 1, STRING + "bad"))
recordio.insert(("g_missing_1", 1, 3, 1, "bad"))
recordio.insert(("g_missing_2", 1, 2, 1, "bad"))
recordio.insert(("h", STRING + "h"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("a", "a"), ("b", "b"), ("c", "c"), ("d", "12"),
("e", "123"), ("f", "f"), ("g", "12"), ("h", "h")],
list(reader.read()))
self.assertEqual(["g_missing_1"], reader.get_not_read())
