def testReadSplitEntries(self):
recordio = RecordIOShard.create("test", compressed=False)
recordio.insert(("a", STRING + "a"))
recordio.insert(("b", 0, 1, 1, STRING + "b"))
recordio.insert(("c", STRING + "c"))
recordio.insert(("d", 0, 2, 1, STRING + "1"))
recordio.insert(("d", 1, 2, 1, "2"))
recordio.insert(("e", 0, 3, 1, STRING + "1"))
recordio.insert(("e", 1, 3, 1, "2"))
recordio.insert(("e", 2, 3, 1, "3"))
recordio.insert(("f", STRING + "f"))
recordio.insert(("g", 0, 2, 2, STRING + "1"))
recordio.insert(("g", 1, 2, 1, "bad"))
recordio.insert(("g", 1, 2, 2, "2"))
recordio.insert(("g_missing_1", 0, 3, 1, STRING + "bad"))
recordio.insert(("g_missing_1", 1, 3, 1, "bad"))
recordio.insert(("g_missing_2", 1, 2, 1, "bad"))
recordio.insert(("h", STRING + "h"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("a", "a"),
("b", "b"),
("c", "c"),
("d", "12"),
("e", "123"),
("f", "f"),
("g", "12"),
("h", "h")], list(reader.read()))
self.assertEqual(["g_missing_1"], reader.get_not_read())
def testReadFromThreeShards(self):
recordio = RecordIOShard.create("test", hi=("1", ))
recordio.insert(("0", STRING + "a"))
recordio.commit()
recordio = RecordIOShard.create("test", lo=("1", ), hi=("3", ))
recordio.insert(("1", STRING + "b"))
recordio.insert(("2", STRING + "c"))
recordio.commit()
recordio = RecordIOShard.create("test", lo=("3", ))
recordio.insert(("3", STRING + "d"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(start_key="0")))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(end_key="4")))
self.assertEqual([("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(start_key="1")))
self.assertEqual([("2", "c"), ("3", "d")],
list(reader.read(start_key="2")))
self.assertEqual([("0", "a"), ("1", "b")],
list(reader.read(end_key="2")))
self.assertEqual([("1", "b"), ("2", "c")],
list(reader.read(start_key="1", end_key="3")))
self.assertEqual([("1", "b")],
list(reader.read(start_key="1", end_key="2")))
def testReadFromThreeShards(self):
recordio = RecordIOShard.create("test", hi=("1",))
recordio.insert(("0", STRING + "a"))
recordio.commit()
recordio = RecordIOShard.create("test", lo=("1",), hi=("3",))
recordio.insert(("1", STRING + "b"))
recordio.insert(("2", STRING + "c"))
recordio.commit()
recordio = RecordIOShard.create("test", lo=("3",))
recordio.insert(("3", STRING + "d"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(start_key="0")))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(end_key="4")))
self.assertEqual([("1", "b"), ("2", "c"), ("3", "d")],
list(reader.read(start_key="1")))
self.assertEqual([("2", "c"), ("3", "d")],
list(reader.read(start_key="2")))
self.assertEqual([("0", "a"), ("1", "b")],
list(reader.read(end_key="2")))
self.assertEqual([("1", "b"), ("2", "c")],
list(reader.read(start_key="1", end_key="3")))
self.assertEqual([("1", "b")],
list(reader.read(start_key="1", end_key="2")))
def testTaskQueue(self):
writer = RecordIOWriter("test")
writer.create(compressed=False)
test_value = test_helper.uncompressableString(MAX_ENTRY_SIZE-1)
entries_to_write = MAX_BLOB_SIZE / MAX_ENTRY_SIZE + 1
for i in range(entries_to_write):
writer.insert(str(i), test_value)
writer.commit_async()
taskq = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
tasks = taskq.GetTasks("recordio-writer")
for task in tasks:
url=task["url"]
args = urlparse.parse_qs(base64.b64decode(task["body"]))
for x in args:
args[x] = args[x][0]
test_helper.requestGet(WriteHandler(), url, args)
assert(len([x for x in RecordIOShard.all()]) > 1)
reader = RecordIOReader("test")
result = {}
for key, value in reader:
result[key] = value
self.assertEqual(len(result), entries_to_write)
for i in range(entries_to_write):
self.assertEqual(result[str(i)], test_value, "Not equal")
def testReadFromInexistingLoShards(self):
recordio_hi = RecordIOShard.create("test", lo="1")
recordio_hi.insert(("1", STRING + "b"))
recordio_hi.insert(("2", STRING + "c"))
recordio_hi.commit()
reader = RecordIOReader("test")
self.assertRaises(RecordIOShardDoesNotExistError, self.readAll, reader)
def do_read(self, single, compressed, entries):
to_read = set([x[0] for x in entries])
last_read = None
start = time.time()
reader = RecordIOReader("loadtest_" + single + "_" +
{ True: "compressed",
False: "uncompressed"}[compressed])
for key, value in reader:
if last_read and last_read >= key:
raise Exception("Results not returned in order! %s >= %s" %
(last_read, key))
last_read = key
if key in to_read:
to_read.remove(key)
if len(to_read):
raise NotEveryThingWrittenError
return time.time() - start, reader.db_stats()
def do_read(self, single, compressed, entries):
to_read = set([x[0] for x in entries])
last_read = None
start = time.time()
reader = RecordIOReader("loadtest_" + single + "_" + {
True: "compressed",
False: "uncompressed"
}[compressed])
for key, value in reader:
if last_read and last_read >= key:
raise Exception("Results not returned in order! %s >= %s" %
(last_read, key))
last_read = key
if key in to_read:
to_read.remove(key)
if len(to_read):
raise NotEveryThingWrittenError
return time.time() - start, reader.db_stats()
def get(self):
values = {}
name = self.request.get("name")
if name:
values["name"] = name
start = self.request.get("start", "")
values["start"] = start
start_key = str(start)
key_values = [(key, repr(value)) for key, value in RecordIOReader(
name).read(start_key=start_key, limit=ENTRIES_PER_PAGE + 1)]
if len(key_values) == ENTRIES_PER_PAGE + 1:
values["next"] = key_values[ENTRIES_PER_PAGE][0]
values["key_values"] = key_values[:ENTRIES_PER_PAGE]
else:
values["names"] = list(RecordIOReader.all_names())
self.response.out.write(
template.render(
os.path.join(os.path.dirname(__file__),
'recordio_handler_editor.html'), values))
def testReadStringMarshalPickle(self):
recordio = RecordIOShard.create("test")
recordio.insert(("string", STRING + "string"))
marshalable = {"a": [1, 2, 3, u"asd"]}
recordio.insert(("marshal", MARSHAL + marshal.dumps(marshalable)))
pickleable = AnyClass()
recordio.insert(("cpickle", CPICKLE + cPickle.dumps(pickleable)))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("cpickle", pickleable), ("marshal", marshalable),
("string", "string")], list(reader))
def testReadSplitEntries(self):
recordio = RecordIOShard.create("test", compressed=False)
recordio.insert(("a", STRING + "a"))
recordio.insert(("b", 0, 1, 1, STRING + "b"))
recordio.insert(("c", STRING + "c"))
recordio.insert(("d", 0, 2, 1, STRING + "1"))
recordio.insert(("d", 1, 2, 1, "2"))
recordio.insert(("e", 0, 3, 1, STRING + "1"))
recordio.insert(("e", 1, 3, 1, "2"))
recordio.insert(("e", 2, 3, 1, "3"))
recordio.insert(("f", STRING + "f"))
recordio.insert(("g", 0, 2, 2, STRING + "1"))
recordio.insert(("g", 1, 2, 1, "bad"))
recordio.insert(("g", 1, 2, 2, "2"))
recordio.insert(("g_missing_1", 0, 3, 1, STRING + "bad"))
recordio.insert(("g_missing_1", 1, 3, 1, "bad"))
recordio.insert(("g_missing_2", 1, 2, 1, "bad"))
recordio.insert(("h", STRING + "h"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("a", "a"), ("b", "b"), ("c", "c"), ("d", "12"),
("e", "123"), ("f", "f"), ("g", "12"), ("h", "h")],
list(reader.read()))
self.assertEqual(["g_missing_1"], reader.get_not_read())
def get(self):
values = {}
name = self.request.get("name")
if name:
values["name"] = name
start = self.request.get("start", "")
values["start"] = start
start_key = str(start)
key_values = [(key, repr(value)) for key, value in
RecordIOReader(name).read(
start_key=start_key,
limit=ENTRIES_PER_PAGE+1)]
if len(key_values) == ENTRIES_PER_PAGE + 1:
values["next"] = key_values[ENTRIES_PER_PAGE][0]
values["key_values"] = key_values[:ENTRIES_PER_PAGE]
else:
values["names"] = list(RecordIOReader.all_names())
self.response.out.write(
template.render(os.path.join(os.path.dirname(__file__),
'recordio_handler_editor.html'), values))
def readFromnOneShard(self, compressed):
recordio = RecordIOShard.create("test", compressed=compressed)
recordio.insert(("0", STRING + "a"))
recordio.insert(("1", STRING + "b"))
recordio.insert(("2", STRING + "c"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")], list(reader))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
list(reader.read(start_key="0")))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
list(reader.read(end_key="3")))
self.assertEqual([("1", "b"), ("2", "c")],
list(reader.read(start_key="1")))
self.assertEqual([("0", "a"), ("1", "b")],
list(reader.read(end_key="2")))
self.assertEqual([("1", "b")],
list(reader.read(start_key="1", end_key="2")))
self.assertTrue("0" in reader)
self.assertFalse("3" in reader)
self.assertEqual(reader["0"], "a")
def readFromnOneShard(self, compressed):
recordio = RecordIOShard.create("test", compressed=compressed)
recordio.insert(("0", STRING + "a"))
recordio.insert(("1", STRING + "b"))
recordio.insert(("2", STRING + "c"))
recordio.commit()
reader = RecordIOReader("test")
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")], list(reader))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
list(reader.read(start_key="0")))
self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
list(reader.read(end_key="3")))
self.assertEqual([("1", "b"), ("2", "c")],
list(reader.read(start_key="1")))
self.assertEqual([("0", "a"), ("1", "b")],
list(reader.read(end_key="2")))
self.assertEqual([("1", "b")],
list(reader.read(start_key="1", end_key="2")))
self.assertTrue("0" in reader)
self.assertFalse("3" in reader)
self.assertEqual(reader["0"], "a")
def testReadFromInexistingHiShards(self):
recordio_lo = RecordIOShard.create("test", hi="1")
recordio_lo.insert(("0", STRING + "a"))
recordio_lo.commit()
reader = RecordIOReader("test")
self.assertRaises(RecordIOShardDoesNotExistError, self.readAll, reader)