def write2MBAndReplace(self, compressed):
test_string = test_helper.uncompressableString(2**21)
updater = RecordIOWriter("test")
updater.create(compressed=compressed)
updater.insert("test", test_string)
updater.commit_sync()
output = []
entries = 0
shards_count = 0
for recordio in RecordIOShard.all():
self.assertTrue(len(recordio.data) >= 1000)
shards_count += 1
for entry in recordio:
output += [entry[-1]]
entries += 1
self.assertTrue(shards_count > 1)
self.assertTrue(entries > 3)
self.assertEqual("".join(output), STRING + test_string, "read != write")
updater.insert("test", "short")
updater.commit_sync(retries=0)
replaced_shards_count = 0
for recordio in RecordIOShard.all():
if replaced_shards_count == 0:
self.assertEqual(1, len(recordio))
for entry in recordio:
self.assertEqual(STRING + "short", entry[-1])
else:
self.assertEqual(0, len(recordio))
for entry in recordio:
self.fail("shouldnt be iterable")
replaced_shards_count += 1
self.assertTrue(len(recordio.data) < 1000)
self.assertTrue(replaced_shards_count > 0)
self.assertTrue(replaced_shards_count <= shards_count)
def testTaskQueue(self):
writer = RecordIOWriter("test")
writer.create(compressed=False)
test_value = test_helper.uncompressableString(MAX_ENTRY_SIZE-1)
entries_to_write = MAX_BLOB_SIZE / MAX_ENTRY_SIZE + 1
for i in range(entries_to_write):
writer.insert(str(i), test_value)
writer.commit_async()
taskq = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
tasks = taskq.GetTasks("recordio-writer")
for task in tasks:
url=task["url"]
args = urlparse.parse_qs(base64.b64decode(task["body"]))
for x in args:
args[x] = args[x][0]
test_helper.requestGet(WriteHandler(), url, args)
assert(len([x for x in RecordIOShard.all()]) > 1)
reader = RecordIOReader("test")
result = {}
for key, value in reader:
result[key] = value
self.assertEqual(len(result), entries_to_write)
for i in range(entries_to_write):
self.assertEqual(result[str(i)], test_value, "Not equal")
def all_names():
"""Returns the names of all existing RecordIOs.
:return: list of RecordIO names
"""
for x in RecordIOShard.all(keys_only=True).filter("index =", True):
yield RecordIOShard.get_name(x.name())
def all_names():
"""Returns the names of all existing RecordIOs.
:return: list of RecordIO names
"""
for x in RecordIOShard.all(keys_only=True).filter("index =", True):
yield RecordIOShard.get_name(x.name())
def writeOneShard(self, compressed):
updater = RecordIOWriter("test")
updater.create(compressed=compressed)
updater.insert("1", "foo")
updater.insert("2", "bar")
updater.commit_sync()
updater = RecordIOWriter("test")
updater.insert("3", "win")
updater.remove("2")
updater.commit_sync()
recordio = RecordIOShard.all().get()
self.assertEqual(recordio.compressed, compressed)
self.assertEqual([x for x in recordio], [("1", STRING + "foo"), ("3", STRING + "win")])
def testWriteStringMarshalPickle(self):
updater = RecordIOWriter("test")
updater.create()
updater.insert("string", "string")
marshalable = {"a": [1,2,3]}
updater.insert("marshal", marshalable)
class AnyClass():
pass
pickleable = AnyClass()
updater.insert("cpickle", pickleable)
updater.commit_sync()
recordio = RecordIOShard.all().get()
self.assertEqual([x for x in recordio],
[("cpickle", CPICKLE + cPickle.dumps(pickleable)),
("marshal", MARSHAL + marshal.dumps(marshalable)),
("string", STRING + "string")])
def testWriteDuringSplit(self):
recordio = RecordIOShard.create("test", compressed=False)
recordio.insert(("1", STRING + "1"))
recordio.insert(("2", STRING + "2"))
lo_shard, hi_shard = recordio.split()
lo_shard.commit()
updater = RecordIOWriter("test")
updater.insert("3", "3")
self.assertRaises(RecordIOShardDoesNotExistError,
updater.commit_shard_,
hi_shard.key().name(), updater.updates)
self.assertRaises(RecordIOWriterNotCompletedError,
updater.commit_sync,
32, 0)
hi_shard.commit()
updater.insert("0", STRING + "0")
updater.commit_sync()
lo_shard, hi_shard = [x for x in RecordIOShard.all()]
self.assertEqual([x[0] for x in lo_shard], ["0", "1"])
self.assertEqual([x[0] for x in hi_shard], ["2", "3"])
