def testCommitToQueueSplitEntries(self):
chunk_size = MAX_ENTRY_SIZE + 1
test_string = test_helper.uncompressableString(chunk_size)
updater = RecordIOWriter("test")
updater.create()
updater.insert("test", test_string)
list(updater.commit_to_queue_())
pull = taskqueue.Queue('recordio-queue')
tasks = list(pull.lease_tasks(60, 100))
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].tag, RecordIOShard.key_name("test"))
updates = marshal.loads(tasks[0].payload)
self.assertEqual([('test', 0, 2), ('test', 1, 2)],
[x[:-2] for x in updates])
self.assertEqual(STRING + test_string, "".join([x[-1] for x in updates]))
def testCommitToQueue(self):
updater = RecordIOWriter("test")
updater.create()
chunk_size = MAX_ENTRY_SIZE - 1
entries_to_write = MAX_TASKQUEUE_BATCH_SIZE / MAX_ENTRY_SIZE + 1
for i in xrange(entries_to_write):
updater.insert(str("%09d" % i),
test_helper.uncompressableString(chunk_size))
list(updater.commit_to_queue_())
pull = taskqueue.Queue('recordio-queue')
tasks = list(pull.lease_tasks(60, 100))
self.assertEqual(len(tasks), 2)
self.assertEqual(tasks[0].tag, RecordIOShard.key_name("test"))
self.assertEqual(tasks[1].tag, RecordIOShard.key_name("test"))
updates_0 = marshal.loads(tasks[0].payload)
updates_1 = marshal.loads(tasks[1].payload)
self.assertEqual([str("%09d" % x) for x in xrange(entries_to_write)],
[x[0] for x in updates_0] + [x[0] for x in updates_1])
self.assertTrue(updates_0[0][1] ==
STRING + test_helper.uncompressableString(chunk_size))
