def write2MBAndReplace(self, compressed): test_string = test_helper.uncompressableString(2**21) updater = RecordIOWriter("test") updater.create(compressed=compressed) updater.insert("test", test_string) updater.commit_sync() output = [] entries = 0 shards_count = 0 for recordio in RecordIOShard.all(): self.assertTrue(len(recordio.data) >= 1000) shards_count += 1 for entry in recordio: output += [entry[-1]] entries += 1 self.assertTrue(shards_count > 1) self.assertTrue(entries > 3) self.assertEqual("".join(output), STRING + test_string, "read != write") updater.insert("test", "short") updater.commit_sync(retries=0) replaced_shards_count = 0 for recordio in RecordIOShard.all(): if replaced_shards_count == 0: self.assertEqual(1, len(recordio)) for entry in recordio: self.assertEqual(STRING + "short", entry[-1]) else: self.assertEqual(0, len(recordio)) for entry in recordio: self.fail("shouldnt be iterable") replaced_shards_count += 1 self.assertTrue(len(recordio.data) < 1000) self.assertTrue(replaced_shards_count > 0) self.assertTrue(replaced_shards_count <= shards_count)
def testTaskQueue(self): writer = RecordIOWriter("test") writer.create(compressed=False) test_value = test_helper.uncompressableString(MAX_ENTRY_SIZE-1) entries_to_write = MAX_BLOB_SIZE / MAX_ENTRY_SIZE + 1 for i in range(entries_to_write): writer.insert(str(i), test_value) writer.commit_async() taskq = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) tasks = taskq.GetTasks("recordio-writer") for task in tasks: url=task["url"] args = urlparse.parse_qs(base64.b64decode(task["body"])) for x in args: args[x] = args[x][0] test_helper.requestGet(WriteHandler(), url, args) assert(len([x for x in RecordIOShard.all()]) > 1) reader = RecordIOReader("test") result = {} for key, value in reader: result[key] = value self.assertEqual(len(result), entries_to_write) for i in range(entries_to_write): self.assertEqual(result[str(i)], test_value, "Not equal")
def all_names(): """Returns the names of all existing RecordIOs. :return: list of RecordIO names """ for x in RecordIOShard.all(keys_only=True).filter("index =", True): yield RecordIOShard.get_name(x.name())
def writeOneShard(self, compressed): updater = RecordIOWriter("test") updater.create(compressed=compressed) updater.insert("1", "foo") updater.insert("2", "bar") updater.commit_sync() updater = RecordIOWriter("test") updater.insert("3", "win") updater.remove("2") updater.commit_sync() recordio = RecordIOShard.all().get() self.assertEqual(recordio.compressed, compressed) self.assertEqual([x for x in recordio], [("1", STRING + "foo"), ("3", STRING + "win")])
def testWriteStringMarshalPickle(self): updater = RecordIOWriter("test") updater.create() updater.insert("string", "string") marshalable = {"a": [1,2,3]} updater.insert("marshal", marshalable) class AnyClass(): pass pickleable = AnyClass() updater.insert("cpickle", pickleable) updater.commit_sync() recordio = RecordIOShard.all().get() self.assertEqual([x for x in recordio], [("cpickle", CPICKLE + cPickle.dumps(pickleable)), ("marshal", MARSHAL + marshal.dumps(marshalable)), ("string", STRING + "string")])
def testWriteDuringSplit(self): recordio = RecordIOShard.create("test", compressed=False) recordio.insert(("1", STRING + "1")) recordio.insert(("2", STRING + "2")) lo_shard, hi_shard = recordio.split() lo_shard.commit() updater = RecordIOWriter("test") updater.insert("3", "3") self.assertRaises(RecordIOShardDoesNotExistError, updater.commit_shard_, hi_shard.key().name(), updater.updates) self.assertRaises(RecordIOWriterNotCompletedError, updater.commit_sync, 32, 0) hi_shard.commit() updater.insert("0", STRING + "0") updater.commit_sync() lo_shard, hi_shard = [x for x in RecordIOShard.all()] self.assertEqual([x[0] for x in lo_shard], ["0", "1"]) self.assertEqual([x[0] for x in hi_shard], ["2", "3"])