def post(self):
name = self.request.get("name")
compressed = not not self.request.get("compressed")
key = self.request.get("key", None)
value = self.request.get("value", None)
if name:
writer = RecordIOWriter(name)
if key == None and value == None:
writer.create(compressed)
elif value == None:
writer.remove(key)
writer.commit_sync()
else:
writer.insert(str(key), eval(value))
writer.commit_sync()
start = None
if key:
start = str(key)
self.redirect("?name=" + str(urllib.quote(name)) + "&start=" +
urllib.quote(start))
delete = self.request.get("delete")
if delete:
writer = RecordIOWriter(delete)
writer.delete()
self.redirect("/recordio/")
def post(self):
name = self.request.get("name")
compressed = not not self.request.get("compressed")
key = self.request.get("key", None)
value = self.request.get("value", None)
if name:
writer = RecordIOWriter(name)
if key == None and value == None:
writer.create(compressed)
elif value == None:
writer.remove(key)
writer.commit_sync()
else:
writer.insert(str(key), eval(value))
writer.commit_sync()
start = None
if key:
start = str(key)
self.redirect("?name=" + str(urllib.quote(name)) + "&start=" +
urllib.quote(start))
delete = self.request.get("delete")
if delete:
writer = RecordIOWriter(delete)
writer.delete()
self.redirect("/recordio/")
def write2MBAndReplace(self, compressed):
test_string = test_helper.uncompressableString(2**21)
updater = RecordIOWriter("test")
updater.create(compressed=compressed)
updater.insert("test", test_string)
updater.commit_sync()
output = []
entries = 0
shards_count = 0
for recordio in RecordIOShard.all():
self.assertTrue(len(recordio.data) >= 1000)
shards_count += 1
for entry in recordio:
output += [entry[-1]]
entries += 1
self.assertTrue(shards_count > 1)
self.assertTrue(entries > 3)
self.assertEqual("".join(output), STRING + test_string, "read != write")
updater.insert("test", "short")
updater.commit_sync(retries=0)
replaced_shards_count = 0
for recordio in RecordIOShard.all():
if replaced_shards_count == 0:
self.assertEqual(1, len(recordio))
for entry in recordio:
self.assertEqual(STRING + "short", entry[-1])
else:
self.assertEqual(0, len(recordio))
for entry in recordio:
self.fail("shouldnt be iterable")
replaced_shards_count += 1
self.assertTrue(len(recordio.data) < 1000)
self.assertTrue(replaced_shards_count > 0)
self.assertTrue(replaced_shards_count <= shards_count)
def do_write(self, single, compressed, entries):
start = time.time()
writer = RecordIOWriter("loadtest_" + single + "_" +
{ True: "compressed",
False: "uncompressed"}[compressed])
writer.create(compressed=compressed)
for entry in entries:
writer.insert(entry[0], entry[1])
writer.commit_sync(retries=10)
return time.time() - start, writer.db_stats()
def do_write(self, single, compressed, entries):
start = time.time()
writer = RecordIOWriter("loadtest_" + single + "_" + {
True: "compressed",
False: "uncompressed"
}[compressed])
writer.create(compressed=compressed)
for entry in entries:
writer.insert(entry[0], entry[1])
writer.commit_sync(retries=10)
return time.time() - start, writer.db_stats()
def writeOneShard(self, compressed):
updater = RecordIOWriter("test")
updater.create(compressed=compressed)
updater.insert("1", "foo")
updater.insert("2", "bar")
updater.commit_sync()
updater = RecordIOWriter("test")
updater.insert("3", "win")
updater.remove("2")
updater.commit_sync()
recordio = RecordIOShard.all().get()
self.assertEqual(recordio.compressed, compressed)
self.assertEqual([x for x in recordio], [("1", STRING + "foo"), ("3", STRING + "win")])
def testWriteStringMarshalPickle(self):
updater = RecordIOWriter("test")
updater.create()
updater.insert("string", "string")
marshalable = {"a": [1,2,3]}
updater.insert("marshal", marshalable)
class AnyClass():
pass
pickleable = AnyClass()
updater.insert("cpickle", pickleable)
updater.commit_sync()
recordio = RecordIOShard.all().get()
self.assertEqual([x for x in recordio],
[("cpickle", CPICKLE + cPickle.dumps(pickleable)),
("marshal", MARSHAL + marshal.dumps(marshalable)),
("string", STRING + "string")])
def testWriteDuringSplit(self):
recordio = RecordIOShard.create("test", compressed=False)
recordio.insert(("1", STRING + "1"))
recordio.insert(("2", STRING + "2"))
lo_shard, hi_shard = recordio.split()
lo_shard.commit()
updater = RecordIOWriter("test")
updater.insert("3", "3")
self.assertRaises(RecordIOShardDoesNotExistError,
updater.commit_shard_,
hi_shard.key().name(), updater.updates)
self.assertRaises(RecordIOWriterNotCompletedError,
updater.commit_sync,
32, 0)
hi_shard.commit()
updater.insert("0", STRING + "0")
updater.commit_sync()
lo_shard, hi_shard = [x for x in RecordIOShard.all()]
self.assertEqual([x[0] for x in lo_shard], ["0", "1"])
self.assertEqual([x[0] for x in hi_shard], ["2", "3"])
def commit_batch(self, tag, batch):
"""Applies a batch of values to a RecordIO and deletes the taskqueue task,
:param tag: The current tag we are working on
:param batch: A list of (tasqueue_task, key_value_list)
:return: True on success
"""
if batch:
done_tasks = []
count = 0
writer = RecordIOWriter(RecordIOShard.get_name(tag))
for done_task, key_values in batch:
done_tasks.append(done_task)
for entry in key_values:
writer.insert_entry_(entry)
count += 1
try:
writer.commit_sync(retries=1)
try:
self.pull.delete_tasks(done_tasks)
except taskqueue.BadTaskStateError:
for task in done_tasks:
if task.was_deleted:
continue
try:
self.pull.delete_tasks(task)
except taskqueue.BadTaskStateError:
logging.debug(
"RecordIO Failed to free task %s on %s" %
task.name, tag)
logging.debug("RecordIO wrote %d entries to %s" %
(count, writer.name))
except RecordIOWriterNotCompletedError:
logging.debug("RecordIO not completed on: %s" % tag)
for task in done_tasks:
self.pull.modify_task_lease(task, 0)
return False
return True
def commit_batch(self, tag, batch):
"""Applies a batch of values to a RecordIO and deletes the taskqueue task,
:param tag: The current tag we are working on
:param batch: A list of (tasqueue_task, key_value_list)
:return: True on success
"""
if batch:
done_tasks = []
count = 0
writer = RecordIOWriter(RecordIOShard.get_name(tag))
for done_task, key_values in batch:
done_tasks.append(done_task)
for entry in key_values:
writer.insert_entry_(entry)
count += 1
try:
writer.commit_sync(retries=1)
try:
self.pull.delete_tasks(done_tasks)
except taskqueue.BadTaskStateError:
for task in done_tasks:
if task.was_deleted:
continue
try:
self.pull.delete_tasks(task)
except taskqueue.BadTaskStateError:
logging.debug("RecordIO Failed to free task %s on %s" %
task.name, tag)
logging.debug("RecordIO wrote %d entries to %s" %
(count, writer.name))
except RecordIOWriterNotCompletedError:
logging.debug("RecordIO not completed on: %s" % tag)
for task in done_tasks:
self.pull.modify_task_lease(task, 0)
return False
return True
