Exemplo n.º 1
0
 def testReadSplitEntries(self):
   recordio = RecordIOShard.create("test", compressed=False)
   recordio.insert(("a", STRING + "a"))
   recordio.insert(("b", 0, 1, 1, STRING + "b"))
   recordio.insert(("c", STRING + "c"))
   recordio.insert(("d", 0, 2, 1, STRING + "1"))
   recordio.insert(("d", 1, 2, 1, "2"))
   recordio.insert(("e", 0, 3, 1, STRING + "1"))
   recordio.insert(("e", 1, 3, 1, "2"))
   recordio.insert(("e", 2, 3, 1, "3"))
   recordio.insert(("f", STRING + "f"))
   recordio.insert(("g", 0, 2, 2, STRING + "1"))
   recordio.insert(("g", 1, 2, 1, "bad"))
   recordio.insert(("g", 1, 2, 2, "2"))
   recordio.insert(("g_missing_1", 0, 3, 1, STRING + "bad"))
   recordio.insert(("g_missing_1", 1, 3, 1, "bad"))
   recordio.insert(("g_missing_2", 1, 2, 1, "bad"))
   recordio.insert(("h", STRING + "h"))
   recordio.commit()
   reader = RecordIOReader("test")
   self.assertEqual([("a", "a"),
                     ("b", "b"),
                     ("c", "c"),
                     ("d", "12"),
                     ("e", "123"),
                     ("f", "f"),
                     ("g", "12"),
                     ("h", "h")], list(reader.read()))
   self.assertEqual(["g_missing_1"], reader.get_not_read())
Exemplo n.º 2
0
 def testReadFromThreeShards(self):
     recordio = RecordIOShard.create("test", hi=("1", ))
     recordio.insert(("0", STRING + "a"))
     recordio.commit()
     recordio = RecordIOShard.create("test", lo=("1", ), hi=("3", ))
     recordio.insert(("1", STRING + "b"))
     recordio.insert(("2", STRING + "c"))
     recordio.commit()
     recordio = RecordIOShard.create("test", lo=("3", ))
     recordio.insert(("3", STRING + "d"))
     recordio.commit()
     reader = RecordIOReader("test")
     self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
                      list(reader))
     self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
                      list(reader.read(start_key="0")))
     self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
                      list(reader.read(end_key="4")))
     self.assertEqual([("1", "b"), ("2", "c"), ("3", "d")],
                      list(reader.read(start_key="1")))
     self.assertEqual([("2", "c"), ("3", "d")],
                      list(reader.read(start_key="2")))
     self.assertEqual([("0", "a"), ("1", "b")],
                      list(reader.read(end_key="2")))
     self.assertEqual([("1", "b"), ("2", "c")],
                      list(reader.read(start_key="1", end_key="3")))
     self.assertEqual([("1", "b")],
                      list(reader.read(start_key="1", end_key="2")))
Exemplo n.º 3
0
 def testReadFromThreeShards(self):
   recordio = RecordIOShard.create("test", hi=("1",))
   recordio.insert(("0", STRING + "a"))
   recordio.commit()
   recordio = RecordIOShard.create("test", lo=("1",), hi=("3",))
   recordio.insert(("1", STRING + "b"))
   recordio.insert(("2", STRING + "c"))
   recordio.commit()
   recordio = RecordIOShard.create("test", lo=("3",))
   recordio.insert(("3", STRING + "d"))
   recordio.commit()
   reader = RecordIOReader("test")
   self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
                    list(reader))
   self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
                    list(reader.read(start_key="0")))
   self.assertEqual([("0", "a"), ("1", "b"), ("2", "c"), ("3", "d")],
                    list(reader.read(end_key="4")))
   self.assertEqual([("1", "b"), ("2", "c"), ("3", "d")],
                    list(reader.read(start_key="1")))
   self.assertEqual([("2", "c"), ("3", "d")],
                    list(reader.read(start_key="2")))
   self.assertEqual([("0", "a"), ("1", "b")],
                    list(reader.read(end_key="2")))
   self.assertEqual([("1", "b"), ("2", "c")],
                    list(reader.read(start_key="1", end_key="3")))
   self.assertEqual([("1", "b")],
                    list(reader.read(start_key="1", end_key="2")))
Exemplo n.º 4
0
 def testTaskQueue(self):
   writer = RecordIOWriter("test")
   writer.create(compressed=False)
   test_value = test_helper.uncompressableString(MAX_ENTRY_SIZE-1)
   entries_to_write = MAX_BLOB_SIZE / MAX_ENTRY_SIZE + 1
   for i in range(entries_to_write):
     writer.insert(str(i), test_value)
   writer.commit_async()
   
   taskq = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
   tasks = taskq.GetTasks("recordio-writer")
   for task in tasks:
     url=task["url"]
     args = urlparse.parse_qs(base64.b64decode(task["body"]))
     for x in args:
       args[x] = args[x][0]
     test_helper.requestGet(WriteHandler(), url, args)
   assert(len([x for x in RecordIOShard.all()]) > 1)
   reader = RecordIOReader("test")
   result = {}
   for key, value in reader:
     result[key] = value
   self.assertEqual(len(result), entries_to_write)
   for i in range(entries_to_write):
     self.assertEqual(result[str(i)], test_value, "Not equal")
Exemplo n.º 5
0
 def testReadFromInexistingLoShards(self):
     recordio_hi = RecordIOShard.create("test", lo="1")
     recordio_hi.insert(("1", STRING + "b"))
     recordio_hi.insert(("2", STRING + "c"))
     recordio_hi.commit()
     reader = RecordIOReader("test")
     self.assertRaises(RecordIOShardDoesNotExistError, self.readAll, reader)
Exemplo n.º 6
0
 def do_read(self, single, compressed, entries):
   to_read = set([x[0] for x in entries])
   last_read = None
   start = time.time()
   reader = RecordIOReader("loadtest_" + single + "_" +
                           { True: "compressed",
                             False: "uncompressed"}[compressed])
   for key, value in reader:
     if last_read and last_read >= key:
       raise Exception("Results not returned in order! %s >= %s" %
                       (last_read, key))
     last_read = key
     if key in to_read:
       to_read.remove(key)
   if len(to_read):
     raise NotEveryThingWrittenError
   return time.time() - start, reader.db_stats()
Exemplo n.º 7
0
 def do_read(self, single, compressed, entries):
     to_read = set([x[0] for x in entries])
     last_read = None
     start = time.time()
     reader = RecordIOReader("loadtest_" + single + "_" + {
         True: "compressed",
         False: "uncompressed"
     }[compressed])
     for key, value in reader:
         if last_read and last_read >= key:
             raise Exception("Results not returned in order! %s >= %s" %
                             (last_read, key))
         last_read = key
         if key in to_read:
             to_read.remove(key)
     if len(to_read):
         raise NotEveryThingWrittenError
     return time.time() - start, reader.db_stats()
Exemplo n.º 8
0
 def get(self):
     values = {}
     name = self.request.get("name")
     if name:
         values["name"] = name
         start = self.request.get("start", "")
         values["start"] = start
         start_key = str(start)
         key_values = [(key, repr(value)) for key, value in RecordIOReader(
             name).read(start_key=start_key, limit=ENTRIES_PER_PAGE + 1)]
         if len(key_values) == ENTRIES_PER_PAGE + 1:
             values["next"] = key_values[ENTRIES_PER_PAGE][0]
         values["key_values"] = key_values[:ENTRIES_PER_PAGE]
     else:
         values["names"] = list(RecordIOReader.all_names())
     self.response.out.write(
         template.render(
             os.path.join(os.path.dirname(__file__),
                          'recordio_handler_editor.html'), values))
Exemplo n.º 9
0
 def testReadStringMarshalPickle(self):
     recordio = RecordIOShard.create("test")
     recordio.insert(("string", STRING + "string"))
     marshalable = {"a": [1, 2, 3, u"asd"]}
     recordio.insert(("marshal", MARSHAL + marshal.dumps(marshalable)))
     pickleable = AnyClass()
     recordio.insert(("cpickle", CPICKLE + cPickle.dumps(pickleable)))
     recordio.commit()
     reader = RecordIOReader("test")
     self.assertEqual([("cpickle", pickleable), ("marshal", marshalable),
                       ("string", "string")], list(reader))
Exemplo n.º 10
0
 def testReadSplitEntries(self):
     recordio = RecordIOShard.create("test", compressed=False)
     recordio.insert(("a", STRING + "a"))
     recordio.insert(("b", 0, 1, 1, STRING + "b"))
     recordio.insert(("c", STRING + "c"))
     recordio.insert(("d", 0, 2, 1, STRING + "1"))
     recordio.insert(("d", 1, 2, 1, "2"))
     recordio.insert(("e", 0, 3, 1, STRING + "1"))
     recordio.insert(("e", 1, 3, 1, "2"))
     recordio.insert(("e", 2, 3, 1, "3"))
     recordio.insert(("f", STRING + "f"))
     recordio.insert(("g", 0, 2, 2, STRING + "1"))
     recordio.insert(("g", 1, 2, 1, "bad"))
     recordio.insert(("g", 1, 2, 2, "2"))
     recordio.insert(("g_missing_1", 0, 3, 1, STRING + "bad"))
     recordio.insert(("g_missing_1", 1, 3, 1, "bad"))
     recordio.insert(("g_missing_2", 1, 2, 1, "bad"))
     recordio.insert(("h", STRING + "h"))
     recordio.commit()
     reader = RecordIOReader("test")
     self.assertEqual([("a", "a"), ("b", "b"), ("c", "c"), ("d", "12"),
                       ("e", "123"), ("f", "f"), ("g", "12"), ("h", "h")],
                      list(reader.read()))
     self.assertEqual(["g_missing_1"], reader.get_not_read())
Exemplo n.º 11
0
 def get(self):
   values = {}
   name = self.request.get("name")
   if name:
     values["name"] = name
     start = self.request.get("start", "")
     values["start"] = start
     start_key = str(start)
     key_values = [(key, repr(value)) for key, value in
                    RecordIOReader(name).read(
                        start_key=start_key,
                        limit=ENTRIES_PER_PAGE+1)]
     if len(key_values) == ENTRIES_PER_PAGE + 1:
       values["next"] = key_values[ENTRIES_PER_PAGE][0]
     values["key_values"] = key_values[:ENTRIES_PER_PAGE]
   else:
     values["names"] = list(RecordIOReader.all_names())
   self.response.out.write(
       template.render(os.path.join(os.path.dirname(__file__),
                       'recordio_handler_editor.html'), values))
Exemplo n.º 12
0
 def readFromnOneShard(self, compressed):
     recordio = RecordIOShard.create("test", compressed=compressed)
     recordio.insert(("0", STRING + "a"))
     recordio.insert(("1", STRING + "b"))
     recordio.insert(("2", STRING + "c"))
     recordio.commit()
     reader = RecordIOReader("test")
     self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")], list(reader))
     self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
                      list(reader.read(start_key="0")))
     self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
                      list(reader.read(end_key="3")))
     self.assertEqual([("1", "b"), ("2", "c")],
                      list(reader.read(start_key="1")))
     self.assertEqual([("0", "a"), ("1", "b")],
                      list(reader.read(end_key="2")))
     self.assertEqual([("1", "b")],
                      list(reader.read(start_key="1", end_key="2")))
     self.assertTrue("0" in reader)
     self.assertFalse("3" in reader)
     self.assertEqual(reader["0"], "a")
Exemplo n.º 13
0
 def readFromnOneShard(self, compressed):
   recordio = RecordIOShard.create("test", compressed=compressed)
   recordio.insert(("0", STRING + "a"))
   recordio.insert(("1", STRING + "b"))
   recordio.insert(("2", STRING + "c"))
   recordio.commit()
   reader = RecordIOReader("test")
   self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")], list(reader))
   self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
                    list(reader.read(start_key="0")))
   self.assertEqual([("0", "a"), ("1", "b"), ("2", "c")],
                    list(reader.read(end_key="3")))
   self.assertEqual([("1", "b"), ("2", "c")],
                    list(reader.read(start_key="1")))
   self.assertEqual([("0", "a"), ("1", "b")],
                    list(reader.read(end_key="2")))
   self.assertEqual([("1", "b")],
                    list(reader.read(start_key="1", end_key="2")))
   self.assertTrue("0" in reader)
   self.assertFalse("3" in reader)
   self.assertEqual(reader["0"], "a")
Exemplo n.º 14
0
 def testReadFromInexistingHiShards(self):
     recordio_lo = RecordIOShard.create("test", hi="1")
     recordio_lo.insert(("0", STRING + "a"))
     recordio_lo.commit()
     reader = RecordIOReader("test")
     self.assertRaises(RecordIOShardDoesNotExistError, self.readAll, reader)