def testSplitNSByScatter_LotsOfData(self):
   """Split lots of data for each shard."""
   testutil._create_entities(range(100),
                             {"80": 80, "50": 50, "30": 30, "10": 10},
                             ns="google")
   expected = [key_range.KeyRange(key_start=None,
                                  key_end=testutil.key("30",
                                                       namespace="google"),
                                  direction="ASC",
                                  include_start=False,
                                  include_end=False,
                                  namespace="google",
                                  _app=self.appid),
               key_range.KeyRange(key_start=testutil.key("30",
                                                         namespace="google"),
                                  key_end=testutil.key("80",
                                                       namespace="google"),
                                  direction="ASC",
                                  include_start=True,
                                  include_end=False,
                                  namespace="google",
                                  _app=self.appid),
               key_range.KeyRange(key_start=testutil.key("80",
                                                         namespace="google"),
                                  key_end=None,
                                  direction="ASC",
                                  include_start=True,
                                  include_end=False,
                                  namespace="google",
                                  _app=self.appid),
              ]
   self._assertEquals_splitNSByScatter(3, expected, ns="google")
 def testSplitNSByScatter_NotEnoughData2(self):
   """Splits should not intersect, if there's not enough data for each."""
   testutil._create_entities(range(10), {"2": 2, "4": 4})
   expected = [key_range.KeyRange(key_start=None,
                                  key_end=testutil.key("2"),
                                  direction="ASC",
                                  include_start=False,
                                  include_end=False,
                                  namespace="",
                                  _app=self.appid),
               key_range.KeyRange(key_start=testutil.key("2"),
                                  key_end=testutil.key("4"),
                                  direction="ASC",
                                  include_start=True,
                                  include_end=False,
                                  namespace="",
                                  _app=self.appid),
               key_range.KeyRange(key_start=testutil.key("4"),
                                  key_end=None,
                                  direction="ASC",
                                  include_start=True,
                                  include_end=False,
                                  namespace="",
                                  _app=self.appid),
               None]
   self._assertEquals_splitNSByScatter(4, expected)
Example #3
0
 def testSplitNSByScatter_LotsOfData(self):
     """Split lots of data for each shard."""
     testutil._create_entities(range(100), {
         "80": 80,
         "50": 50,
         "30": 30,
         "10": 10
     },
                               ns="google")
     expected = [
         key_range.KeyRange(key_start=None,
                            key_end=testutil.key("30", namespace="google"),
                            direction="ASC",
                            include_start=False,
                            include_end=False,
                            namespace="google",
                            _app=self.appid),
         key_range.KeyRange(key_start=testutil.key("30",
                                                   namespace="google"),
                            key_end=testutil.key("80", namespace="google"),
                            direction="ASC",
                            include_start=True,
                            include_end=False,
                            namespace="google",
                            _app=self.appid),
         key_range.KeyRange(key_start=testutil.key("80",
                                                   namespace="google"),
                            key_end=None,
                            direction="ASC",
                            include_start=True,
                            include_end=False,
                            namespace="google",
                            _app=self.appid),
     ]
     self._assertEquals_splitNSByScatter(3, expected, ns="google")
Example #4
0
 def testSplitNSByScatter_NotEnoughData2(self):
     """Splits should not intersect, if there's not enough data for each."""
     testutil._create_entities(range(10), {"2": 2, "4": 4})
     expected = [
         key_range.KeyRange(key_start=None,
                            key_end=testutil.key("2"),
                            direction="ASC",
                            include_start=False,
                            include_end=False,
                            namespace="",
                            _app=self.appid),
         key_range.KeyRange(key_start=testutil.key("2"),
                            key_end=testutil.key("4"),
                            direction="ASC",
                            include_start=True,
                            include_end=False,
                            namespace="",
                            _app=self.appid),
         key_range.KeyRange(key_start=testutil.key("4"),
                            key_end=None,
                            direction="ASC",
                            include_start=True,
                            include_end=False,
                            namespace="",
                            _app=self.appid), None
     ]
     self._assertEquals_splitNSByScatter(4, expected)
  def testToKeyRangesByShard_UnevenNamespaces(self):
    namespaces = [str(i) for i in range(3)]
    testutil._create_entities(range(10), {"5": 5}, namespaces[0])
    testutil._create_entities(range(10), {"5": 5, "6": 6}, namespaces[1])
    testutil._create_entities(range(10), {"5": 5, "6": 6, "7": 7},
                              namespaces[2])
    shards = 3

    expected = [
        # shard 1
        key_range.KeyRange(key_start=None,
                           key_end=testutil.key("5", namespace="0"),
                           direction="ASC",
                           include_start=False,
                           include_end=False,
                           namespace="0",
                           _app=self.appid),
        key_range.KeyRange(key_start=None,
                           key_end=testutil.key("5", namespace="1"),
                           direction="ASC",
                           include_start=False,
                           include_end=False,
                           namespace="1",
                           _app=self.appid),
        key_range.KeyRange(key_start=None,
                           key_end=testutil.key("6", namespace="2"),
                           direction="ASC",
                           include_start=False,
                           include_end=False,
                           namespace="2",
                           _app=self.appid),
        # shard 2
        key_range.KeyRange(key_start=testutil.key("5", namespace="0"),
                           key_end=None,
                           direction="ASC",
                           include_start=True,
                           include_end=False,
                           namespace="0",
                           _app=self.appid),
        key_range.KeyRange(key_start=testutil.key("5", namespace="1"),
                           key_end=testutil.key("6", namespace="1"),
                           direction="ASC",
                           include_start=True,
                           include_end=False,
                           namespace="1",
                           _app=self.appid),
        key_range.KeyRange(key_start=testutil.key("6", namespace="2"),
                           key_end=testutil.key("7", namespace="2"),
                           direction="ASC",
                           include_start=True,
                           include_end=False,
                           namespace="2",
                           _app=self.appid),
        # shard 3
        key_range.KeyRange(key_start=testutil.key("6", namespace="1"),
                           key_end=None,
                           direction="ASC",
                           include_start=True,
                           include_end=False,
                           namespace="1",
                           _app=self.appid),
        key_range.KeyRange(key_start=testutil.key("7", namespace="2"),
                           key_end=None,
                           direction="ASC",
                           include_start=True,
                           include_end=False,
                           namespace="2",
                           _app=self.appid),
    ]
    kranges_by_shard = (self.reader_cls._to_key_ranges_by_shard(
        self.appid, namespaces, shards,
        model.QuerySpec(entity_kind="TestEntity")))
    self.assertEquals(shards, len(kranges_by_shard))

    expected.sort()
    results = []
    for kranges in kranges_by_shard:
      results.extend(list(kranges))
    results.sort()
    self.assertEquals(expected, results)
Example #6
0
    def testToKeyRangesByShard_UnevenNamespaces(self):
        namespaces = [str(i) for i in range(3)]
        testutil._create_entities(range(10), {"5": 5}, namespaces[0])
        testutil._create_entities(range(10), {"5": 5, "6": 6}, namespaces[1])
        testutil._create_entities(range(10), {
            "5": 5,
            "6": 6,
            "7": 7
        }, namespaces[2])
        shards = 3

        expected = [
            # shard 1
            key_range.KeyRange(key_start=None,
                               key_end=testutil.key("5", namespace="0"),
                               direction="ASC",
                               include_start=False,
                               include_end=False,
                               namespace="0",
                               _app=self.appid),
            key_range.KeyRange(key_start=None,
                               key_end=testutil.key("5", namespace="1"),
                               direction="ASC",
                               include_start=False,
                               include_end=False,
                               namespace="1",
                               _app=self.appid),
            key_range.KeyRange(key_start=None,
                               key_end=testutil.key("6", namespace="2"),
                               direction="ASC",
                               include_start=False,
                               include_end=False,
                               namespace="2",
                               _app=self.appid),
            # shard 2
            key_range.KeyRange(key_start=testutil.key("5", namespace="0"),
                               key_end=None,
                               direction="ASC",
                               include_start=True,
                               include_end=False,
                               namespace="0",
                               _app=self.appid),
            key_range.KeyRange(key_start=testutil.key("5", namespace="1"),
                               key_end=testutil.key("6", namespace="1"),
                               direction="ASC",
                               include_start=True,
                               include_end=False,
                               namespace="1",
                               _app=self.appid),
            key_range.KeyRange(key_start=testutil.key("6", namespace="2"),
                               key_end=testutil.key("7", namespace="2"),
                               direction="ASC",
                               include_start=True,
                               include_end=False,
                               namespace="2",
                               _app=self.appid),
            # shard 3
            key_range.KeyRange(key_start=testutil.key("6", namespace="1"),
                               key_end=None,
                               direction="ASC",
                               include_start=True,
                               include_end=False,
                               namespace="1",
                               _app=self.appid),
            key_range.KeyRange(key_start=testutil.key("7", namespace="2"),
                               key_end=None,
                               direction="ASC",
                               include_start=True,
                               include_end=False,
                               namespace="2",
                               _app=self.appid),
        ]
        kranges_by_shard = (self.reader_cls._to_key_ranges_by_shard(
            self.appid, namespaces, shards,
            model.QuerySpec(entity_kind="TestEntity")))
        self.assertEquals(shards, len(kranges_by_shard))

        expected.sort()
        results = []
        for kranges in kranges_by_shard:
            results.extend(list(kranges))
        results.sort()
        self.assertEquals(expected, results)