def testSplitNSByScatter_LotsOfData(self):
"""Split lots of data for each shard."""
testutil._create_entities(range(100),
{"80": 80, "50": 50, "30": 30, "10": 10},
ns="google")
expected = [key_range.KeyRange(key_start=None,
key_end=testutil.key("30",
namespace="google"),
direction="ASC",
include_start=False,
include_end=False,
namespace="google",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("30",
namespace="google"),
key_end=testutil.key("80",
namespace="google"),
direction="ASC",
include_start=True,
include_end=False,
namespace="google",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("80",
namespace="google"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="google",
_app=self.appid),
]
self._assertEquals_splitNSByScatter(3, expected, ns="google")
def testSplitNSByScatter_NotEnoughData2(self):
"""Splits should not intersect, if there's not enough data for each."""
testutil._create_entities(range(10), {"2": 2, "4": 4})
expected = [key_range.KeyRange(key_start=None,
key_end=testutil.key("2"),
direction="ASC",
include_start=False,
include_end=False,
namespace="",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("2"),
key_end=testutil.key("4"),
direction="ASC",
include_start=True,
include_end=False,
namespace="",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("4"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="",
_app=self.appid),
None]
self._assertEquals_splitNSByScatter(4, expected)
def testSplitNSByScatter_LotsOfData(self):
"""Split lots of data for each shard."""
testutil._create_entities(range(100), {
"80": 80,
"50": 50,
"30": 30,
"10": 10
},
ns="google")
expected = [
key_range.KeyRange(key_start=None,
key_end=testutil.key("30", namespace="google"),
direction="ASC",
include_start=False,
include_end=False,
namespace="google",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("30",
namespace="google"),
key_end=testutil.key("80", namespace="google"),
direction="ASC",
include_start=True,
include_end=False,
namespace="google",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("80",
namespace="google"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="google",
_app=self.appid),
]
self._assertEquals_splitNSByScatter(3, expected, ns="google")
def testSplitNSByScatter_NotEnoughData2(self):
"""Splits should not intersect, if there's not enough data for each."""
testutil._create_entities(range(10), {"2": 2, "4": 4})
expected = [
key_range.KeyRange(key_start=None,
key_end=testutil.key("2"),
direction="ASC",
include_start=False,
include_end=False,
namespace="",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("2"),
key_end=testutil.key("4"),
direction="ASC",
include_start=True,
include_end=False,
namespace="",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("4"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="",
_app=self.appid), None
]
self._assertEquals_splitNSByScatter(4, expected)
def testToKeyRangesByShard_UnevenNamespaces(self):
namespaces = [str(i) for i in range(3)]
testutil._create_entities(range(10), {"5": 5}, namespaces[0])
testutil._create_entities(range(10), {"5": 5, "6": 6}, namespaces[1])
testutil._create_entities(range(10), {"5": 5, "6": 6, "7": 7},
namespaces[2])
shards = 3
expected = [
# shard 1
key_range.KeyRange(key_start=None,
key_end=testutil.key("5", namespace="0"),
direction="ASC",
include_start=False,
include_end=False,
namespace="0",
_app=self.appid),
key_range.KeyRange(key_start=None,
key_end=testutil.key("5", namespace="1"),
direction="ASC",
include_start=False,
include_end=False,
namespace="1",
_app=self.appid),
key_range.KeyRange(key_start=None,
key_end=testutil.key("6", namespace="2"),
direction="ASC",
include_start=False,
include_end=False,
namespace="2",
_app=self.appid),
# shard 2
key_range.KeyRange(key_start=testutil.key("5", namespace="0"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="0",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("5", namespace="1"),
key_end=testutil.key("6", namespace="1"),
direction="ASC",
include_start=True,
include_end=False,
namespace="1",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("6", namespace="2"),
key_end=testutil.key("7", namespace="2"),
direction="ASC",
include_start=True,
include_end=False,
namespace="2",
_app=self.appid),
# shard 3
key_range.KeyRange(key_start=testutil.key("6", namespace="1"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="1",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("7", namespace="2"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="2",
_app=self.appid),
]
kranges_by_shard = (self.reader_cls._to_key_ranges_by_shard(
self.appid, namespaces, shards,
model.QuerySpec(entity_kind="TestEntity")))
self.assertEquals(shards, len(kranges_by_shard))
expected.sort()
results = []
for kranges in kranges_by_shard:
results.extend(list(kranges))
results.sort()
self.assertEquals(expected, results)
def testToKeyRangesByShard_UnevenNamespaces(self):
namespaces = [str(i) for i in range(3)]
testutil._create_entities(range(10), {"5": 5}, namespaces[0])
testutil._create_entities(range(10), {"5": 5, "6": 6}, namespaces[1])
testutil._create_entities(range(10), {
"5": 5,
"6": 6,
"7": 7
}, namespaces[2])
shards = 3
expected = [
# shard 1
key_range.KeyRange(key_start=None,
key_end=testutil.key("5", namespace="0"),
direction="ASC",
include_start=False,
include_end=False,
namespace="0",
_app=self.appid),
key_range.KeyRange(key_start=None,
key_end=testutil.key("5", namespace="1"),
direction="ASC",
include_start=False,
include_end=False,
namespace="1",
_app=self.appid),
key_range.KeyRange(key_start=None,
key_end=testutil.key("6", namespace="2"),
direction="ASC",
include_start=False,
include_end=False,
namespace="2",
_app=self.appid),
# shard 2
key_range.KeyRange(key_start=testutil.key("5", namespace="0"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="0",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("5", namespace="1"),
key_end=testutil.key("6", namespace="1"),
direction="ASC",
include_start=True,
include_end=False,
namespace="1",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("6", namespace="2"),
key_end=testutil.key("7", namespace="2"),
direction="ASC",
include_start=True,
include_end=False,
namespace="2",
_app=self.appid),
# shard 3
key_range.KeyRange(key_start=testutil.key("6", namespace="1"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="1",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("7", namespace="2"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="2",
_app=self.appid),
]
kranges_by_shard = (self.reader_cls._to_key_ranges_by_shard(
self.appid, namespaces, shards,
model.QuerySpec(entity_kind="TestEntity")))
self.assertEquals(shards, len(kranges_by_shard))
expected.sort()
results = []
for kranges in kranges_by_shard:
results.extend(list(kranges))
results.sort()
self.assertEquals(expected, results)