def testCursors(self):
qs = model.QuerySpec(TestModel, model_class_path=ENTITY_KIND)
kr = key_range.KeyRange(key_start=key(1),
key_end=key(10000),
direction="ASC")
json = {
'key_range': kr.to_json(),
'query_spec': qs.to_json(),
'cursor': None
}
entities = []
while True:
model_iter = DjangoModelIterator.from_json(json)
c = False
count = 0
for entity in model_iter:
count += 1
entities.append(entity)
if count == 10:
c = True
break
if c:
json = model_iter.to_json()
else:
break
self.assertEquals(100, len(entities))
self.assertEquals(self.expected_entities, entities)
def _create_iter(self, iter_cls, entity_kind):
kranges = [key_range.KeyRange(namespace=ns) for ns in self.namespaces]
kranges = key_ranges.KeyRangesFactory.create_from_list(kranges)
query_spec = model.QuerySpec(
entity_kind=util.get_short_name(entity_kind),
batch_size=10,
filters=self.filters,
model_class_path=entity_kind)
itr = db_iters.RangeIteratorFactory.create_key_ranges_iterator(
kranges, query_spec, iter_cls)
return itr
def _create_iter(self, entity_kind):
query_spec = model.QuerySpec(
entity_kind=util.get_short_name(entity_kind),
batch_size=10,
filters=self.filters,
model_class_path=entity_kind)
p_range = property_range.PropertyRange(self.filters, entity_kind)
ns_range = namespace_range.NamespaceRange(self.namespaces[0],
self.namespaces[-1])
itr = db_iters.RangeIteratorFactory.create_property_range_iterator(
p_range, ns_range, query_spec)
return itr
def _get_query_spec(cls, params):
"""Construct a model.QuerySpec from model.MapperSpec."""
entity_kind = params[cls.ENTITY_KIND_PARAM]
filters = params.get(cls.FILTERS_PARAM)
app = params.get(cls._APP_PARAM)
ns = params.get(cls.NAMESPACE_PARAM)
return model.QuerySpec(
entity_kind=cls._get_raw_entity_kind(entity_kind),
keys_only=bool(params.get(cls.KEYS_ONLY_PARAM, False)),
filters=filters,
batch_size=int(params.get(cls.BATCH_SIZE_PARAM, cls._BATCH_SIZE)),
model_class_path=entity_kind,
app=app,
ns=ns)
def _create_iter(self, iter_cls, entity_kind):
key_start = db.Key.from_path(util.get_short_name(entity_kind),
"0",
namespace=self.namespace)
key_end = db.Key.from_path(util.get_short_name(entity_kind),
"999",
namespace=self.namespace)
krange = key_range.KeyRange(key_start,
key_end,
include_start=True,
include_end=True,
namespace=self.namespace)
query_spec = model.QuerySpec(
entity_kind=util.get_short_name(entity_kind),
batch_size=10,
filters=self.filters,
model_class_path=entity_kind)
return iter_cls(krange, query_spec)
def testToKeyRangesByShard_UnevenNamespaces(self):
namespaces = [str(i) for i in range(3)]
testutil._create_entities(range(10), {"5": 5}, namespaces[0])
testutil._create_entities(range(10), {"5": 5, "6": 6}, namespaces[1])
testutil._create_entities(range(10), {
"5": 5,
"6": 6,
"7": 7
}, namespaces[2])
shards = 3
expected = [
# shard 1
key_range.KeyRange(key_start=None,
key_end=testutil.key("5", namespace="0"),
direction="ASC",
include_start=False,
include_end=False,
namespace="0",
_app=self.appid),
key_range.KeyRange(key_start=None,
key_end=testutil.key("5", namespace="1"),
direction="ASC",
include_start=False,
include_end=False,
namespace="1",
_app=self.appid),
key_range.KeyRange(key_start=None,
key_end=testutil.key("6", namespace="2"),
direction="ASC",
include_start=False,
include_end=False,
namespace="2",
_app=self.appid),
# shard 2
key_range.KeyRange(key_start=testutil.key("5", namespace="0"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="0",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("5", namespace="1"),
key_end=testutil.key("6", namespace="1"),
direction="ASC",
include_start=True,
include_end=False,
namespace="1",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("6", namespace="2"),
key_end=testutil.key("7", namespace="2"),
direction="ASC",
include_start=True,
include_end=False,
namespace="2",
_app=self.appid),
# shard 3
key_range.KeyRange(key_start=testutil.key("6", namespace="1"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="1",
_app=self.appid),
key_range.KeyRange(key_start=testutil.key("7", namespace="2"),
key_end=None,
direction="ASC",
include_start=True,
include_end=False,
namespace="2",
_app=self.appid),
]
kranges_by_shard = (self.reader_cls._to_key_ranges_by_shard(
self.appid, namespaces, shards,
model.QuerySpec(entity_kind="TestEntity")))
self.assertEquals(shards, len(kranges_by_shard))
expected.sort()
results = []
for kranges in kranges_by_shard:
results.extend(list(kranges))
results.sort()
self.assertEquals(expected, results)