def _iter_key_range(self, k_range):
raw_entity_kind = util.get_short_name(self._entity_kind)
query = k_range.make_ascending_datastore_query(raw_entity_kind,
keys_only=True)
for key in query.Run(config=datastore_query.QueryOptions(
batch_size=self._batch_size)):
yield key, key
def _split_input_from_namespace(cls, app, namespace, entity_kind_name,
shard_count):
"""Return KeyRange objects. Helper for _split_input_from_params."""
raw_entity_kind = util.get_short_name(entity_kind_name)
ds_query = datastore.Query(kind=raw_entity_kind,
namespace=namespace,
_app=app,
keys_only=True)
ds_query.Order("__key__")
first_entity_key_list = ds_query.Get(1)
if not first_entity_key_list:
logging.warning("Could not retrieve an entity of type %s." %
raw_entity_kind)
return []
first_entity_key = first_entity_key_list[0]
ds_query.Order(("__key__", datastore.Query.DESCENDING))
try:
last_entity_key, = ds_query.Get(1)
except db.NeedIndexError, e:
logging.warning("Cannot create accurate approximation of keyspace, "
"guessing instead. Please address this problem: %s", e)
last_entity_key = key_range.KeyRange.guess_end_key(raw_entity_kind,
first_entity_key)
def __iter__(self):
"""Create a generator for low level entities in the range.
Iterating through entries moves query range past the consumed entries.
Yields:
next entry.
"""
raw_entity_kind = util.get_short_name(self._entity_kind)
while True:
if self._current_key_range is None:
break
while True:
query = self._current_key_range.make_ascending_datastore_query(
raw_entity_kind)
results = query.Get(limit=self._batch_size)
if not results:
self._advance_key_range()
break
for entity in results:
self._current_key_range.advance(entity.key())
yield entity
def _iter_key_range(self, k_range):
raw_entity_kind = util.get_short_name(self._entity_kind)
query = k_range.make_ascending_datastore_query(
raw_entity_kind)
for entity in query.Run(
config=datastore_query.QueryOptions(batch_size=self._batch_size)):
yield entity.key(), entity
def __iter__(self):
"""Create a generator for low level entities in the range.
Iterating through entries moves query range past the consumed entries.
Yields:
next entry.
"""
raw_entity_kind = util.get_short_name(self._entity_kind)
while True:
if self._current_key_range is None:
break
while True:
query = self._current_key_range.make_ascending_datastore_query(
raw_entity_kind)
results = query.Get(limit=self._batch_size)
if not results:
self._advance_key_range()
break
for entity in results:
self._current_key_range.advance(entity.key())
yield entity
def _split_input_from_namespace(cls, app, namespace, entity_kind_name,
shard_count):
"""Return KeyRange objects. Helper for _split_input_from_params."""
raw_entity_kind = util.get_short_name(entity_kind_name)
ds_query = datastore.Query(kind=raw_entity_kind,
namespace=namespace,
_app=app,
keys_only=True)
ds_query.Order("__key__")
first_entity_key_list = ds_query.Get(1)
if not first_entity_key_list:
logging.warning("Could not retrieve an entity of type %s." %
raw_entity_kind)
return []
first_entity_key = first_entity_key_list[0]
ds_query.Order(("__key__", datastore.Query.DESCENDING))
try:
last_entity_key, = ds_query.Get(1)
except db.NeedIndexError, e:
logging.warning(
"Cannot create accurate approximation of keyspace, "
"guessing instead. Please address this problem: %s", e)
last_entity_key = key_range.KeyRange.guess_end_key(
raw_entity_kind, first_entity_key)
def _get_raw_entity_kind(cls, model_classpath):
entity_type = util.for_name(model_classpath)
if isinstance(entity_type, db.Model):
return entity_type.kind()
elif isinstance(entity_type, (ndb.Model, ndb.MetaModel)):
return entity_type._get_kind()
else:
return util.get_short_name(model_classpath)
def _get_raw_entity_kind(cls, model_classpath):
entity_type = util.for_name(model_classpath)
if isinstance(entity_type, db.Model):
return entity_type.kind()
elif isinstance(entity_type, (ndb.Model, ndb.MetaModel)):
return entity_type._get_kind()
else:
return util.get_short_name(model_classpath)
def _split_input_from_namespace(cls, app, namespace, entity_kind_name,
shard_count):
"""Return KeyRange objects. Helper for _split_input_from_params."""
raw_entity_kind = util.get_short_name(entity_kind_name)
if shard_count == 1:
return [key_range.KeyRange(namespace=namespace, _app=app)]
ds_query = datastore.Query(kind=raw_entity_kind,
namespace=namespace,
_app=app,
keys_only=True)
ds_query.Order("__scatter__")
random_keys = ds_query.Get(shard_count * cls._OVERSAMPLING_FACTOR)
if not random_keys:
return [key_range.KeyRange(namespace=namespace, _app=app)]
else:
random_keys = cls._choose_split_points(random_keys, shard_count)
key_ranges = []
key_ranges.append(key_range.KeyRange(
key_start=None,
key_end=random_keys[0],
direction=key_range.KeyRange.ASC,
include_start=False,
include_end=False,
namespace=namespace,
_app=app))
for i in range(0, len(random_keys) - 1):
key_ranges.append(key_range.KeyRange(
key_start=random_keys[i],
key_end=random_keys[i+1],
direction=key_range.KeyRange.ASC,
include_start=True,
include_end=False,
namespace=namespace,
_app=app))
key_ranges.append(key_range.KeyRange(
key_start=random_keys[-1],
key_end=None,
direction=key_range.KeyRange.ASC,
include_start=True,
include_end=False,
namespace=namespace,
_app=app))
return key_ranges
def _split_input_from_namespace(cls, app, namespace, entity_kind_name,
shard_count):
"""Return KeyRange objects. Helper for _split_input_from_params."""
raw_entity_kind = util.get_short_name(entity_kind_name)
if shard_count == 1:
return [key_range.KeyRange(namespace=namespace, _app=app)]
ds_query = datastore.Query(kind=raw_entity_kind,
namespace=namespace,
_app=app,
keys_only=True)
ds_query.Order("__scatter__")
random_keys = ds_query.Get(shard_count * cls._OVERSAMPLING_FACTOR)
if not random_keys:
return [key_range.KeyRange(namespace=namespace, _app=app)]
random_keys.sort()
split_points_count = shard_count - 1
if len(random_keys) > split_points_count:
random_keys = [
random_keys[len(random_keys) * i / split_points_count]
for i in range(split_points_count)
]
key_ranges = []
key_ranges.append(
key_range.KeyRange(key_start=None,
key_end=random_keys[0],
direction=key_range.KeyRange.ASC,
include_start=False,
include_end=False,
namespace=namespace))
for i in range(0, len(random_keys) - 1):
key_ranges.append(
key_range.KeyRange(key_start=random_keys[i],
key_end=random_keys[i + 1],
direction=key_range.KeyRange.ASC,
include_start=True,
include_end=False,
namespace=namespace))
key_ranges.append(
key_range.KeyRange(key_start=random_keys[-1],
key_end=None,
direction=key_range.KeyRange.ASC,
include_start=True,
include_end=False,
namespace=namespace))
return key_ranges
def _split_input_from_namespace(cls, app, namespace, entity_kind_name,
shard_count):
"""Return KeyRange objects. Helper for _split_input_from_params.
If there are not enough Entities to make all of the given shards, the
returned list of KeyRanges will include Nones. The returned list will
contain KeyRanges ordered lexographically with any Nones appearing at the
end.
"""
raw_entity_kind = util.get_short_name(entity_kind_name)
if shard_count == 1:
return [key_range.KeyRange(namespace=namespace, _app=app)]
ds_query = datastore.Query(kind=raw_entity_kind,
namespace=namespace,
_app=app,
keys_only=True)
ds_query.Order("__scatter__")
random_keys = ds_query.Get(shard_count * cls._OVERSAMPLING_FACTOR)
if not random_keys or len(random_keys) < shard_count:
return ([key_range.KeyRange(namespace=namespace, _app=app)] +
[None] * (shard_count - 1))
else:
random_keys = cls._choose_split_points(random_keys, shard_count)
key_ranges = []
key_ranges.append(key_range.KeyRange(
key_start=None,
key_end=random_keys[0],
direction=key_range.KeyRange.ASC,
include_start=False,
include_end=False,
namespace=namespace,
_app=app))
for i in range(0, len(random_keys) - 1):
key_ranges.append(key_range.KeyRange(
key_start=random_keys[i],
key_end=random_keys[i+1],
direction=key_range.KeyRange.ASC,
include_start=True,
include_end=False,
namespace=namespace,
_app=app))
key_ranges.append(key_range.KeyRange(
key_start=random_keys[-1],
key_end=None,
direction=key_range.KeyRange.ASC,
include_start=True,
include_end=False,
namespace=namespace,
_app=app))
return key_ranges