def testParseBool(self):
self.assertEquals(True, util.parse_bool(True))
self.assertEquals(False, util.parse_bool(False))
self.assertEquals(True, util.parse_bool("True"))
self.assertEquals(False, util.parse_bool("False"))
self.assertEquals(True, util.parse_bool(1))
self.assertEquals(False, util.parse_bool(0))
self.assertEquals(True, util.parse_bool("on"))
self.assertEquals(False, util.parse_bool("off"))
def split_input(cls, mapper_spec):
"""Splits query into shards without fetching query results.
Tries as best as it can to split the whole query result set into equal
shards. Due to difficulty of making the perfect split, resulting shards'
sizes might differ significantly from each other. The actual number of
shards might also be less then requested (even 1), though it is never
greater.
Current implementation does key-lexicographic order splitting. It requires
query not to specify any __key__-based ordering. If an index for
query.order('-__key__') query is not present, an inaccurate guess at
sharding will be made by splitting the full key range.
Args:
mapper_spec: MapperSpec with params containing 'entity_kind'.
May also have 'batch_size' in the params to specify the number
of entities to process in each batch.
Returns:
A list of InputReader objects of length <= number_of_shards. These
may be DatastoreInputReader or DatastoreKeyInputReader objects.
Raises:
BadReaderParamsError: required parameters are missing or invalid.
"""
if mapper_spec.input_reader_class() != cls:
raise BadReaderParamsError("Input reader class mismatch")
params = mapper_spec.params
if cls.ENTITY_KIND_PARAM not in params:
raise BadReaderParamsError(
"Missing mapper parameter 'entity_kind'")
entity_kind_name = params[cls.ENTITY_KIND_PARAM]
shard_count = mapper_spec.shard_count
app = params.get(cls._APP_PARAM)
# keys_only remains for backwards compatability. It may go away.
keys_only = util.parse_bool(params.get(cls.KEYS_ONLY_PARAM, False))
if keys_only:
raise BadReaderParamsError("The keys_only parameter is obsolete. "
"Use DatastoreKeyInputReader instead.")
# Fail fast if Model cannot be located.
util.for_name(entity_kind_name)
return cls._split_input_from_params(app, entity_kind_name, params,
shard_count)
def split_input(cls, mapper_spec):
"""Splits query into shards without fetching query results.
Tries as best as it can to split the whole query result set into equal
shards. Due to difficulty of making the perfect split, resulting shards'
sizes might differ significantly from each other. The actual number of
shards might also be less then requested (even 1), though it is never
greater.
Current implementation does key-lexicographic order splitting. It requires
query not to specify any __key__-based ordering. If an index for
query.order('-__key__') query is not present, an inaccurate guess at
sharding will be made by splitting the full key range.
Args:
mapper_spec: MapperSpec with params containing 'entity_kind'.
May also have 'batch_size' in the params to specify the number
of entities to process in each batch.
Returns:
A list of InputReader objects of length <= number_of_shards. These
may be DatastoreInputReader or DatastoreKeyInputReader objects.
Raises:
BadReaderParamsError: required parameters are missing or invalid.
"""
if mapper_spec.input_reader_class() != cls:
raise BadReaderParamsError("Input reader class mismatch")
params = mapper_spec.params
if cls.ENTITY_KIND_PARAM not in params:
raise BadReaderParamsError("Missing mapper parameter 'entity_kind'")
entity_kind_name = params[cls.ENTITY_KIND_PARAM]
shard_count = mapper_spec.shard_count
app = params.get(cls._APP_PARAM)
# keys_only remains for backwards compatability. It may go away.
keys_only = util.parse_bool(params.get(cls.KEYS_ONLY_PARAM, False))
if keys_only:
raise BadReaderParamsError("The keys_only parameter is obsolete. "
"Use DatastoreKeyInputReader instead.")
# Fail fast if Model cannot be located.
util.for_name(entity_kind_name)
return cls._split_input_from_params(
app, entity_kind_name, params, shard_count)
def validate(cls, mapper_spec):
"""Validates mapper spec and all mapper parameters.
Args:
mapper_spec: The MapperSpec for this InputReader.
Raises:
BadReaderParamsError: required parameters are missing or invalid.
"""
cls._common_validate(mapper_spec)
params = mapper_spec.params
keys_only = util.parse_bool(params.get(cls.KEYS_ONLY_PARAM, False))
if keys_only:
raise BadReaderParamsError("The keys_only parameter is obsolete. "
"Use DatastoreKeyInputReader instead.")
entity_kind_name = params[cls.ENTITY_KIND_PARAM]
# Fail fast if Model cannot be located.
try:
util.for_name(entity_kind_name)
except ImportError, e:
raise BadReaderParamsError("Bad entity kind: %s" % e)
def validate(cls, mapper_spec):
"""Validates mapper spec and all mapper parameters.
Args:
mapper_spec: The MapperSpec for this InputReader.
Raises:
BadReaderParamsError: required parameters are missing or invalid.
"""
cls._common_validate(mapper_spec)
params = mapper_spec.params
keys_only = util.parse_bool(params.get(cls.KEYS_ONLY_PARAM, False))
if keys_only:
raise BadReaderParamsError("The keys_only parameter is obsolete. "
"Use DatastoreKeyInputReader instead.")
entity_kind_name = params[cls.ENTITY_KIND_PARAM]
# Fail fast if Model cannot be located.
try:
util.for_name(entity_kind_name)
except ImportError, e:
raise BadReaderParamsError("Bad entity kind: %s" % e)
