def testParseBool(self): self.assertEquals(True, util.parse_bool(True)) self.assertEquals(False, util.parse_bool(False)) self.assertEquals(True, util.parse_bool("True")) self.assertEquals(False, util.parse_bool("False")) self.assertEquals(True, util.parse_bool(1)) self.assertEquals(False, util.parse_bool(0)) self.assertEquals(True, util.parse_bool("on")) self.assertEquals(False, util.parse_bool("off"))
def split_input(cls, mapper_spec): """Splits query into shards without fetching query results. Tries as best as it can to split the whole query result set into equal shards. Due to difficulty of making the perfect split, resulting shards' sizes might differ significantly from each other. The actual number of shards might also be less then requested (even 1), though it is never greater. Current implementation does key-lexicographic order splitting. It requires query not to specify any __key__-based ordering. If an index for query.order('-__key__') query is not present, an inaccurate guess at sharding will be made by splitting the full key range. Args: mapper_spec: MapperSpec with params containing 'entity_kind'. May also have 'batch_size' in the params to specify the number of entities to process in each batch. Returns: A list of InputReader objects of length <= number_of_shards. These may be DatastoreInputReader or DatastoreKeyInputReader objects. Raises: BadReaderParamsError: required parameters are missing or invalid. """ if mapper_spec.input_reader_class() != cls: raise BadReaderParamsError("Input reader class mismatch") params = mapper_spec.params if cls.ENTITY_KIND_PARAM not in params: raise BadReaderParamsError( "Missing mapper parameter 'entity_kind'") entity_kind_name = params[cls.ENTITY_KIND_PARAM] shard_count = mapper_spec.shard_count app = params.get(cls._APP_PARAM) # keys_only remains for backwards compatability. It may go away. keys_only = util.parse_bool(params.get(cls.KEYS_ONLY_PARAM, False)) if keys_only: raise BadReaderParamsError("The keys_only parameter is obsolete. " "Use DatastoreKeyInputReader instead.") # Fail fast if Model cannot be located. util.for_name(entity_kind_name) return cls._split_input_from_params(app, entity_kind_name, params, shard_count)
def split_input(cls, mapper_spec): """Splits query into shards without fetching query results. Tries as best as it can to split the whole query result set into equal shards. Due to difficulty of making the perfect split, resulting shards' sizes might differ significantly from each other. The actual number of shards might also be less then requested (even 1), though it is never greater. Current implementation does key-lexicographic order splitting. It requires query not to specify any __key__-based ordering. If an index for query.order('-__key__') query is not present, an inaccurate guess at sharding will be made by splitting the full key range. Args: mapper_spec: MapperSpec with params containing 'entity_kind'. May also have 'batch_size' in the params to specify the number of entities to process in each batch. Returns: A list of InputReader objects of length <= number_of_shards. These may be DatastoreInputReader or DatastoreKeyInputReader objects. Raises: BadReaderParamsError: required parameters are missing or invalid. """ if mapper_spec.input_reader_class() != cls: raise BadReaderParamsError("Input reader class mismatch") params = mapper_spec.params if cls.ENTITY_KIND_PARAM not in params: raise BadReaderParamsError("Missing mapper parameter 'entity_kind'") entity_kind_name = params[cls.ENTITY_KIND_PARAM] shard_count = mapper_spec.shard_count app = params.get(cls._APP_PARAM) # keys_only remains for backwards compatability. It may go away. keys_only = util.parse_bool(params.get(cls.KEYS_ONLY_PARAM, False)) if keys_only: raise BadReaderParamsError("The keys_only parameter is obsolete. " "Use DatastoreKeyInputReader instead.") # Fail fast if Model cannot be located. util.for_name(entity_kind_name) return cls._split_input_from_params( app, entity_kind_name, params, shard_count)
def validate(cls, mapper_spec): """Validates mapper spec and all mapper parameters. Args: mapper_spec: The MapperSpec for this InputReader. Raises: BadReaderParamsError: required parameters are missing or invalid. """ cls._common_validate(mapper_spec) params = mapper_spec.params keys_only = util.parse_bool(params.get(cls.KEYS_ONLY_PARAM, False)) if keys_only: raise BadReaderParamsError("The keys_only parameter is obsolete. " "Use DatastoreKeyInputReader instead.") entity_kind_name = params[cls.ENTITY_KIND_PARAM] # Fail fast if Model cannot be located. try: util.for_name(entity_kind_name) except ImportError, e: raise BadReaderParamsError("Bad entity kind: %s" % e)