def validate(cls, job_config): """Validate mapper specification. Args: job_config: map_job.JobConfig. Raises: BadReaderParamsError: if the specification is invalid for any reason such as missing the bucket name or providing an invalid bucket name. """ reader_params = job_config.input_reader_params if cls.BUCKET_NAME_PARAM not in reader_params: raise errors.BadReaderParamsError( "%s is required for Google Cloud Storage" % cls.BUCKET_NAME_PARAM) try: cloudstorage.validate_bucket_name( reader_params[cls.BUCKET_NAME_PARAM]) except ValueError as error: raise errors.BadReaderParamsError("Bad bucket name, %s" % (error)) if cls.OBJECT_NAMES_PARAM not in reader_params: raise errors.BadReaderParamsError( "%s is required for Google Cloud Storage" % cls.OBJECT_NAMES_PARAM) filenames = reader_params[cls.OBJECT_NAMES_PARAM] if not isinstance(filenames, list): raise errors.BadReaderParamsError( "Object name list is not a list but a %s" % filenames.__class__.__name__) for filename in filenames: if not isinstance(filename, six.string_types): raise errors.BadReaderParamsError( "Object name is not a string but a %s" % filename.__class__.__name__) if cls.DELIMITER_PARAM in reader_params: delimiter = reader_params[cls.DELIMITER_PARAM] if not isinstance(delimiter, six.string_types): raise errors.BadReaderParamsError( "%s is not a string but a %s" % (cls.DELIMITER_PARAM, type(delimiter))) if cls.BUFFER_SIZE_PARAM in reader_params: buffer_size = reader_params[cls.BUFFER_SIZE_PARAM] if not isinstance(buffer_size, int): raise errors.BadReaderParamsError( "%s is not an int but a %s" % (cls.BUFFER_SIZE_PARAM, type(buffer_size))) if cls.PATH_FILTER_PARAM in reader_params: path_filter = reader_params[cls.PATH_FILTER_PARAM] if not isinstance(path_filter, PathFilter): raise errors.BadReaderParamsError( "%s is not an instance of PathFilter but %s." % (cls.PATH_FILTER_PARAM, type(path_filter)))
def validate(cls, job_config): """Validate mapper specification. Args: job_config: map_job.JobConfig. Raises: BadReaderParamsError: if the specification is invalid for any reason such as missing the bucket name or providing an invalid bucket name. """ reader_params = job_config.input_reader_params if cls.BUCKET_NAME_PARAM not in reader_params: raise errors.BadReaderParamsError("%s is required for Google Cloud Storage" % cls.BUCKET_NAME_PARAM) try: cloudstorage.validate_bucket_name(reader_params[cls.BUCKET_NAME_PARAM]) except ValueError, error: raise errors.BadReaderParamsError("Bad bucket name, %s" % (error))
def validate(cls, mapper_spec): """Validate mapper specification. Args: mapper_spec: an instance of model.MapperSpec. Raises: BadWriterParamsError: if the specification is invalid for any reason such as missing the bucket name or providing an invalid bucket name. """ writer_spec = cls.get_params(mapper_spec, allow_old=False) if cls.BUCKET_NAME_PARAM not in writer_spec: raise errors.BadWriterParamsError( "%s is required for Google Cloud Storage" % cls.BUCKET_NAME_PARAM) try: cloudstorage.validate_bucket_name( writer_spec[cls.BUCKET_NAME_PARAM]) except ValueError, error: raise errors.BadWriterParamsError("Bad bucket name, %s" % (error))
def validate(cls, job_config): """Validate mapper specification. Args: job_config: map_job.JobConfig. Raises: BadReaderParamsError: if the specification is invalid for any reason such as missing the bucket name or providing an invalid bucket name. """ reader_params = job_config.input_reader_params if cls.BUCKET_NAME_PARAM not in reader_params: raise errors.BadReaderParamsError( "%s is required for Google Cloud Storage" % cls.BUCKET_NAME_PARAM) try: cloudstorage.validate_bucket_name( reader_params[cls.BUCKET_NAME_PARAM]) except ValueError, error: raise errors.BadReaderParamsError("Bad bucket name, %s" % (error))