def validate(cls, job_config):
"""Validate mapper specification.
Args:
job_config: map_job.JobConfig.
Raises:
BadReaderParamsError: if the specification is invalid for any reason such
as missing the bucket name or providing an invalid bucket name.
"""
reader_params = job_config.input_reader_params
if cls.BUCKET_NAME_PARAM not in reader_params:
raise errors.BadReaderParamsError(
"%s is required for Google Cloud Storage" %
cls.BUCKET_NAME_PARAM)
try:
cloudstorage.validate_bucket_name(
reader_params[cls.BUCKET_NAME_PARAM])
except ValueError as error:
raise errors.BadReaderParamsError("Bad bucket name, %s" % (error))
if cls.OBJECT_NAMES_PARAM not in reader_params:
raise errors.BadReaderParamsError(
"%s is required for Google Cloud Storage" %
cls.OBJECT_NAMES_PARAM)
filenames = reader_params[cls.OBJECT_NAMES_PARAM]
if not isinstance(filenames, list):
raise errors.BadReaderParamsError(
"Object name list is not a list but a %s" %
filenames.__class__.__name__)
for filename in filenames:
if not isinstance(filename, str):
raise errors.BadReaderParamsError(
"Object name is not a string but a %s" %
filename.__class__.__name__)
if cls.DELIMITER_PARAM in reader_params:
delimiter = reader_params[cls.DELIMITER_PARAM]
if not isinstance(delimiter, str):
raise errors.BadReaderParamsError(
"%s is not a string but a %s" %
(cls.DELIMITER_PARAM, type(delimiter)))
if cls.BUFFER_SIZE_PARAM in reader_params:
buffer_size = reader_params[cls.BUFFER_SIZE_PARAM]
if not isinstance(buffer_size, int):
raise errors.BadReaderParamsError(
"%s is not an int but a %s" %
(cls.BUFFER_SIZE_PARAM, type(buffer_size)))
if cls.PATH_FILTER_PARAM in reader_params:
path_filter = reader_params[cls.PATH_FILTER_PARAM]
if not isinstance(path_filter, PathFilter):
raise errors.BadReaderParamsError(
"%s is not an instance of PathFilter but %s." %
(cls.PATH_FILTER_PARAM, type(path_filter)))
def validate(cls, mapper_spec):
"""Validate mapper specification.
Args:
mapper_spec: an instance of model.MapperSpec.
Raises:
BadWriterParamsError if the specification is invalid for any reason such
as missing the bucket name or providing an invalid bucket name.
"""
writer_spec = _get_params(mapper_spec, allow_old=False)
if cls.BUCKET_NAME_PARAM not in writer_spec:
raise errors.BadWriterParamsError(
"%s is required for Google Cloud Storage" %
cls.BUCKET_NAME_PARAM)
try:
cloudstorage.validate_bucket_name(
writer_spec[cls.BUCKET_NAME_PARAM])
except ValueError, error:
raise errors.BadWriterParamsError("Bad bucket name, %s" % (error))
def validate(cls, mapper_spec):
"""Validate mapper specification.
Args:
mapper_spec: an instance of model.MapperSpec.
Raises:
BadWriterParamsError if the specification is invalid for any reason such
as missing the bucket name or providing an invalid bucket name.
"""
writer_spec = _get_params(mapper_spec, allow_old=False)
if cls.BUCKET_NAME_PARAM not in writer_spec:
raise errors.BadWriterParamsError(
"%s is required for Google Cloud Storage" %
cls.BUCKET_NAME_PARAM)
try:
cloudstorage.validate_bucket_name(
writer_spec[cls.BUCKET_NAME_PARAM])
except ValueError, error:
raise errors.BadWriterParamsError("Bad bucket name, %s" % (error))
def validate(cls, job_config):
"""Validate mapper specification.
Args:
job_config: map_job.JobConfig.
Raises:
BadReaderParamsError: if the specification is invalid for any reason such
as missing the bucket name or providing an invalid bucket name.
"""
reader_params = job_config.input_reader_params
if cls.BUCKET_NAME_PARAM not in reader_params:
raise errors.BadReaderParamsError(
"%s is required for Google Cloud Storage" %
cls.BUCKET_NAME_PARAM)
try:
cloudstorage.validate_bucket_name(
reader_params[cls.BUCKET_NAME_PARAM])
except ValueError, error:
raise errors.BadReaderParamsError("Bad bucket name, %s" % (error))
def validate(cls, job_config):
"""Validate mapper specification.
Args:
job_config: map_job.JobConfig.
Raises:
BadReaderParamsError: if the specification is invalid for any reason such
as missing the bucket name or providing an invalid bucket name.
"""
reader_params = job_config.input_reader_params
if cls.BUCKET_NAME_PARAM not in reader_params:
raise errors.BadReaderParamsError(
"%s is required for Google Cloud Storage" %
cls.BUCKET_NAME_PARAM)
try:
cloudstorage.validate_bucket_name(
reader_params[cls.BUCKET_NAME_PARAM])
except ValueError as error:
raise errors.BadReaderParamsError("Bad bucket name, %s" % (error))
if cls.OBJECT_NAMES_PARAM not in reader_params:
raise errors.BadReaderParamsError(
"%s is required for Google Cloud Storage" %
cls.OBJECT_NAMES_PARAM)
filenames = reader_params[cls.OBJECT_NAMES_PARAM]
if not isinstance(filenames, list):
raise errors.BadReaderParamsError(
"Object name list is not a list but a %s" %
filenames.__class__.__name__)
for filename in filenames:
if not isinstance(filename, str):
raise errors.BadReaderParamsError(
"Object name is not a string but a %s" %
filename.__class__.__name__)
if cls.DELIMITER_PARAM in reader_params:
delimiter = reader_params[cls.DELIMITER_PARAM]
if not isinstance(delimiter, str):
raise errors.BadReaderParamsError(
"%s is not a string but a %s" %
(cls.DELIMITER_PARAM, type(delimiter)))
if cls.BUFFER_SIZE_PARAM in reader_params:
buffer_size = reader_params[cls.BUFFER_SIZE_PARAM]
if not isinstance(buffer_size, int):
raise errors.BadReaderParamsError(
"%s is not an int but a %s" %
(cls.BUFFER_SIZE_PARAM, type(buffer_size)))
if cls.PATH_FILTER_PARAM in reader_params:
path_filter = reader_params[cls.PATH_FILTER_PARAM]
if not isinstance(path_filter, PathFilter):
raise errors.BadReaderParamsError(
"%s is not an instance of PathFilter but %s." %
(cls.PATH_FILTER_PARAM, type(path_filter)))
