def validate(self):
super().validate()
if not self.config.get('tfdl_config'):
raise rv.ConfigError('You must specify a template for the backend '
"configuration - use 'with_template'.")
if not isinstance(self.config.get('tfdl_config'), dict):
raise rv.ConfigError(
'tfdl_config must be of type dict, got {}'.format(
type(self.config.get('tfdl_config'))))
if self.require_task and not self.task:
raise rv.ConfigError('You must specify the task this backend '
"is for - use 'with_task'.")
if self.require_task and not isinstance(self.task,
SemanticSegmentationConfig):
raise rv.ConfigError('Task set with with_task must be of type'
' SemanticSegmentationConfig, got {}.'.format(
type(self.task)))
if self.task and (self.task.predict_chip_size != self.task.chip_size):
raise rv.ConfigError(
'TFDeepLab Backend does not currently support predict_chip_size'
' that is not equal to training chip_size.')
return True
def validate(self):
super().validate()
if self.task is None:
raise rv.ConfigError(
'task not set for EvalCommandConfig. Use with_task or '
'with_experiment')
check_task_type(self.task)
if self.scenes is None:
raise rv.ConfigError(
'scenes not set for EvalCommandConfig. Use with_scenes or '
'with_experiment')
check_scenes_type(self.scenes)
if self.evaluators is None:
raise rv.ConfigError(
'evaluators not set. Use with_evaluators or with_experiment')
if not isinstance(self.evaluators, list):
raise rv.ConfigError(
'evaluators must be a list of EvaluatorConfig objects, got {}'.
format(type(self.evaluators)))
for evaluator in self.evaluators:
if not issubclass(type(evaluator), EvaluatorConfig):
if not isinstance(evaluator, str):
raise rv.ConfigError(
'evaluators must be a subclass of EvaluatorConfig or string,'
' got {}'.format(type(evaluator)))
def validate(self):
vector_output = self.config.get('vector_output')
if vector_output and not isinstance(vector_output, list):
for vo in vector_output:
if not hasattr(vo, 'mode'):
raise rv.ConfigError(
'The attribute vector_output of'
' SemanticSegmentationRasterStoreConfig'
' must be either trivial, a protobuf configuration'
' object, or a list of'
' appropriate dictionaries.')
if vo.mode not in self.valid_modes:
raise rv.ConfigError(
'mode key in vector_output dictionary must be one of {}'
.format(self.valid_modes))
elif vector_output and isinstance(vector_output, list):
for vo in vector_output:
if not isinstance(vo, dict):
raise rv.ConfigError(
'The attribute vector_output of'
' SemanticSegmentationRasterStoreConfig'
' must be either trivial, a protobuf configuration'
' object, or a list of'
' appropriate dictionaries.')
if 'mode' not in vo.keys(
) or vo['mode'] not in self.valid_modes:
raise rv.ConfigError(
'mode key in vector_output dictionary must be one of {}'
.format(self.valid_modes))
def validate(self):
super().validate()
vector_source = self.config.get('vector_source')
if vector_source is None:
raise rv.ConfigError(
'You must specify a vector_source for the RasterizedSourceConfig. '
'Use "with_vector_source"')
if not isinstance(vector_source, VectorSourceConfig):
raise rv.ConfigError(
'vector source must be a child of class VectorSourceConfig, got {}'
.format(type(vector_source)))
if vector_source.has_null_class_bufs():
raise rv.ConfigError(
'Setting buffer to None for a class in the vector_source is not allowed '
'for RasterizedSourceConfig.')
if self.config.get('rasterizer_options') is None:
raise rv.ConfigError(
'You must configure the rasterizer for the RasterizedSourceConfig. '
'Use "with_rasterizer_options"')
if not isinstance(self.config.get('rasterizer_options'),
RasterizedSourceConfig.RasterizerOptions):
raise rv.ConfigError(
'rasterizer_options must be of type '
'GeoJSONSourceConfig.RasterizerOptions, got'.format(
type(self.config.get('rasterizer_options'))))
def validate(self):
if 'class_map' not in self.config:
raise rv.ConfigError('Class map required for this task. '
'Use "with_classes"')
if not isinstance(self.config['class_map'], ClassMap):
raise rv.ConfigError(
'Class map set with "with_classes" must be of type ClassMap, got {}'
.format(type(self.config['class_map'])))
def validate(self):
if self.config.get('class_map') is None:
raise rv.ConfigError(
'class_map not set for ClassificationEvaluatorConfig. '
'Use "with_class_map".')
if not isinstance(self.config.get('class_map'), ClassMap):
raise rv.ConfigError(
'class_map set with "with_class_map" must be of type ClassMap, got {}'.
format(type(self.config.get('class_map'))))
def validate(self):
super().validate()
if self.scenes is None:
raise rv.ConfigError('scenes not set for AnalyzeCommandConfig. Use'
' with_scenes or with_experiment')
check_scenes_type(self.scenes)
if self.analyzers is None:
raise rv.ConfigError(
'analyzers not set. Use with_analyzers or with_experiment')
check_analyzers_type(self.analyzers)
def check_analyzers_type(analyzers):
if not isinstance(analyzers, list):
raise rv.ConfigError(
'analyzers must be a list of AnalyzerConfig objects, got {}'.
format(type(analyzers)))
for analyzer in analyzers:
if not issubclass(type(analyzer), AnalyzerConfig):
if not isinstance(analyzer, str):
raise rv.ConfigError(
'analyzers must be of class AnalyzerConfig or string, got {}'
.format(type(analyzer)))
def check_scenes_type(scenes):
if not isinstance(scenes, list):
raise rv.ConfigError(
'scenes must be a list of SceneConfig objects, got {}'.format(
type(scenes)))
for scene in scenes:
if not isinstance(scene, SceneConfig):
if not isinstance(scene, str):
raise rv.ConfigError(
'scene must be a SceneConfig object or str, got {}'.format(
type(scene)))
def validate(self):
super().validate()
if self.task is None:
raise rv.ConfigError('Task not set for TrainCommandConfig. Use '
'with_task or with_experiment')
check_task_type(self.task)
if self.backend is None:
raise rv.ConfigError('Backend not set for TrainCommandConfig. Use '
'with_task or with_experiment')
check_backend_type(self.backend)
def validate(self):
source = self.config.get('source')
if source is None:
raise rv.ConfigError(
'You must set the source for SemanticSegmentationLabelSourceConfig'
' Use "with_raster_source".')
if not isinstance(source, (str, RasterSourceConfig)):
raise rv.ConfigError(
'raster_source must be of type str or RasterSourceConfig, got {}'.
format(type(source)))
def validate(self):
super().validate()
if self.config.get('uris') is None:
raise rv.ConfigError(
'You must specify uris for the RasterioSourceConfig. Use '
'"with_uris".')
if not isinstance(self.config.get('uris'), list):
raise rv.ConfigError(
'uris set with "with_uris" must be a list, got {}'.format(
type(self.config.get('uris'))))
for uri in self.config.get('uris'):
if not isinstance(uri, str):
raise rv.ConfigError('uri must be a string, got {}'.format(
type(uri)))
def validate(self):
super().validate()
vector_source = self.config.get('vector_source')
if vector_source is None:
raise rv.ConfigError(
'You must set the vector_source for ObjectDetectionLabelSourceConfig'
' Use "with_vector_source".')
if not isinstance(vector_source, VectorSourceConfig):
raise rv.ConfigError(
'vector source must be a child of class VectorSourceConfig, got {}'
.format(type(vector_source)))
if vector_source.has_null_class_bufs():
raise rv.ConfigError(
'Setting buffer to None for a class in the vector_source is not allowed '
'for ObjectDetectionLabelSourceConfig.')
def with_label_source(self, label_source: Union[str, LabelSourceConfig]):
"""
Sets the raster source for this scene.
Args:
label_source: Can either be a label source configuration, or
a string. If a string, the registry will be queried
to grab the default LabelSourceConfig for the string.
Note:
A task must be set with `with_task` before calling this, if calling
with a string.
"""
b = deepcopy(self)
if isinstance(label_source, LabelSourceConfig):
b.config['label_source'] = label_source
else:
if not self.task:
raise rv.ConfigError(
"You must set a task with '.with_task' before "
'creating a default label store for {}'.format(
label_source))
provider = rv._registry.get_label_source_default_provider(
self.task.task_type, label_source)
b.config['label_source'] = provider.construct(label_source)
return b
def validate(self):
if self.config.get('uri') is None:
raise rv.ConfigError(
'GeoJSONVectorSourceConfigBuilder requires uri which '
'can be set using "with_uri".')
super().validate()
def _discover_config_file_locations(self, profile):
result = []
# Allow for user to specify specific config file
# in the RASTERVISION_CONFIG env variable.
env_specified_path = os.environ.get('RV_CONFIG')
if env_specified_path:
result.append(env_specified_path)
# Allow user to specify config directory that will
# contain profile configs in RASTERVISION_CONFIG_DIR
# env variable. Otherwise, use "$HOME/.rastervision"
env_specified_dir_path = os.environ.get('RV_CONFIG_DIR')
if env_specified_dir_path:
result.append(os.path.join(env_specified_dir_path, profile))
else:
result.append(os.path.join(self.rv_home, profile))
result.append(os.path.join(os.getcwd(), '.rastervision'))
# Filter out any that do not exist.
results_that_exist = list(filter(lambda x: os.path.exists(x), result))
# If the profile is not default, and there is no config that exists,
# then throw an error.
if not any(results_that_exist) and profile != RVConfig.DEFAULT_PROFILE:
raise rv.ConfigError('Configuration Profile {} not found. '
'Checked: {}'.format(profile,
', '.join(result)))
return results_that_exist
def load_bundle_files(self, bundle_dir):
if not self.model_uri:
raise rv.ConfigError('model_uri is not set.')
local_model_uri = os.path.join(bundle_dir, self.model_uri)
return self.to_builder() \
.with_model_uri(local_model_uri) \
.build()
def build(self):
"""Build this configuration.
Set any values into the TF object detection pipeline config as
necessary.
"""
self.validate()
b = deepcopy(self)
# Check if a pretrained model was assigned.
pretrained_model = b.config.get('pretrained_model_uri')
if pretrained_model:
b = b.with_config({'fineTuneCheckpoint': pretrained_model})
else:
b = b.with_config(
{
'fineTuneCheckpoint': ''
}, ignore_missing_keys=True)
for config_mod, ignore_missing_keys, set_missing_keys in b.config_mods:
try:
set_nested_keys(b.config['tfod_config'], config_mod,
ignore_missing_keys, set_missing_keys)
except Exception as e:
raise rv.ConfigError(
'Error setting configuration {}'.format(config_mod)) from e
return TFObjectDetectionConfig(**b.config)
def _validate(bufs):
if bufs is not None:
for c, v in bufs.items():
if not (v is None or isinstance(v, numbers.Number)):
raise rv.ConfigError(
'Buffer size {} must be a number or None.'.format(
v))
def update_for_command(self,
command_type,
experiment_config,
context=None):
if command_type == rv.PREDICT:
if not self.uri:
# Construct the URI for this prediction store,
# using the scene ID.
root = experiment_config.predict_uri
uri = None
for c in context:
if isinstance(c, rv.SceneConfig):
uri = os.path.join(root, '{}.tif'.format(c.id))
if uri:
self.uri = uri
else:
raise rv.ConfigError(
'SemanticSegmentationRasterStoreConfig has no '
'URI set, and is not associated with a SceneConfig.')
# Construct URIs for vector predictions
for vo in self.vector_output:
for c in context:
if isinstance(c,
rv.SceneConfig) and ('uri' not in vo.keys()
or not vo['uri']):
root = experiment_config.predict_uri
mode = vo['mode']
class_id = vo['class_id']
vo['uri'] = os.path.join(
root,
'{}-{}-{}.geojson'.format(c.id, class_id, mode))
def validate(self):
if self.config.get('uri') is None:
raise rv.ConfigError(
'VectorTileVectorSourceConfigBuilder requires uri which '
'can be set using "with_uri".')
if self.config.get('zoom') is None:
raise rv.ConfigError(
'VectorTileVectorSourceConfigBuilder requires zoom which '
'can be set using "with_zoom".')
# If not set explicitly, set it using default value.
if self.config.get('id_field') is None:
self.with_id_field()
super().validate()
def validate(self):
super().validate()
if self.require_task and not self.task:
raise rv.ConfigError('You must specify the task this backend '
"is for - use 'with_task'.")
return True
def save_bundle_files(self, bundle_dir):
if not self.model_uri:
raise rv.ConfigError('model_uri is not set.')
local_path, base_name = self.bundle_file(self.model_uri, bundle_dir)
new_config = self.to_builder() \
.with_model_uri(base_name) \
.build()
return (new_config, [local_path])
def load_bundle_files(self, bundle_dir):
model_uri = self.backend_opts.model_uri
if not model_uri:
raise rv.ConfigError('model_uri is not set.')
local_model_uri = join(bundle_dir, model_uri)
return self.to_builder() \
.with_model_uri(local_model_uri) \
.build()
def validate(self):
super().validate()
if not self.config.get('tfod_config'):
raise rv.ConfigError('You must specify a template for the backend '
"configuration - use 'with_template'.")
if not isinstance(self.config.get('tfod_config'), dict):
raise rv.ConfigError(
'tfod_config must be of type dict, got {}'.format(
type(self.config.get('tfod_config'))))
if self.require_task and not self.task:
raise rv.ConfigError('You must specify the task this backend '
"is for - use 'with_task'.")
if self.require_task and not isinstance(self.task,
ObjectDetectionConfig):
raise rv.ConfigError(
'Task set with with_task must be of type ObjectDetectionConfig, got {}.'.
format(type(self.task)))
return True
def validate(self):
super().validate()
if not isinstance(self.config, dict):
raise rv.ConfigError(
'config must be of type dict, got {}'.format(
type(self.config.get('config'))))
if self.require_task and not self.task:
raise rv.ConfigError('You must specify the task this backend '
"is for - use 'with_task'.")
if self.require_task and not isinstance(self.task,
SemanticSegmentationConfig):
raise rv.ConfigError('Task set with with_task must be of type'
' SemanticSegmentationConfig, got {}.'.format(
type(self.task)))
return True
def with_model_defaults(self, model_defaults_key):
"""Sets the backend configuration and pretrained model defaults
according to the model defaults configuration.
"""
model_defaults = RVConfig.get_instance().get_model_defaults()
if self.backend_type in model_defaults:
backend_defaults = model_defaults[self.backend_type]
if model_defaults_key in backend_defaults:
return self._load_model_defaults(
backend_defaults[model_defaults_key])
else:
raise rv.ConfigError('No defaults found for model key {}'
.format(model_defaults_key))
else:
raise rv.ConfigError('No model defaults for backend {}'
.format(self.backend_type))
return self
def validate(self):
super().validate()
# Segmentation masks are stored as uint8 to save space, so can only handle 256
# classes. If this is really needed, we can add an option for saving with uint16.
max_classes = 256
if len(self.config['class_map']) > max_classes:
raise rv.ConfigError(
'Cannot use more than {} classes with semantic segmentation.'.
format(max_classes))
def validate(self):
super().validate()
if not self.command_class:
raise rv.ConfigError(
'AuxCommandConfigBuilder requires the command_class be set.')
if self.config is None:
raise rv.ConfigError(
'AuxCommandConfigBuilder requires a configuration be set, either '
'through with_config or by setting a dict with the property "config" '
'in an experiment custom configuration dict with the command name '
'as the key in the experiment custom configuration')
if self.command_class.options.required_fields:
for field in self.command_class.options.required_fields:
if field not in self.config:
raise rv.ConfigError('{} command requires the field {} '
'be set in the configuration.'.format(
self.command_type, field))
def validate(self):
super().validate()
if not self.config.get('kc_config'):
raise rv.ConfigError('You must specify a template for the backend '
'configuration - use "with_template".')
if not isinstance(self.config.get('kc_config'), dict):
raise rv.ConfigError(
'kc_config must be of type dict, got {}'.format(
type(self.config.get('kc_config'))))
if self.require_task and not self.task:
raise rv.ConfigError('You must specify the task this backend '
'is for - use "with_task".')
if self.require_task and not isinstance(self.task,
ChipClassificationConfig):
raise rv.ConfigError('Task set with with_task must be of type '
'ChipClassificationConfig, got {}.'.format(
type(self.task)))