def validate(self): super().validate() if not self.config.get('tfdl_config'): raise rv.ConfigError('You must specify a template for the backend ' "configuration - use 'with_template'.") if not isinstance(self.config.get('tfdl_config'), dict): raise rv.ConfigError( 'tfdl_config must be of type dict, got {}'.format( type(self.config.get('tfdl_config')))) if self.require_task and not self.task: raise rv.ConfigError('You must specify the task this backend ' "is for - use 'with_task'.") if self.require_task and not isinstance(self.task, SemanticSegmentationConfig): raise rv.ConfigError('Task set with with_task must be of type' ' SemanticSegmentationConfig, got {}.'.format( type(self.task))) if self.task and (self.task.predict_chip_size != self.task.chip_size): raise rv.ConfigError( 'TFDeepLab Backend does not currently support predict_chip_size' ' that is not equal to training chip_size.') return True
def validate(self): super().validate() if self.task is None: raise rv.ConfigError( 'task not set for EvalCommandConfig. Use with_task or ' 'with_experiment') check_task_type(self.task) if self.scenes is None: raise rv.ConfigError( 'scenes not set for EvalCommandConfig. Use with_scenes or ' 'with_experiment') check_scenes_type(self.scenes) if self.evaluators is None: raise rv.ConfigError( 'evaluators not set. Use with_evaluators or with_experiment') if not isinstance(self.evaluators, list): raise rv.ConfigError( 'evaluators must be a list of EvaluatorConfig objects, got {}'. format(type(self.evaluators))) for evaluator in self.evaluators: if not issubclass(type(evaluator), EvaluatorConfig): if not isinstance(evaluator, str): raise rv.ConfigError( 'evaluators must be a subclass of EvaluatorConfig or string,' ' got {}'.format(type(evaluator)))
def validate(self): vector_output = self.config.get('vector_output') if vector_output and not isinstance(vector_output, list): for vo in vector_output: if not hasattr(vo, 'mode'): raise rv.ConfigError( 'The attribute vector_output of' ' SemanticSegmentationRasterStoreConfig' ' must be either trivial, a protobuf configuration' ' object, or a list of' ' appropriate dictionaries.') if vo.mode not in self.valid_modes: raise rv.ConfigError( 'mode key in vector_output dictionary must be one of {}' .format(self.valid_modes)) elif vector_output and isinstance(vector_output, list): for vo in vector_output: if not isinstance(vo, dict): raise rv.ConfigError( 'The attribute vector_output of' ' SemanticSegmentationRasterStoreConfig' ' must be either trivial, a protobuf configuration' ' object, or a list of' ' appropriate dictionaries.') if 'mode' not in vo.keys( ) or vo['mode'] not in self.valid_modes: raise rv.ConfigError( 'mode key in vector_output dictionary must be one of {}' .format(self.valid_modes))
def validate(self): super().validate() vector_source = self.config.get('vector_source') if vector_source is None: raise rv.ConfigError( 'You must specify a vector_source for the RasterizedSourceConfig. ' 'Use "with_vector_source"') if not isinstance(vector_source, VectorSourceConfig): raise rv.ConfigError( 'vector source must be a child of class VectorSourceConfig, got {}' .format(type(vector_source))) if vector_source.has_null_class_bufs(): raise rv.ConfigError( 'Setting buffer to None for a class in the vector_source is not allowed ' 'for RasterizedSourceConfig.') if self.config.get('rasterizer_options') is None: raise rv.ConfigError( 'You must configure the rasterizer for the RasterizedSourceConfig. ' 'Use "with_rasterizer_options"') if not isinstance(self.config.get('rasterizer_options'), RasterizedSourceConfig.RasterizerOptions): raise rv.ConfigError( 'rasterizer_options must be of type ' 'GeoJSONSourceConfig.RasterizerOptions, got'.format( type(self.config.get('rasterizer_options'))))
def validate(self): if 'class_map' not in self.config: raise rv.ConfigError('Class map required for this task. ' 'Use "with_classes"') if not isinstance(self.config['class_map'], ClassMap): raise rv.ConfigError( 'Class map set with "with_classes" must be of type ClassMap, got {}' .format(type(self.config['class_map'])))
def validate(self): if self.config.get('class_map') is None: raise rv.ConfigError( 'class_map not set for ClassificationEvaluatorConfig. ' 'Use "with_class_map".') if not isinstance(self.config.get('class_map'), ClassMap): raise rv.ConfigError( 'class_map set with "with_class_map" must be of type ClassMap, got {}'. format(type(self.config.get('class_map'))))
def validate(self): super().validate() if self.scenes is None: raise rv.ConfigError('scenes not set for AnalyzeCommandConfig. Use' ' with_scenes or with_experiment') check_scenes_type(self.scenes) if self.analyzers is None: raise rv.ConfigError( 'analyzers not set. Use with_analyzers or with_experiment') check_analyzers_type(self.analyzers)
def check_analyzers_type(analyzers): if not isinstance(analyzers, list): raise rv.ConfigError( 'analyzers must be a list of AnalyzerConfig objects, got {}'. format(type(analyzers))) for analyzer in analyzers: if not issubclass(type(analyzer), AnalyzerConfig): if not isinstance(analyzer, str): raise rv.ConfigError( 'analyzers must be of class AnalyzerConfig or string, got {}' .format(type(analyzer)))
def check_scenes_type(scenes): if not isinstance(scenes, list): raise rv.ConfigError( 'scenes must be a list of SceneConfig objects, got {}'.format( type(scenes))) for scene in scenes: if not isinstance(scene, SceneConfig): if not isinstance(scene, str): raise rv.ConfigError( 'scene must be a SceneConfig object or str, got {}'.format( type(scene)))
def validate(self): super().validate() if self.task is None: raise rv.ConfigError('Task not set for TrainCommandConfig. Use ' 'with_task or with_experiment') check_task_type(self.task) if self.backend is None: raise rv.ConfigError('Backend not set for TrainCommandConfig. Use ' 'with_task or with_experiment') check_backend_type(self.backend)
def validate(self): source = self.config.get('source') if source is None: raise rv.ConfigError( 'You must set the source for SemanticSegmentationLabelSourceConfig' ' Use "with_raster_source".') if not isinstance(source, (str, RasterSourceConfig)): raise rv.ConfigError( 'raster_source must be of type str or RasterSourceConfig, got {}'. format(type(source)))
def validate(self): super().validate() if self.config.get('uris') is None: raise rv.ConfigError( 'You must specify uris for the RasterioSourceConfig. Use ' '"with_uris".') if not isinstance(self.config.get('uris'), list): raise rv.ConfigError( 'uris set with "with_uris" must be a list, got {}'.format( type(self.config.get('uris')))) for uri in self.config.get('uris'): if not isinstance(uri, str): raise rv.ConfigError('uri must be a string, got {}'.format( type(uri)))
def validate(self): super().validate() vector_source = self.config.get('vector_source') if vector_source is None: raise rv.ConfigError( 'You must set the vector_source for ObjectDetectionLabelSourceConfig' ' Use "with_vector_source".') if not isinstance(vector_source, VectorSourceConfig): raise rv.ConfigError( 'vector source must be a child of class VectorSourceConfig, got {}' .format(type(vector_source))) if vector_source.has_null_class_bufs(): raise rv.ConfigError( 'Setting buffer to None for a class in the vector_source is not allowed ' 'for ObjectDetectionLabelSourceConfig.')
def with_label_source(self, label_source: Union[str, LabelSourceConfig]): """ Sets the raster source for this scene. Args: label_source: Can either be a label source configuration, or a string. If a string, the registry will be queried to grab the default LabelSourceConfig for the string. Note: A task must be set with `with_task` before calling this, if calling with a string. """ b = deepcopy(self) if isinstance(label_source, LabelSourceConfig): b.config['label_source'] = label_source else: if not self.task: raise rv.ConfigError( "You must set a task with '.with_task' before " 'creating a default label store for {}'.format( label_source)) provider = rv._registry.get_label_source_default_provider( self.task.task_type, label_source) b.config['label_source'] = provider.construct(label_source) return b
def validate(self): if self.config.get('uri') is None: raise rv.ConfigError( 'GeoJSONVectorSourceConfigBuilder requires uri which ' 'can be set using "with_uri".') super().validate()
def _discover_config_file_locations(self, profile): result = [] # Allow for user to specify specific config file # in the RASTERVISION_CONFIG env variable. env_specified_path = os.environ.get('RV_CONFIG') if env_specified_path: result.append(env_specified_path) # Allow user to specify config directory that will # contain profile configs in RASTERVISION_CONFIG_DIR # env variable. Otherwise, use "$HOME/.rastervision" env_specified_dir_path = os.environ.get('RV_CONFIG_DIR') if env_specified_dir_path: result.append(os.path.join(env_specified_dir_path, profile)) else: result.append(os.path.join(self.rv_home, profile)) result.append(os.path.join(os.getcwd(), '.rastervision')) # Filter out any that do not exist. results_that_exist = list(filter(lambda x: os.path.exists(x), result)) # If the profile is not default, and there is no config that exists, # then throw an error. if not any(results_that_exist) and profile != RVConfig.DEFAULT_PROFILE: raise rv.ConfigError('Configuration Profile {} not found. ' 'Checked: {}'.format(profile, ', '.join(result))) return results_that_exist
def load_bundle_files(self, bundle_dir): if not self.model_uri: raise rv.ConfigError('model_uri is not set.') local_model_uri = os.path.join(bundle_dir, self.model_uri) return self.to_builder() \ .with_model_uri(local_model_uri) \ .build()
def build(self): """Build this configuration. Set any values into the TF object detection pipeline config as necessary. """ self.validate() b = deepcopy(self) # Check if a pretrained model was assigned. pretrained_model = b.config.get('pretrained_model_uri') if pretrained_model: b = b.with_config({'fineTuneCheckpoint': pretrained_model}) else: b = b.with_config( { 'fineTuneCheckpoint': '' }, ignore_missing_keys=True) for config_mod, ignore_missing_keys, set_missing_keys in b.config_mods: try: set_nested_keys(b.config['tfod_config'], config_mod, ignore_missing_keys, set_missing_keys) except Exception as e: raise rv.ConfigError( 'Error setting configuration {}'.format(config_mod)) from e return TFObjectDetectionConfig(**b.config)
def _validate(bufs): if bufs is not None: for c, v in bufs.items(): if not (v is None or isinstance(v, numbers.Number)): raise rv.ConfigError( 'Buffer size {} must be a number or None.'.format( v))
def update_for_command(self, command_type, experiment_config, context=None): if command_type == rv.PREDICT: if not self.uri: # Construct the URI for this prediction store, # using the scene ID. root = experiment_config.predict_uri uri = None for c in context: if isinstance(c, rv.SceneConfig): uri = os.path.join(root, '{}.tif'.format(c.id)) if uri: self.uri = uri else: raise rv.ConfigError( 'SemanticSegmentationRasterStoreConfig has no ' 'URI set, and is not associated with a SceneConfig.') # Construct URIs for vector predictions for vo in self.vector_output: for c in context: if isinstance(c, rv.SceneConfig) and ('uri' not in vo.keys() or not vo['uri']): root = experiment_config.predict_uri mode = vo['mode'] class_id = vo['class_id'] vo['uri'] = os.path.join( root, '{}-{}-{}.geojson'.format(c.id, class_id, mode))
def validate(self): if self.config.get('uri') is None: raise rv.ConfigError( 'VectorTileVectorSourceConfigBuilder requires uri which ' 'can be set using "with_uri".') if self.config.get('zoom') is None: raise rv.ConfigError( 'VectorTileVectorSourceConfigBuilder requires zoom which ' 'can be set using "with_zoom".') # If not set explicitly, set it using default value. if self.config.get('id_field') is None: self.with_id_field() super().validate()
def validate(self): super().validate() if self.require_task and not self.task: raise rv.ConfigError('You must specify the task this backend ' "is for - use 'with_task'.") return True
def save_bundle_files(self, bundle_dir): if not self.model_uri: raise rv.ConfigError('model_uri is not set.') local_path, base_name = self.bundle_file(self.model_uri, bundle_dir) new_config = self.to_builder() \ .with_model_uri(base_name) \ .build() return (new_config, [local_path])
def load_bundle_files(self, bundle_dir): model_uri = self.backend_opts.model_uri if not model_uri: raise rv.ConfigError('model_uri is not set.') local_model_uri = join(bundle_dir, model_uri) return self.to_builder() \ .with_model_uri(local_model_uri) \ .build()
def validate(self): super().validate() if not self.config.get('tfod_config'): raise rv.ConfigError('You must specify a template for the backend ' "configuration - use 'with_template'.") if not isinstance(self.config.get('tfod_config'), dict): raise rv.ConfigError( 'tfod_config must be of type dict, got {}'.format( type(self.config.get('tfod_config')))) if self.require_task and not self.task: raise rv.ConfigError('You must specify the task this backend ' "is for - use 'with_task'.") if self.require_task and not isinstance(self.task, ObjectDetectionConfig): raise rv.ConfigError( 'Task set with with_task must be of type ObjectDetectionConfig, got {}.'. format(type(self.task))) return True
def validate(self): super().validate() if not isinstance(self.config, dict): raise rv.ConfigError( 'config must be of type dict, got {}'.format( type(self.config.get('config')))) if self.require_task and not self.task: raise rv.ConfigError('You must specify the task this backend ' "is for - use 'with_task'.") if self.require_task and not isinstance(self.task, SemanticSegmentationConfig): raise rv.ConfigError('Task set with with_task must be of type' ' SemanticSegmentationConfig, got {}.'.format( type(self.task))) return True
def with_model_defaults(self, model_defaults_key): """Sets the backend configuration and pretrained model defaults according to the model defaults configuration. """ model_defaults = RVConfig.get_instance().get_model_defaults() if self.backend_type in model_defaults: backend_defaults = model_defaults[self.backend_type] if model_defaults_key in backend_defaults: return self._load_model_defaults( backend_defaults[model_defaults_key]) else: raise rv.ConfigError('No defaults found for model key {}' .format(model_defaults_key)) else: raise rv.ConfigError('No model defaults for backend {}' .format(self.backend_type)) return self
def validate(self): super().validate() # Segmentation masks are stored as uint8 to save space, so can only handle 256 # classes. If this is really needed, we can add an option for saving with uint16. max_classes = 256 if len(self.config['class_map']) > max_classes: raise rv.ConfigError( 'Cannot use more than {} classes with semantic segmentation.'. format(max_classes))
def validate(self): super().validate() if not self.command_class: raise rv.ConfigError( 'AuxCommandConfigBuilder requires the command_class be set.') if self.config is None: raise rv.ConfigError( 'AuxCommandConfigBuilder requires a configuration be set, either ' 'through with_config or by setting a dict with the property "config" ' 'in an experiment custom configuration dict with the command name ' 'as the key in the experiment custom configuration') if self.command_class.options.required_fields: for field in self.command_class.options.required_fields: if field not in self.config: raise rv.ConfigError('{} command requires the field {} ' 'be set in the configuration.'.format( self.command_type, field))
def validate(self): super().validate() if not self.config.get('kc_config'): raise rv.ConfigError('You must specify a template for the backend ' 'configuration - use "with_template".') if not isinstance(self.config.get('kc_config'), dict): raise rv.ConfigError( 'kc_config must be of type dict, got {}'.format( type(self.config.get('kc_config')))) if self.require_task and not self.task: raise rv.ConfigError('You must specify the task this backend ' 'is for - use "with_task".') if self.require_task and not isinstance(self.task, ChipClassificationConfig): raise rv.ConfigError('Task set with with_task must be of type ' 'ChipClassificationConfig, got {}.'.format( type(self.task)))