class PyTorchLearnerBackendConfig(BackendConfig):
model: ModelConfig
solver: SolverConfig
log_tensorboard: bool = Field(
True, description='If True, log events to Tensorboard log files.')
run_tensorboard: bool = Field(
False,
description='If True, run Tensorboard server pointing at log files.')
augmentors: List[str] = Field(
default_augmentors,
description='Names of albumentations augmentors to use for training '
f'batches. Choices include: {augmentor_list}. Alternatively, a custom '
'transform can be provided via the aug_transform option.')
base_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied to all datasets: training, validation, and test. '
'This transformation is in addition to the resizing due to img_sz. '
'This is useful for, for example, applying the same normalization to '
'all datasets.')
aug_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied as data augmentation to the training dataset. This '
'transform is applied before base_transform. If provided, the '
'augmentors option is ignored.')
test_mode: bool = Field(
False,
description=
('This field is passed along to the LearnerConfig which is returned by '
'get_learner_config(). For more info, see the docs for'
'pytorch_learner.learner_config.LearnerConfig.test_mode.'))
plot_options: Optional[PlotOptions] = Field(
PlotOptions(), description='Options to control plotting.')
img_sz: Optional[PositiveInt] = Field(
None,
description='Length of a side of each image in pixels. This is the '
'size to transform it to during training, not the size in the raw '
'dataset. Defaults to train_chip_sz in the pipeline config.')
num_workers: int = Field(
4, description='The number of workers to use in PyTorch to read data.')
# validators
_base_tf = validator(
'base_transform', allow_reuse=True)(validate_albumentation_transform)
_aug_tf = validator(
'aug_transform', allow_reuse=True)(validate_albumentation_transform)
def get_bundle_filenames(self):
return ['model-bundle.zip']
def get_learner_config(self, pipeline):
raise NotImplementedError()
def build(self, pipeline, tmp_dir):
raise NotImplementedError()
class PlotOptions(Config):
"""Config related to plotting."""
transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied to each image before it is plotted. Mainly useful '
'for undoing any data transformation that you do not want included in '
'the plot, such as normalization.')
# validators
_tf = validator('transform',
allow_reuse=True)(validate_albumentation_transform)
class PlotOptions(Config):
"""Config related to plotting."""
transform: Optional[dict] = Field(
A.to_dict(MinMaxNormalize()),
description='An Albumentations transform serialized as a dict that '
'will be applied to each image before it is plotted. Mainly useful '
'for undoing any data transformation that you do not want included in '
'the plot, such as normalization. The default value will shift and scale the '
'image so the values range from 0.0 to 1.0 which is the expected range for '
'the plotting function. This default is useful for cases where the values after '
'normalization are close to zero which makes the plot difficult to see.'
)
# validators
_tf = validator(
'transform', allow_reuse=True)(validate_albumentation_transform)
class DataConfig(Config):
"""Config related to dataset for training and testing."""
class_names: List[str] = Field([], description='Names of classes.')
class_colors: Optional[Union[List[str], List[List]]] = Field(
None,
description=('Colors used to display classes. '
'Can be color 3-tuples in list form.'))
img_sz: PosInt = Field(
256,
description=
('Length of a side of each image in pixels. This is the size to transform '
'it to during training, not the size in the raw dataset.'))
train_sz: Optional[int] = Field(
None,
description=
('If set, the number of training images to use. If fewer images exist, '
'then an exception will be raised.'))
num_workers: int = Field(
4,
description='Number of workers to use when DataLoader makes batches.')
augmentors: List[str] = Field(
default_augmentors,
description='Names of albumentations augmentors to use for training '
f'batches. Choices include: {augmentors}. Alternatively, a custom '
'transform can be provided via the aug_transform option.')
base_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied to all datasets: training, validation, and test. '
'This transformation is in addition to the resizing due to img_sz. '
'This is useful for, for example, applying the same normalization to '
'all datasets.')
aug_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied as data augmentation to the training dataset. This '
'transform is applied before base_transform. If provided, the '
'augmentors option is ignored.')
plot_options: Optional[PlotOptions] = Field(
PlotOptions(), description='Options to control plotting.')
preview_batch_limit: Optional[int] = Field(
None,
description=
('Optional limit on the number of items in the preview plots produced '
'during training.'))
# validators
_base_tf = validator('base_transform',
allow_reuse=True)(validate_albumentation_transform)
_aug_tf = validator('aug_transform',
allow_reuse=True)(validate_albumentation_transform)
def update(self, learner: Optional['LearnerConfig'] = None):
if not self.class_colors:
self.class_colors = [color_to_triple() for _ in self.class_names]
def validate_augmentors(self):
self.validate_list('augmentors', augmentors)
def validate_config(self):
self.validate_augmentors()
def make_datasets(self) -> Tuple[Dataset, Dataset, Dataset]:
raise NotImplementedError()
class DataConfig(Config):
"""Config related to dataset for training and testing."""
uri: Union[None, str, List[str]] = Field(
None,
description=
('URI of the dataset. This can be a zip file, a list of zip files, or a '
'directory which contains a set of zip files.'))
train_sz: Optional[int] = Field(
None,
description=
('If set, the number of training images to use. If fewer images exist, '
'then an exception will be raised.'))
group_uris: Union[None, List[Union[str, List[str]]]] = Field(
None,
description=
('This can be set instead of uri in order to specify groups of chips. Each '
'element in the list is expected to be an object of the same form accepted by '
'the uri field. The purpose of separating chips into groups is to be able to '
'use the group_train_sz field.'))
group_train_sz: Optional[int] = Field(
None,
description=
('If group_uris is set, this can be used to specify the number of chips to use '
'per group.'))
data_format: Optional[str] = Field(None,
description='Name of dataset format.')
class_names: List[str] = Field([], description='Names of classes.')
class_colors: Union[None, List[str], List[List]] = Field(
None,
description=('Colors used to display classes. '
'Can be color 3-tuples in list form.'))
img_sz: PositiveInt = Field(
256,
description=
('Length of a side of each image in pixels. This is the size to transform '
'it to during training, not the size in the raw dataset.'))
num_workers: int = Field(
4,
description='Number of workers to use when DataLoader makes batches.')
augmentors: List[str] = Field(
default_augmentors,
description='Names of albumentations augmentors to use for training '
f'batches. Choices include: {augmentors}. Alternatively, a custom '
'transform can be provided via the aug_transform option.')
base_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied to all datasets: training, validation, and test. '
'This transformation is in addition to the resizing due to img_sz. '
'This is useful for, for example, applying the same normalization to '
'all datasets.')
aug_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied as data augmentation to the training dataset. This '
'transform is applied before base_transform. If provided, the '
'augmentors option is ignored.')
plot_options: Optional[PlotOptions] = Field(
PlotOptions(), description='Options to control plotting.')
# validators
_base_tf = validator('base_transform',
allow_reuse=True)(validate_albumentation_transform)
_aug_tf = validator('aug_transform',
allow_reuse=True)(validate_albumentation_transform)
def update(self, learner: Optional['LearnerConfig'] = None):
if not self.class_colors:
self.class_colors = [color_to_triple() for _ in self.class_names]
def validate_augmentors(self):
self.validate_list('augmentors', augmentors)
def validate_config(self):
self.validate_augmentors()
class DataConfig(Config):
"""Config related to dataset for training and testing."""
uri: Optional[Union[str, List[str]]] = Field(
None,
description=
('URI of the dataset. This can be a zip file, a list of zip files, or a '
'directory which contains a set of zip files.'))
train_sz: Optional[int] = Field(
None,
description=
('If set, the number of training images to use. If fewer images exist, '
'then an exception will be raised.'))
group_uris: Optional[List[Union[str, List[str]]]] = Field(
None,
description=
'This can be set instead of uri in order to specify groups of chips. '
'Each element in the list is expected to be an object of the same '
'form accepted by the uri field. The purpose of separating chips into '
'groups is to be able to use the group_train_sz field.')
group_train_sz: Optional[Union[int, List[int]]] = Field(
None,
description='If group_uris is set, this can be used to specify the '
'number of chips to use per group. Only applies to training chips. '
'This can either be a single value that will be used for all groups '
'or a list of values (one for each group).')
group_train_sz_rel: Optional[Union[Proportion, List[Proportion]]] = Field(
None,
description='Relative version of group_train_sz. Must be a float '
'in [0, 1]. If group_uris is set, this can be used to specify the '
'proportion of the total chips in each group to use per group. '
'Only applies to training chips. This can either be a single value '
'that will be used for all groups or a list of values '
'(one for each group).')
data_format: Optional[str] = Field(None,
description='Name of dataset format.')
class_names: List[str] = Field([], description='Names of classes.')
class_colors: Optional[Union[List[str], List[List]]] = Field(
None,
description=('Colors used to display classes. '
'Can be color 3-tuples in list form.'))
img_sz: PositiveInt = Field(
256,
description=
('Length of a side of each image in pixels. This is the size to transform '
'it to during training, not the size in the raw dataset.'))
num_workers: int = Field(
4,
description='Number of workers to use when DataLoader makes batches.')
augmentors: List[str] = Field(
default_augmentors,
description='Names of albumentations augmentors to use for training '
f'batches. Choices include: {augmentors}. Alternatively, a custom '
'transform can be provided via the aug_transform option.')
base_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied to all datasets: training, validation, and test. '
'This transformation is in addition to the resizing due to img_sz. '
'This is useful for, for example, applying the same normalization to '
'all datasets.')
aug_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied as data augmentation to the training dataset. This '
'transform is applied before base_transform. If provided, the '
'augmentors option is ignored.')
plot_options: Optional[PlotOptions] = Field(
PlotOptions(), description='Options to control plotting.')
preview_batch_limit: Optional[int] = Field(
None,
description=
('Optional limit on the number of items in the preview plots produced '
'during training.'))
# validators
_base_tf = validator('base_transform',
allow_reuse=True)(validate_albumentation_transform)
_aug_tf = validator('aug_transform',
allow_reuse=True)(validate_albumentation_transform)
def update(self, learner: Optional['LearnerConfig'] = None):
if not self.class_colors:
self.class_colors = [color_to_triple() for _ in self.class_names]
def validate_augmentors(self):
self.validate_list('augmentors', augmentors)
def validate_config(self):
self.validate_augmentors()
self.validate_group_uris()
def validate_group_uris(self):
has_group_train_sz = self.group_train_sz is not None
has_group_train_sz_rel = self.group_train_sz_rel is not None
has_group_uris = self.group_uris is not None
if has_group_train_sz and has_group_train_sz_rel:
raise ConfigError('Only one of group_train_sz and '
'group_train_sz_rel should be specified.')
if has_group_train_sz and not has_group_uris:
raise ConfigError('group_train_sz specified without group_uris.')
if has_group_train_sz_rel and not has_group_uris:
raise ConfigError(
'group_train_sz_rel specified without group_uris.')
if has_group_train_sz and sequence_like(self.group_train_sz):
if len(self.group_train_sz) != len(self.group_uris):
raise ConfigError('len(group_train_sz) != len(group_uris).')
if has_group_train_sz_rel and sequence_like(self.group_train_sz_rel):
if len(self.group_train_sz_rel) != len(self.group_uris):
raise ConfigError(
'len(group_train_sz_rel) != len(group_uris).')
class PyTorchLearnerBackendConfig(BackendConfig):
model: ModelConfig
solver: SolverConfig
log_tensorboard: bool = Field(
True, description='If True, log events to Tensorboard log files.')
run_tensorboard: bool = Field(
False,
description='If True, run Tensorboard server pointing at log files.')
augmentors: List[str] = Field(
default_augmentors,
description='Names of albumentations augmentors to use for training '
f'batches. Choices include: {augmentor_list}. Alternatively, a custom '
'transform can be provided via the aug_transform option.')
base_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied to all datasets: training, validation, and test. '
'This transformation is in addition to the resizing due to img_sz. '
'This is useful for, for example, applying the same normalization to '
'all datasets.')
aug_transform: Optional[dict] = Field(
None,
description='An Albumentations transform serialized as a dict that '
'will be applied as data augmentation to the training dataset. This '
'transform is applied before base_transform. If provided, the '
'augmentors option is ignored.')
test_mode: bool = Field(
False,
description=
('This field is passed along to the LearnerConfig which is returned by '
'get_learner_config(). For more info, see the docs for'
'pytorch_learner.learner_config.LearnerConfig.test_mode.'))
plot_options: Optional[PlotOptions] = Field(
PlotOptions(), description='Options to control plotting.')
img_sz: Optional[PositiveInt] = Field(
None,
description='Length of a side of each image in pixels. This is the '
'size to transform it to during training, not the size in the raw '
'dataset. Defaults to train_chip_sz in the pipeline config.')
num_workers: int = Field(
4, description='The number of workers to use in PyTorch to read data.')
group_uris: Optional[List[Union[str, List[str]]]] = Field(
None,
description=
'This can be set instead of uri in order to specify groups of chips. '
'Each element in the list is expected to be an object of the same '
'form accepted by the uri field. The purpose of separating chips into '
'groups is to be able to use the group_train_sz field.')
group_train_sz: Optional[Union[int, List[int]]] = Field(
None,
description='If group_uris is set, this can be used to specify the '
'number of chips to use per group. Only applies to training chips. '
'This can either be a single value that will be used for all groups '
'or a list of values (one for each group).')
group_train_sz_rel: Optional[Union[Proportion, List[Proportion]]] = Field(
None,
description='Relative version of group_train_sz. Must be a float '
'in [0, 1]. If group_uris is set, this can be used to specify the '
'proportion of the total chips in each group to use per group. '
'Only applies to training chips. This can either be a single value '
'that will be used for all groups or a list of values '
'(one for each group).')
preview_batch_limit: Optional[int] = Field(
None,
description=
('Optional limit on the number of items in the preview plots produced '
'during training.'))
# validators
_base_tf = validator('base_transform',
allow_reuse=True)(validate_albumentation_transform)
_aug_tf = validator('aug_transform',
allow_reuse=True)(validate_albumentation_transform)
def get_bundle_filenames(self):
return ['model-bundle.zip']
def get_learner_config(self, pipeline):
raise NotImplementedError()
def build(self, pipeline, tmp_dir):
raise NotImplementedError()