def validate(cls, v, field: ModelField):
"""Pydantic validator."""
from pydantic.utils import sequence_like
if not sequence_like(v):
raise TypeError(
trans._(
'Value is not a valid sequence: {value}',
deferred=True,
value=v,
)
)
if not field.sub_fields:
return cls(v)
type_field = field.sub_fields[0]
errors = []
for i, v_ in enumerate(v):
_valid_value, error = type_field.validate(v_, {}, loc=f'[{i}]')
if error:
errors.append(error)
if errors:
from pydantic import ValidationError
raise ValidationError(errors, cls) # type: ignore
return cls(v)
def validate_group_uris(self):
has_group_train_sz = self.group_train_sz is not None
has_group_train_sz_rel = self.group_train_sz_rel is not None
has_group_uris = self.group_uris is not None
if has_group_train_sz and has_group_train_sz_rel:
raise ConfigError('Only one of group_train_sz and '
'group_train_sz_rel should be specified.')
if has_group_train_sz and not has_group_uris:
raise ConfigError('group_train_sz specified without group_uris.')
if has_group_train_sz_rel and not has_group_uris:
raise ConfigError(
'group_train_sz_rel specified without group_uris.')
if has_group_train_sz and sequence_like(self.group_train_sz):
if len(self.group_train_sz) != len(self.group_uris):
raise ConfigError('len(group_train_sz) != len(group_uris).')
if has_group_train_sz_rel and sequence_like(self.group_train_sz_rel):
if len(self.group_train_sz_rel) != len(self.group_uris):
raise ConfigError(
'len(group_train_sz_rel) != len(group_uris).')
def validate(cls, v, field: 'ModelField'):
"""Pydantic validator."""
from pydantic.utils import sequence_like
if isinstance(v, dict):
data = v.get("selection", [])
current = v.get("_current", None)
elif isinstance(v, Selection):
data = v._set
current = v._current
else:
data = v
current = None
if not sequence_like(data):
raise TypeError(
trans._(
'Value is not a valid sequence: {data}',
deferred=True,
data=data,
))
# no type parameter was provided, just return
if not field.sub_fields:
obj = cls(data=data)
obj._current_ = current
return obj
# Selection[type] parameter was provided. Validate contents
type_field = field.sub_fields[0]
errors = []
for i, v_ in enumerate(data):
_, error = type_field.validate(v_, {}, loc=f'[{i}]')
if error:
errors.append(error)
if current is not None:
_, error = type_field.validate(current, {}, loc='current')
if error:
errors.append(error)
if errors:
from pydantic import ValidationError
raise ValidationError(errors, cls) # type: ignore
obj = cls(data=data)
obj._current_ = current
return obj
def get_datasets(self) -> Tuple[Dataset, Dataset, Dataset]:
"""Returns train, validation, and test DataSets."""
cfg = self.cfg
if isinstance(cfg.data, GeoDataConfig):
return self._get_datasets()
if cfg.data.group_uris is None:
return self._get_datasets(cfg.data.uri)
if cfg.data.uri is not None:
log.warn('Both DataConfig.uri and DataConfig.group_uris '
'specified. Only DataConfig.group_uris will be used.')
train_ds_lst, valid_ds_lst, test_ds_lst = [], [], []
group_sizes = None
if cfg.data.group_train_sz is not None:
group_sizes = cfg.data.group_train_sz
elif cfg.data.group_train_sz_rel is not None:
group_sizes = cfg.data.group_train_sz_rel
if not sequence_like(group_sizes):
group_sizes = [group_sizes] * len(cfg.data.group_uris)
for uri, sz in zip(cfg.data.group_uris, group_sizes):
train_ds, valid_ds, test_ds = self._get_datasets(uri)
if sz is not None:
if isinstance(sz, float):
sz = int(len(train_ds) * sz)
train_inds = list(range(len(train_ds)))
random.seed(1234)
random.shuffle(train_inds)
train_inds = train_inds[:sz]
train_ds = Subset(train_ds, train_inds)
train_ds_lst.append(train_ds)
valid_ds_lst.append(valid_ds)
test_ds_lst.append(test_ds)
train_ds, valid_ds, test_ds = (ConcatDataset(train_ds_lst),
ConcatDataset(valid_ds_lst),
ConcatDataset(test_ds_lst))
return train_ds, valid_ds, test_ds
