def validate(cls, v, field: ModelField): """Pydantic validator.""" from pydantic.utils import sequence_like if not sequence_like(v): raise TypeError( trans._( 'Value is not a valid sequence: {value}', deferred=True, value=v, ) ) if not field.sub_fields: return cls(v) type_field = field.sub_fields[0] errors = [] for i, v_ in enumerate(v): _valid_value, error = type_field.validate(v_, {}, loc=f'[{i}]') if error: errors.append(error) if errors: from pydantic import ValidationError raise ValidationError(errors, cls) # type: ignore return cls(v)
def validate_group_uris(self): has_group_train_sz = self.group_train_sz is not None has_group_train_sz_rel = self.group_train_sz_rel is not None has_group_uris = self.group_uris is not None if has_group_train_sz and has_group_train_sz_rel: raise ConfigError('Only one of group_train_sz and ' 'group_train_sz_rel should be specified.') if has_group_train_sz and not has_group_uris: raise ConfigError('group_train_sz specified without group_uris.') if has_group_train_sz_rel and not has_group_uris: raise ConfigError( 'group_train_sz_rel specified without group_uris.') if has_group_train_sz and sequence_like(self.group_train_sz): if len(self.group_train_sz) != len(self.group_uris): raise ConfigError('len(group_train_sz) != len(group_uris).') if has_group_train_sz_rel and sequence_like(self.group_train_sz_rel): if len(self.group_train_sz_rel) != len(self.group_uris): raise ConfigError( 'len(group_train_sz_rel) != len(group_uris).')
def validate(cls, v, field: 'ModelField'): """Pydantic validator.""" from pydantic.utils import sequence_like if isinstance(v, dict): data = v.get("selection", []) current = v.get("_current", None) elif isinstance(v, Selection): data = v._set current = v._current else: data = v current = None if not sequence_like(data): raise TypeError( trans._( 'Value is not a valid sequence: {data}', deferred=True, data=data, )) # no type parameter was provided, just return if not field.sub_fields: obj = cls(data=data) obj._current_ = current return obj # Selection[type] parameter was provided. Validate contents type_field = field.sub_fields[0] errors = [] for i, v_ in enumerate(data): _, error = type_field.validate(v_, {}, loc=f'[{i}]') if error: errors.append(error) if current is not None: _, error = type_field.validate(current, {}, loc='current') if error: errors.append(error) if errors: from pydantic import ValidationError raise ValidationError(errors, cls) # type: ignore obj = cls(data=data) obj._current_ = current return obj
def get_datasets(self) -> Tuple[Dataset, Dataset, Dataset]: """Returns train, validation, and test DataSets.""" cfg = self.cfg if isinstance(cfg.data, GeoDataConfig): return self._get_datasets() if cfg.data.group_uris is None: return self._get_datasets(cfg.data.uri) if cfg.data.uri is not None: log.warn('Both DataConfig.uri and DataConfig.group_uris ' 'specified. Only DataConfig.group_uris will be used.') train_ds_lst, valid_ds_lst, test_ds_lst = [], [], [] group_sizes = None if cfg.data.group_train_sz is not None: group_sizes = cfg.data.group_train_sz elif cfg.data.group_train_sz_rel is not None: group_sizes = cfg.data.group_train_sz_rel if not sequence_like(group_sizes): group_sizes = [group_sizes] * len(cfg.data.group_uris) for uri, sz in zip(cfg.data.group_uris, group_sizes): train_ds, valid_ds, test_ds = self._get_datasets(uri) if sz is not None: if isinstance(sz, float): sz = int(len(train_ds) * sz) train_inds = list(range(len(train_ds))) random.seed(1234) random.shuffle(train_inds) train_inds = train_inds[:sz] train_ds = Subset(train_ds, train_inds) train_ds_lst.append(train_ds) valid_ds_lst.append(valid_ds) test_ds_lst.append(test_ds) train_ds, valid_ds, test_ds = (ConcatDataset(train_ds_lst), ConcatDataset(valid_ds_lst), ConcatDataset(test_ds_lst)) return train_ds, valid_ds, test_ds