def forward(self, sample: Any) -> Any: self.callback.on_load_sample(sample, self.stage) with self._current_stage_context: if self.pre_tensor_transform is not None: with self._pre_tensor_transform_context: sample = self.pre_tensor_transform(sample) self.callback.on_pre_tensor_transform(sample, self.stage) if self.to_tensor_transform is not None: with self._to_tensor_transform_context: sample = self.to_tensor_transform(sample) self.callback.on_to_tensor_transform(sample, self.stage) if self.assert_contains_tensor: if not _contains_any_tensor(sample): raise MisconfigurationException( "When ``to_tensor_transform`` is overriden, " "``DataPipeline`` expects the outputs to be ``tensors``" ) with self._post_tensor_transform_context: sample = self.post_tensor_transform(sample) self.callback.on_post_tensor_transform(sample, self.stage) return sample
def before_collate(self, samples: Any) -> Any: """Override to apply transformations to samples""" if _contains_any_tensor(samples): return samples elif isinstance(samples, (list, tuple)) and len(samples) > 0 and all(isinstance(s, str) for s in samples): return [self._tokenize_fn({self._input: s}) for s in samples] raise MisconfigurationException("samples can only be tensors or a list of sentences.")
def before_collate(self, samples: Any) -> Any: """Override to apply transformations to samples""" if _contains_any_tensor(samples, dtype=(Tensor, np.ndarray)): return samples if isinstance(samples, str): samples = pd.read_csv(samples) if isinstance(samples, DataFrame): samples = [samples] dfs = _pre_transform( samples, self._numerical_input, self._categorical_input, self._codes, self._mean, self._std ) return _dfs_to_samples(dfs, self._categorical_input, self._numerical_input)
def before_collate(self, samples: Any) -> Any: if _contains_any_tensor(samples): return samples if isinstance(samples, str): samples = [samples] if isinstance(samples, (list, tuple)) and all(isinstance(p, str) for p in samples): outputs = [] for sample in samples: output = self._loader(sample) outputs.append(self._valid_transform(output)) return outputs raise MisconfigurationException("The samples should either be a tensor, a list of paths or a path.")
def before_collate(self, samples: Any) -> Any: if _contains_any_tensor(samples): return samples if isinstance(samples, str): samples = [samples] if isinstance(samples, (list, tuple)) and all( isinstance(p, str) for p in samples): outputs = [] for sample in samples: try: output = self._loader(sample) transform = self._valid_transform if self._use_valid_transform else self._train_transform outputs.append(transform(output)) except UnidentifiedImageError: print(f'Skipping: could not read file {sample}') return outputs raise MisconfigurationException( "The samples should either be a tensor or a list of paths.")