def forward(self, sample: Any) -> Any:
self.callback.on_load_sample(sample, self.stage)
with self._current_stage_context:
if self.pre_tensor_transform is not None:
with self._pre_tensor_transform_context:
sample = self.pre_tensor_transform(sample)
self.callback.on_pre_tensor_transform(sample, self.stage)
if self.to_tensor_transform is not None:
with self._to_tensor_transform_context:
sample = self.to_tensor_transform(sample)
self.callback.on_to_tensor_transform(sample, self.stage)
if self.assert_contains_tensor:
if not _contains_any_tensor(sample):
raise MisconfigurationException(
"When ``to_tensor_transform`` is overriden, "
"``DataPipeline`` expects the outputs to be ``tensors``"
)
with self._post_tensor_transform_context:
sample = self.post_tensor_transform(sample)
self.callback.on_post_tensor_transform(sample, self.stage)
return sample
def before_collate(self, samples: Any) -> Any:
"""Override to apply transformations to samples"""
if _contains_any_tensor(samples):
return samples
elif isinstance(samples, (list, tuple)) and len(samples) > 0 and all(isinstance(s, str) for s in samples):
return [self._tokenize_fn({self._input: s}) for s in samples]
raise MisconfigurationException("samples can only be tensors or a list of sentences.")
def before_collate(self, samples: Any) -> Any:
"""Override to apply transformations to samples"""
if _contains_any_tensor(samples, dtype=(Tensor, np.ndarray)):
return samples
if isinstance(samples, str):
samples = pd.read_csv(samples)
if isinstance(samples, DataFrame):
samples = [samples]
dfs = _pre_transform(
samples, self._numerical_input, self._categorical_input, self._codes, self._mean, self._std
)
return _dfs_to_samples(dfs, self._categorical_input, self._numerical_input)
def before_collate(self, samples: Any) -> Any:
if _contains_any_tensor(samples):
return samples
if isinstance(samples, str):
samples = [samples]
if isinstance(samples, (list, tuple)) and all(isinstance(p, str) for p in samples):
outputs = []
for sample in samples:
output = self._loader(sample)
outputs.append(self._valid_transform(output))
return outputs
raise MisconfigurationException("The samples should either be a tensor, a list of paths or a path.")
def before_collate(self, samples: Any) -> Any:
if _contains_any_tensor(samples):
return samples
if isinstance(samples, str):
samples = [samples]
if isinstance(samples, (list, tuple)) and all(
isinstance(p, str) for p in samples):
outputs = []
for sample in samples:
try:
output = self._loader(sample)
transform = self._valid_transform if self._use_valid_transform else self._train_transform
outputs.append(transform(output))
except UnidentifiedImageError:
print(f'Skipping: could not read file {sample}')
return outputs
raise MisconfigurationException(
"The samples should either be a tensor or a list of paths.")
