def create_validation_data_loader(
self,
data: Dataset,
module: DeepARLightningModule,
**kwargs,
) -> Iterable:
transformation = self._create_instance_splitter(
module, "validation") + SelectFields(TRAINING_INPUT_NAMES)
validation_instances = transformation.apply(data)
return DataLoader(
IterableDataset(validation_instances),
batch_size=self.batch_size,
**kwargs,
)
def create_training_data_loader(
self,
data: Dataset,
module: DeepARLightningModule,
shuffle_buffer_length: Optional[int] = None,
**kwargs,
) -> Iterable:
transformation = self._create_instance_splitter(
module, "training") + SelectFields(TRAINING_INPUT_NAMES)
training_instances = transformation.apply(
Cyclic(data) if shuffle_buffer_length is None else PseudoShuffled(
Cyclic(data), shuffle_buffer_length=shuffle_buffer_length))
return IterableSlice(
iter(
DataLoader(
IterableDataset(training_instances),
batch_size=self.batch_size,
**kwargs,
)),
self.num_batches_per_epoch,
)