def _get_iterator(self, split: str):
dataset_identifier = self._get_resource_id(element=f"{split}.pd")
dataset_resource = self.storage_connector.get_resource(
identifier=dataset_identifier)
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return KDDIterator(dataset_resource), meta
def dataset_meta(self) -> DatasetMeta:
iterator_meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return MetaFactory.get_dataset_meta(identifier="identifier_1",
dataset_name="TEST DATASET",
dataset_tag="train",
iterator_meta=iterator_meta)
def _get_iterator(self, split: str):
dataset_identifier = self._get_resource_id(element="reuters.hdf5")
dataset_resource = self.storage_connector.get_resource(
identifier=dataset_identifier)
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return ReutersIterator(dataset_resource, split), meta
def _get_iterator(self):
dataset_identifier = self._get_resource_id(element="news_groups.hdf5")
dataset_resource = self.storage_connector.get_resource(
identifier=dataset_identifier)
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return NewsGroupsIterator(dataset_resource), meta
def _get_iterator(self,
noise_std: float,
interval: List[float],
num_samples: int,
seed: int = 1):
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return NoisyXCubedIterator(seed, noise_std, interval,
num_samples), meta
def _get_iterator(self,
split: str,
length: float,
num_samples: List[int],
seed: int = 1,
translation: List[int] = None):
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return XORSquaresIterator(seed, length, num_samples, translation), meta
def _get_iterator(self, split: str):
"""Supported splits: train, val, test
"""
dataset_identifier = self._get_resource_id(element="atis_dataset.hdf5")
dataset_resource = self.storage_connector.get_resource(
identifier=dataset_identifier)
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return AtisIterator(dataset_resource, split), meta
def iterator(self) -> str:
targets = [1]*100 + [2]*200 + [3]*300
sequence_targets = torch.Tensor(targets)
sequence_samples = torch.ones_like(sequence_targets)
iterator = SequenceDatasetIterator([sequence_samples, sequence_targets])
iterator_meta = MetaFactory.get_iterator_meta(sample_pos=0, target_pos=1, tag_pos=1)
meta = MetaFactory.get_dataset_meta(identifier="dataset id",
dataset_name="dataset",
dataset_tag="full",
iterator_meta=iterator_meta)
return InformedDatasetFactory.get_dataset_iterator(iterator, meta)
def _get_iterator(self,
split: str,
scale_factor: float,
noise_std: float,
num_samples: List[int],
seed: int = 1,
translation: List[int] = None):
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return CirclesIterator(seed, noise_std, num_samples, scale_factor,
translation), meta
def _get_iterator(self,
split: str,
noise_std: float,
num_samples: List[int],
seed: int = 1,
translation: List[float] = None,
scaling: List[int] = None):
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return HalfMoonIterator(seed, noise_std, num_samples, translation,
scaling), meta
def _get_iterator(self,
split: str,
num_samples: List[int],
classes: List[int],
hypercube: List[Tuple[int, int]],
seed: int = 1):
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return UniformNoiseIterator(seed=seed,
num_samples=num_samples,
classes=classes,
hypercube=hypercube), meta
def _get_iterator(self,
split: str,
class_label: int,
radius: float,
start_degree: float,
end_degree: float,
num_samples: int,
seed: int = 1,
translation: List[int] = None,
noise_std: int = 0):
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return CircularSegmentIterator(seed, class_label, radius, start_degree,
end_degree, num_samples, noise_std,
translation), meta
def _get_iterator(self):
sample_identifier = self._get_resource_id(element="samples.pt")
target_identifier = self._get_resource_id(element="targets.pt")
sample_resource = self.storage_connector.get_resource(
identifier=sample_identifier)
target_resource = self.storage_connector.get_resource(
identifier=target_identifier)
text_sample_resource = StreamedTextResource.from_streamed_resouce(
sample_resource)
text_target_resource = StreamedTextResource.from_streamed_resouce(
target_resource)
meta = MetaFactory.get_iterator_meta(sample_pos=0,
target_pos=1,
tag_pos=2)
return ArrhythmiaIterator(text_sample_resource,
text_target_resource), meta
def _get_iterator(self, split: str, high_level_targets: bool = True):
dataset_identifier = self._get_resource_id(element="trec_dataset.hdf5")
dataset_resource = self.storage_connector.get_resource(identifier=dataset_identifier)
meta = MetaFactory.get_iterator_meta(sample_pos=0, target_pos=1, tag_pos=2)
return TrecIterator(dataset_resource, split, high_level_targets), meta
def _get_iterator(self, split: str, class_label: int, seed: int, num_samples: int, covariance: np.array, mean: Tuple[int, int]):
meta = MetaFactory.get_iterator_meta(sample_pos=0, target_pos=1, tag_pos=2)
return GaussianIterator(seed, class_label, num_samples, covariance, mean), meta