def load_parsed_dataset(path: Path, freq: str) -> Iterator[Any]: yield from FileDataset(path, freq)
def load_file_dataset(path: Path, freq: str) -> Iterator[Any]: return iter(FileDataset(path, freq))
def load_file_dataset_cached(path: Path, freq: str) -> Iterator[Any]: return iter(FileDataset(path, freq, cache=True))
def load_file_dataset_numpy(path: Path, freq: str) -> Iterator[Any]: for item in FileDataset(path, freq): item["start"] = pd.Period(item["start"]) item["target"] = np.array(item["target"]) yield item
def _get_datasets(self): freq = self.hyperparameters["time_freq"] return { name: FileDataset(self.channels[name], freq) for name in DATASETS if name in self.channels }
def load_parsed_dataset(path: Path, freq: str) -> Iterator[Any]: for item in FileDataset(path, freq): yield TimeSeriesItem( start=item["start"], target=item["target"], item="ABC" )
def test_train_loader_goes_over_all_data(num_workers) -> None: batch_size = 4 num_batches_per_epoch = 4 X = 3 simple_data = [{ "start": "2012-01-01", "target": np.random.uniform(size=40).astype(float).tolist(), "item_id": i, } for i in range(batch_size * num_batches_per_epoch * X)] num_passes = 5 num_epochs = X * num_passes def test_dataset(dataset): class ExactlyOneSampler(InstanceSampler): def __call__(self, ts: np.ndarray, a: int, b: int) -> np.ndarray: window_size = b - a + 1 assert window_size > 0 return np.array([a]) transformation = InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=ExactlyOneSampler(), past_length=10, future_length=5, dummy_value=1.0, ) dl = TrainDataLoader( dataset=dataset, transform=transformation, batch_size=batch_size, stack_fn=partial(batchify, ctx=current_context()), num_workers=num_workers, num_batches_per_epoch=num_batches_per_epoch, ) item_ids = defaultdict(int) for epoch in range(num_epochs): for batch in dl: for item_id in batch["item_id"]: item_ids[item_id] += 1 for i in range(len(dataset)): assert num_passes - 1 <= item_ids[i] <= num_passes + 1 test_dataset(ListDataset(simple_data, freq="1H")) with tempfile.TemporaryDirectory() as tmpdir: with open(tmpdir + "/data.json", "w") as f: for data in simple_data: json.dump(data, f) f.write("\n") test_dataset(FileDataset(Path(tmpdir), freq="1H")) test_dataset(FileDataset(Path(tmpdir), freq="1H", cache=True))
def load_parsed_dataset(path: Path, freq: str) -> Iterator[Any]: for item in FileDataset(path, freq): yield TimeSeriesItem(start=item['start'], target=item['target'], item='ABC')
def load_file_dataset_numpy(path: Path, freq: str) -> Iterator[Any]: for item in FileDataset(path, freq): item['start'] = pd.Timestamp(item['start']) item['target'] = np.array(item['target']) yield item