def dataloaders_with_covariates(data_with_covariates): training_cutoff = "2016-09-01" max_encoder_length = 36 max_prediction_length = 6 training = TimeSeriesDataSet( data_with_covariates[lambda x: x.date < training_cutoff], time_idx="time_idx", target="volume", # weight="weight", group_ids=["agency", "sku"], time_varying_known_reals=["discount"], time_varying_unknown_reals=["volume"], static_categoricals=["agency"], max_encoder_length=max_encoder_length, max_prediction_length=max_prediction_length, add_relative_time_idx=True, target_normalizer=GroupNormalizer(groups=["agency", "sku"], coerce_positive=False), ) validation = TimeSeriesDataSet.from_dataset( training, data_with_covariates, min_prediction_idx=training.index.time.max() + 1) batch_size = 4 train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0) val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size, num_workers=0) return dict(train=train_dataloader, val=val_dataloader)
def dataloaders_fixed_window_without_covariates(): data = generate_ar_data(seasonality=10.0, timesteps=400, n_series=10) validation = data.series.iloc[:2] max_encoder_length = 60 max_prediction_length = 20 training = TimeSeriesDataSet( data[lambda x: ~x.series.isin(validation)], time_idx="time_idx", target="value", categorical_encoders={"series": NaNLabelEncoder().fit(data.series)}, group_ids=["series"], static_categoricals=[], max_encoder_length=max_encoder_length, max_prediction_length=max_prediction_length, time_varying_unknown_reals=["value"], target_normalizer=EncoderNormalizer(), ) validation = TimeSeriesDataSet.from_dataset( training, data[lambda x: x.series.isin(validation)], stop_randomization=True, ) batch_size = 4 train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0) val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size, num_workers=0) return dict(train=train_dataloader, val=val_dataloader)
def multiple_dataloaders_with_covariates(data_with_covariates, request): training_cutoff = "2016-09-01" max_encoder_length = 36 max_prediction_length = 6 params = request.param params.setdefault("target", "volume") training = TimeSeriesDataSet( data_with_covariates[lambda x: x.date < training_cutoff], time_idx="time_idx", # weight="weight", group_ids=["agency", "sku"], max_encoder_length=max_encoder_length, max_prediction_length=max_prediction_length, add_relative_time_idx=True, **params # fixture parametrization ) validation = TimeSeriesDataSet.from_dataset( training, data_with_covariates, min_prediction_idx=training.index.time.max() + 1) batch_size = 4 train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0) val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size, num_workers=0) return dict(train=train_dataloader, val=val_dataloader)
def get_examples(self, year: int, encoder_length: dict, prediction_length: dict) -> TimeSeriesDataSet: solar_df_filtered = self.solar_df[(self.solar_df['year'] == year)] group_length = 2 * encoder_length['max'] - 1 num_groups = int(np.floor(solar_df_filtered.shape[0] / group_length)) solar_df_filtered = solar_df_filtered[1:(num_groups * group_length + 1)] solar_df_filtered['group'] = np.repeat(np.arange(num_groups), group_length) examples = TimeSeriesDataSet( solar_df_filtered, group_ids=["group"], target="power", time_idx="time_idx", min_encoder_length=encoder_length['min'], max_encoder_length=encoder_length['max'], min_prediction_length=prediction_length['min'], max_prediction_length=prediction_length['max'], time_varying_unknown_reals=["power"], time_varying_known_reals=[ "cloudcover_low", "cloudcover_mid", "cloudcover_high" ], time_varying_known_categoricals=["seasons"], allow_missings=True, ) return examples
def make_dataloaders(data_with_covariates, **kwargs): training_cutoff = "2016-09-01" max_encoder_length = 4 max_prediction_length = 3 kwargs.setdefault("target", "volume") kwargs.setdefault("group_ids", ["agency", "sku"]) kwargs.setdefault("add_relative_time_idx", True) kwargs.setdefault("time_varying_unknown_reals", ["volume"]) training = TimeSeriesDataSet( data_with_covariates[lambda x: x.date < training_cutoff].copy(), time_idx="time_idx", max_encoder_length=max_encoder_length, max_prediction_length=max_prediction_length, **kwargs, # fixture parametrization ) validation = TimeSeriesDataSet.from_dataset( training, data_with_covariates.copy(), min_prediction_idx=training.index.time.max() + 1) train_dataloader = training.to_dataloader(train=True, batch_size=2, num_workers=0) val_dataloader = validation.to_dataloader(train=False, batch_size=2, num_workers=0) test_dataloader = validation.to_dataloader(train=False, batch_size=1, num_workers=0) return dict(train=train_dataloader, val=val_dataloader, test=test_dataloader)
def transform_data(self, data, past_lags, index_label, target_label, train_val_split): self.past_lags = past_lags self.oldest_lag = int(max(self.past_lags)) + 1 self.index_label = index_label self.target_label = target_label # External train and validation sets X = data[[index_label]] y = data[[target_label]] self.training = (X.loc[:int(len(data) * train_val_split)], y.loc[:int(len(data) * train_val_split)]) self.validation = (X.loc[int(len(data) * train_val_split):], y.loc[int(len(data) * train_val_split):]) # intern train and validation sets, they use dataloaders to optimize the training routine # time index are epoch values # data["time_idx"] = (data[self.index_label] - pd.Timestamp("1970-01-01")) // pd.Timedelta("1s") data["time_idx"] = data.index data['group_id'] = 'series' max_prediction_length = self.oldest_lag max_encoder_length = self.oldest_lag # training_cutoff = data["time_idx"].max() - max_prediction_length self.intern_training = TimeSeriesDataSet( data[:int(len(data) * train_val_split)], time_idx="time_idx", group_ids=["group_id"], target=self.target_label, min_encoder_length=0, max_encoder_length=max_encoder_length, min_prediction_length=1, max_prediction_length=max_prediction_length, static_categoricals=["group_id"], # time_varying_unknown_reals=[self.target_label], # the docs says that the max_lag < max_encoder_length # lags={self.target_label: list(self.past_lags[1:-1] + 1)}, add_relative_time_idx=True, add_target_scales=True, add_encoder_length=True, # allow_missings=True ) # create validation set (predict=True) which means to predict the last max_prediction_length points in time # for each series self._intern_validation = TimeSeriesDataSet.from_dataset( self.intern_training, data, predict=True, stop_randomization=True) # store the last input to use as encoder data to next predictions self.last_period = data.iloc[-(self.oldest_lag * 2 + 1):].copy()
def test_dataset(test_data): training = TimeSeriesDataSet( test_data, time_idx="time_idx", target="volume", time_varying_known_reals=["price_regular"], group_ids=["agency", "sku"], static_categoricals=["agency"], max_encoder_length=5, max_prediction_length=2, randomize_length=None, ) return training
def save_time_series(self): """ Download preprocessing file and creates data in a format suited for temporal fusion """ PREPROCESS_URL = 'https://raw.githubusercontent.com/AWarno/CancerOptimization/main/preprocess_data.py' FILE_PATH = 'data/preprocess_data.py' DATA_PATH = 'data/data.csv' FEATURES = ['dose', 'time'] GROUP_ID = 'series' # Data file already exists so we don't need to generate it if os.path.isfile(DATA_PATH): return # Preprocessing file already exists so we don't need to download it again if not os.path.isfile(FILE_PATH): wget.download(PREPROCESS_URL, FILE_PATH) os.system('python ' + FILE_PATH) dataset = pd.read_csv(DATA_PATH) n = dataset[GROUP_ID].astype(int).max() dataset['target'] = dataset['target'].astype(float) dataset['time_idx'] = dataset['time_idx'].astype(int) training = TimeSeriesDataSet( dataset[dataset[GROUP_ID].apply(lambda x: int(x) < int(n * 0.7))], time_idx='time_idx', target='target', group_ids=[GROUP_ID], min_encoder_length=20, max_encoder_length=20, min_prediction_length=1, max_prediction_length=1, static_categoricals=[], static_reals=[], time_varying_known_categoricals=[], variable_groups={}, time_varying_known_reals=['time_idx'], time_varying_unknown_categoricals=[], time_varying_unknown_reals=['target'] + FEATURES, add_relative_time_idx=True, add_target_scales=False, add_encoder_length=True, categorical_encoders={GROUP_ID: NaNLabelEncoder().fit(dataset.series)}, ) training.save(self.TIMESERIES_PATH)
def test_prediction_with_dataloder_raw(data_with_covariates, tmp_path): # tests correct concatenation of raw output test_data = data_with_covariates.copy() np.random.seed(2) test_data = test_data.sample(frac=0.5) dataset = TimeSeriesDataSet( test_data, time_idx="time_idx", max_encoder_length=8, max_prediction_length=10, min_prediction_length=1, min_encoder_length=1, target="volume", group_ids=["agency", "sku"], constant_fill_strategy=dict(volume=0.0), allow_missing_timesteps=True, time_varying_unknown_reals=["volume"], time_varying_known_reals=["time_idx"], target_normalizer=GroupNormalizer(groups=["agency", "sku"]), ) net = TemporalFusionTransformer.from_dataset( dataset, learning_rate=1e-6, hidden_size=4, attention_head_size=1, dropout=0.2, hidden_continuous_size=2, log_interval=1, log_val_interval=1, log_gradient_flow=True, ) logger = TensorBoardLogger(tmp_path) trainer = pl.Trainer(max_epochs=1, gradient_clip_val=1e-6, logger=logger) trainer.fit(net, train_dataloaders=dataset.to_dataloader(batch_size=4, num_workers=0)) # choose small batch size to provoke issue res = net.predict(dataset.to_dataloader(batch_size=2, num_workers=0), mode="raw") # check that interpretation works net.interpret_output(res)["attention"] assert net.interpret_output(res.iget( slice(1)))["attention"].size() == torch.Size( (1, net.hparams.max_encoder_length))
def _create_dataset(self, df, valid_p=0.2): df = df_utils.check_dataframe(df) df = self._handle_missing_data(df) df = df[["ds", "y"]] df["time_idx"] = range(df.shape[0]) df["series"] = 0 self.n_data = df.shape[0] self.set_auto_batch_epoch(self.n_data) training_cutoff = df.shape[0] - int(valid_p * df.shape[0]) training = TimeSeriesDataSet( df.iloc[:training_cutoff], time_idx="time_idx", target="y", categorical_encoders={"series": NaNLabelEncoder().fit(df.series)}, group_ids=["series"], min_encoder_length=self.context_length, max_encoder_length=self.context_length, max_prediction_length=self.prediction_length, min_prediction_length=self.prediction_length, time_varying_unknown_reals=["y"], target_normalizer=GroupNormalizer(groups=["series"]), randomize_length=None, add_relative_time_idx=False, add_target_scales=False, ) validation = TimeSeriesDataSet.from_dataset( training, df, min_prediction_idx=training_cutoff) train_dataloader = training.to_dataloader(train=True, batch_size=self.batch_size, num_workers=self.num_workers) val_dataloader = validation.to_dataloader(train=False, batch_size=self.batch_size, num_workers=self.num_workers) return training, train_dataloader, val_dataloader
def load_data( self, data: DataFrame, time_idx: Optional[str] = None, target: Optional[Union[str, List[str]]] = None, group_ids: Optional[List[str]] = None, parameters: Optional[Dict[str, Any]] = None, **time_series_dataset_kwargs: Any, ): if self.training: time_series_dataset = TimeSeriesDataSet( data, time_idx=time_idx, group_ids=group_ids, target=target, **time_series_dataset_kwargs) parameters = time_series_dataset.get_parameters() # Add some sample data so that we can recreate the `TimeSeriesDataSet` later on parameters["data_sample"] = data.iloc[[0]].to_dict() self.parameters = parameters else: if parameters is None: raise MisconfigurationException( "Loading data for evaluation or inference requires parameters from the train data. Either " "construct the train data at the same time as evaluation and inference or provide the train " "`datamodule.parameters` to `from_data_frame` in the `parameters` argument." ) parameters = copy(parameters) parameters.pop("data_sample") time_series_dataset = TimeSeriesDataSet.from_parameters( parameters, data, stop_randomization=True, ) return time_series_dataset
training = TimeSeriesDataSet( data[lambda x: x.time_idx <= training_cutoff], time_idx="time_idx", target="volume", group_ids=["agency", "sku"], min_encoder_length=max_encoder_length // 2, # allow encoder lengths from 0 to max_prediction_length max_encoder_length=max_encoder_length, min_prediction_length=1, max_prediction_length=max_prediction_length, static_categoricals=["agency", "sku"], static_reals=["avg_population_2017", "avg_yearly_household_income_2017"], time_varying_known_categoricals=["special_days", "month"], variable_groups={"special_days": special_days}, # group of categorical variables can be treated as one variable time_varying_known_reals=["time_idx", "price_regular", "discount_in_percent"], time_varying_unknown_categoricals=[], time_varying_unknown_reals=[ "volume", "log_volume", "industry_volume", "soda_volume", "avg_max_temp", "avg_volume_by_agency", "avg_volume_by_sku", ], target_normalizer=GroupNormalizer( groups=["agency", "sku"], transformation="softplus", center=False ), # use softplus with beta=1.0 and normalize by group add_relative_time_idx=True, add_target_scales=True, add_encoder_length=True, )
max_encoder_length = 150 max_prediction_length = 20 training_cutoff = data["time_idx"].max() - max_prediction_length context_length = max_encoder_length prediction_length = max_prediction_length training = TimeSeriesDataSet( data[lambda x: x.time_idx < training_cutoff], time_idx="time_idx", target="value", categorical_encoders={"series": NaNLabelEncoder().fit(data.series)}, group_ids=["series"], min_encoder_length=context_length, max_encoder_length=context_length, max_prediction_length=prediction_length, min_prediction_length=prediction_length, time_varying_unknown_reals=["value"], randomize_length=None, add_relative_time_idx=False, add_target_scales=False, ) validation = TimeSeriesDataSet.from_dataset(training, data, min_prediction_idx=training_cutoff) batch_size = 128 train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=2)
def predict(self, future_dataframe): """ Predicts based on the future_dataframe. Should be called only after make_future_dataframe is called Args: future_dataframe: DataFrame form make_future_dataframe function Returns: forecast dataframe """ if self.fitted is False: log.warning("Model has not been fitted. Predictions will be random.") future_dataframe = future_dataframe.copy(deep=True) testing = TimeSeriesDataSet( future_dataframe, time_idx="time_idx", target="y", categorical_encoders={"series": NaNLabelEncoder().fit(future_dataframe.series)}, group_ids=["series"], min_encoder_length=self.context_length, max_encoder_length=self.context_length, max_prediction_length=self.prediction_length, min_prediction_length=self.prediction_length, time_varying_known_reals=["time_idx"], time_varying_unknown_reals=["y"], target_normalizer=GroupNormalizer(groups=["series"], transformation="softplus", center=False), add_relative_time_idx=True, add_target_scales=True, add_encoder_length=True, ) new_raw_predictions, new_x = self.model.predict(testing, mode="raw", return_x=True) y_predicted = self.model.to_prediction(new_raw_predictions).detach().cpu() # [0, : new_x["decoder_lengths"][0]] y_predicted = y_predicted.detach().numpy() def pad_with(vector, pad_width, iaxis, kwargs): pad_value = kwargs.get("padder", np.nan) vector[: pad_width[0]] = pad_value vector[-pad_width[1] :] = pad_value y_pred_padded = np.pad(y_predicted, self.prediction_length, pad_with)[ self.prediction_length : -1, self.prediction_length : -self.prediction_length ] y_pred_padded = np.vstack([np.roll(y_pred_padded[:, i], i, axis=0) for i in range(y_pred_padded.shape[1])]).T result = pd.DataFrame( np.ones(shape=(len(future_dataframe), (2 + self.prediction_length))) * np.nan, columns=["ds", "y"] + [f"yhat{i}" for i in range(1, self.prediction_length + 1)], ) result["ds"] = future_dataframe["ds"] result.loc[: len(future_dataframe) - (self.periods + 1), "y"] = ( future_dataframe["y"].iloc[: len(future_dataframe) - (self.periods)].values ) first_part = result.iloc[: self.context_length] second_part = result.iloc[self.context_length :] second_part.loc[:, [col for col in second_part.columns[2:]]] = y_pred_padded result = pd.concat([first_part, second_part]) for i in range(1, self.prediction_length + 1): result[f"residual{i}"] = result[f"yhat{i}"] - result["y"] return result
data["cumFoam"] = data["cumFoam"] + 1e6 training = TimeSeriesDataSet( data[lambda x: x.date <= 300], time_idx="date", group_ids=["symbol"], target="open", #["open", "high", "low", "volume"], allow_missings=True, #group_ids=["agency", "sku"], min_encoder_length= 50, #max_encoder_length // 2, # allow encoder lengths from 0 to max_prediction_length max_encoder_length=100, #max_encoder_length, min_prediction_length=20, max_prediction_length=20, #max_prediction_length, static_categoricals=["symbol"], #static_reals=["avg_population_2017", "avg_yearly_household_income_2017"], #time_varying_known_categoricals=["special_days", "month"], #variable_groups={"special_days": special_days}, # group of categorical variables can be treated as one variable time_varying_known_reals=meta["knownReals"], #time_varying_unknown_categoricals=[],#meta["features"], time_varying_unknown_reals=meta["unknownReals"], #target_normalizer=NaNLabelEncoder(add_nan=True), target_normalizer=None, #GroupNormalizer( # groups=["symbol"], transformation="softplus", center=False #), # use softplus with beta=1.0 and normalize by group #add_relative_time_idx=True, #add_target_scales=True, add_encoder_length=True, categorical_encoders={"symbol": NaNLabelEncoder(add_nan=True)}) validation = TimeSeriesDataSet.from_dataset(training,
max_encoder_length = 60 max_prediction_length = 20 df_train_nbeats = df_train.copy() df_train_nbeats = df_train_nbeats.reset_index() df_train_nbeats = df_train_nbeats.reset_index() df_train_nbeats["group"] = 0 df_train_nbeats_sub, df_train_nbeats_val = utilities.split_ts(df_train_nbeats) nbeats_training = TimeSeriesDataSet( df_train_nbeats_sub, time_idx="index", target="y", categorical_encoders={ "group": NaNLabelEncoder().fit(df_train_nbeats_sub["group"]) }, group_ids=["group"], time_varying_unknown_reals=["y"], max_encoder_length=max_encoder_length, max_prediction_length=max_prediction_length, ) nbeats_validation = TimeSeriesDataSet.from_dataset(nbeats_training, df_train_nbeats_val) # %% batch_size = 128 nbeats_train_dataloader = nbeats_training.to_dataloader(train=True, batch_size=batch_size, num_workers=0) nbeats_val_dataloader = nbeats_validation.to_dataloader(train=False, batch_size=batch_size,
validation = data.series.sample(20) max_encoder_length = 60 max_prediction_length = 20 training_cutoff = data["time_idx"].max() - max_prediction_length training = TimeSeriesDataSet( data[lambda x: ~x.series.isin(validation)], time_idx="time_idx", target="value", categorical_encoders={"series": NaNLabelEncoder().fit(data.series)}, group_ids=["series"], static_categoricals=["static"], min_encoder_length=max_encoder_length, max_encoder_length=max_encoder_length, min_prediction_length=max_prediction_length, max_prediction_length=max_prediction_length, time_varying_unknown_reals=["value"], time_varying_known_reals=["time_idx"], target_normalizer=GroupNormalizer(groups=["series"]), add_relative_time_idx=False, add_target_scales=True, randomize_length=None, ) validation = TimeSeriesDataSet.from_dataset( training, data[lambda x: x.series.isin(validation)], # predict=True, stop_randomization=True, )
#value=np.arange(30), group=np.repeat(np.arange(3), 10), time_idx=np.tile(np.arange(10), 3), )) test_data # %% from pytorch_forecasting import TimeSeriesDataSet # create the dataset from the pandas dataframe dataset = TimeSeriesDataSet( test_data, group_ids=["group"], target="value", time_idx="time_idx", min_encoder_length=5, max_encoder_length=5, min_prediction_length=2, max_prediction_length=2, time_varying_unknown_reals=["value"], ) # %% dataset.get_parameters() # %% [markdown] # Now, we take a look at the output of the dataloader. It's `x` will be fed to the model's forward method, that is why it is so important to understand it. # %% # convert the dataset to a dataloader dataloader = dataset.to_dataloader(batch_size=4)