def test_special_date_feature_set_daily_squared_exponential(): date_indices = pd.date_range(start="2016-12-24", end="2016-12-29", freq="D") reference_features = np.array( [ [ 1.00000e00, 3.67879e-01, 1.83156e-02, 1.23410e-04, 1.12535e-07, 0.00000e00, ], [ 3.67879e-01, 1.00000e00, 3.67879e-01, 1.83156e-02, 1.23410e-04, 1.12535e-07, ], ], dtype=float, ) squared_exp_kernel = squared_exponential_kernel(alpha=1.0) sfs = SpecialDateFeatureSet([CHRISTMAS_EVE, CHRISTMAS_DAY], squared_exp_kernel) computed_features = sfs(date_indices) np.testing.assert_almost_equal(computed_features, reference_features, decimal=6)
def generate_pts_m5_dataset( dataset_path: Path, pandas_freq: str, prediction_length: int = 28, alpha: float = 0.5, ): cal_path = f"{dataset_path}/calendar.csv" sales_path = f"{dataset_path}/sales_train_validation.csv" sales_test_path = f"{dataset_path}/sales_train_evaluation.csv" sell_prices_path = f"{dataset_path}/sell_prices.csv" if not os.path.exists(cal_path) or not os.path.exists(sales_path): raise RuntimeError( f"M5 data is available on Kaggle (https://www.kaggle.com/c/m5-forecasting-accuracy/data). " f"You first need to agree to the terms of the competition before being able to download the data. " f"After you have done that, please copy the files into {dataset_path}." ) # Read M5 data from dataset_path calendar = pd.read_csv(cal_path, parse_dates=True) calendar.sort_index(inplace=True) calendar.date = pd.to_datetime(calendar.date) sales_train_validation = pd.read_csv( sales_path, index_col=[ "id", "item_id", "dept_id", "cat_id", "store_id", "state_id" ], ) sales_train_validation.sort_index(inplace=True) sales_train_evaluation = pd.read_csv( sales_test_path, index_col=[ "id", "item_id", "dept_id", "cat_id", "store_id", "state_id" ], ) sales_train_evaluation.sort_index(inplace=True) sell_prices = pd.read_csv(sell_prices_path, index_col=["item_id", "store_id"]) sell_prices.sort_index(inplace=True) @lru_cache(maxsize=None) def get_sell_price(item_id, store_id): return calendar.merge(sell_prices.loc[item_id, store_id], on=["wm_yr_wk"], how="left").sell_price # Build dynamic features kernel = squared_exponential_kernel(alpha=alpha) event_1 = CustomDateFeatureSet( calendar[calendar.event_name_1.notna()].date, kernel) event_2 = CustomDateFeatureSet( calendar[calendar.event_name_2.notna()].date, kernel) snap_CA = CustomDateFeatureSet(calendar[calendar.snap_CA == 1].date, kernel) snap_TX = CustomDateFeatureSet(calendar[calendar.snap_TX == 1].date, kernel) snap_WI = CustomDateFeatureSet(calendar[calendar.snap_WI == 1].date, kernel) time_index = pd.to_datetime(calendar.date) event_1_feature = event_1(time_index) event_2_feature = event_2(time_index) snap_CA_feature = snap_CA(time_index) snap_TX_feature = snap_TX(time_index) snap_WI_feature = snap_WI(time_index) # Build static features sales_train_validation["state"] = pd.CategoricalIndex( sales_train_validation.index.get_level_values(5)).codes sales_train_validation["store"] = pd.CategoricalIndex( sales_train_validation.index.get_level_values(4)).codes sales_train_validation["cat"] = pd.CategoricalIndex( sales_train_validation.index.get_level_values(3)).codes sales_train_validation["dept"] = pd.CategoricalIndex( sales_train_validation.index.get_level_values(2)).codes sales_train_validation["item"] = pd.CategoricalIndex( sales_train_validation.index.get_level_values(1)).codes sales_train_evaluation["state"] = pd.CategoricalIndex( sales_train_evaluation.index.get_level_values(5)).codes sales_train_evaluation["store"] = pd.CategoricalIndex( sales_train_evaluation.index.get_level_values(4)).codes sales_train_evaluation["cat"] = pd.CategoricalIndex( sales_train_evaluation.index.get_level_values(3)).codes sales_train_evaluation["dept"] = pd.CategoricalIndex( sales_train_evaluation.index.get_level_values(2)).codes sales_train_evaluation["item"] = pd.CategoricalIndex( sales_train_evaluation.index.get_level_values(1)).codes feat_static_cat = [ { "name": "state_id", "cardinality": len(sales_train_validation["state"].unique()), }, { "name": "store_id", "cardinality": len(sales_train_validation["store"].unique()), }, { "name": "cat_id", "cardinality": len(sales_train_validation["cat"].unique()) }, { "name": "dept_id", "cardinality": len(sales_train_validation["dept"].unique()), }, { "name": "item_id", "cardinality": len(sales_train_validation["item"].unique()), }, ] feat_dynamic_real = [ { "name": "sell_price", "cardinality": 1 }, { "name": "event_1", "cardinality": 1 }, { "name": "event_2", "cardinality": 1 }, { "name": "snap", "cardinality": 1 }, ] # Build training set train_file = dataset_path / "train" / "data.json" train_ds = [] for index, item in sales_train_validation.iterrows(): id, item_id, dept_id, cat_id, store_id, state_id = index start_index = np.nonzero(item.iloc[:1913].values)[0][0] start_date = time_index[start_index] time_series = {} state_enc, store_enc, cat_enc, dept_enc, item_enc = item.iloc[1913:] time_series["start"] = str(start_date) time_series["item_id"] = id[:-11] time_series["feat_static_cat"] = [ state_enc, store_enc, cat_enc, dept_enc, item_enc, ] sell_price = get_sell_price(item_id, store_id) snap_feature = { "CA": snap_CA_feature, "TX": snap_TX_feature, "WI": snap_WI_feature, }[state_id] time_series["target"] = (item.iloc[start_index:1913].values.astype( np.float32).tolist()) time_series["feat_dynamic_real"] = (np.concatenate( ( np.expand_dims(sell_price.iloc[start_index:1913].values, 0), event_1_feature[:, start_index:1913], event_2_feature[:, start_index:1913], snap_feature[:, start_index:1913], ), 0, ).astype(np.float32).tolist()) train_ds.append(time_series.copy()) # Build training set train_file = dataset_path / "train" / "data.json" save_to_file(train_file, train_ds) # Create metadata file meta_file = dataset_path / "metadata.json" with open(meta_file, "w") as f: f.write( json.dumps({ "freq": pandas_freq, "prediction_length": prediction_length, "feat_static_cat": feat_static_cat, "feat_dynamic_real": feat_dynamic_real, "cardinality": len(train_ds), })) # Build testing set test_file = dataset_path / "test" / "data.json" test_ds = [] for index, item in sales_train_evaluation.iterrows(): id, item_id, dept_id, cat_id, store_id, state_id = index start_index = np.nonzero(item.iloc[:1941].values)[0][0] start_date = time_index[start_index] time_series = {} state_enc, store_enc, cat_enc, dept_enc, item_enc = item.iloc[1941:] time_series["start"] = str(start_date) time_series["item_id"] = id[:-11] time_series["feat_static_cat"] = [ state_enc, store_enc, cat_enc, dept_enc, item_enc, ] sell_price = get_sell_price(item_id, store_id) snap_feature = { "CA": snap_CA_feature, "TX": snap_TX_feature, "WI": snap_WI_feature, }[state_id] time_series["target"] = (item.iloc[start_index:1941].values.astype( np.float32).tolist()) time_series["feat_dynamic_real"] = (np.concatenate( ( np.expand_dims(sell_price.iloc[start_index:1941].values, 0), event_1_feature[:, start_index:1941], event_2_feature[:, start_index:1941], snap_feature[:, start_index:1941], ), 0, ).astype(np.float32).tolist()) test_ds.append(time_series.copy()) save_to_file(test_file, test_ds)