def test_advanced_degrees(self, config, input_df, input_dataset_columns): config["additional_parameters_STL"] = { "seasonal_deg": "1", "trend_deg": "1", "low_pass_deg": "1" } dku_config = STLConfig() dku_config.add_parameters(config, input_dataset_columns) timeseries_preparator = TimeseriesPreparator(dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = STLDecomposition(dku_config) result_df = decomposition.fit(df_prepared) assert result_df.shape == (26, 6) np.testing.assert_array_equal( np.round(result_df.value1_trend.values, 4), np.array([ 547017.8314, 537486.722, 528097.1954, 518846.2605, 509728.8989, 500744.2034, 491895.324, 483188.5115, 474630.5299, 466256.2782, 458496.2869, 454985.6935, 453114.0625, 452740.149, 453810.1866, 456404.7768, 463218.9767, 470913.292, 478947.2522, 487217.229, 495684.7824, 504325.6079, 513126.1746, 522081.8564, 531195.1428, 540473.2835 ])) assert np.mean(result_df["value1_trend"]) == 492101.0195351211 assert np.mean(result_df["value1_seasonal"]) == 32625.652227975654 assert np.mean(result_df["value1_residuals"]) == -5345.248686173698 config["additional_parameters_STL"] = { "seasonal_deg": "1", "trend_deg": "0" } dku_config = STLConfig() dku_config.add_parameters(config, input_dataset_columns) timeseries_preparator = TimeseriesPreparator(dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = STLDecomposition(dku_config) result_df = decomposition.fit(df_prepared) assert result_df.shape == (26, 6) np.testing.assert_array_equal( np.round(result_df.value1_seasonal.values, 4), np.array([ 334926.5396, 363552.8324, 380642.7497, 151182.772, -168020.8919, -209675.4339, -276299.0916, -289677.7104, -278165.1873, -126041.2679, -8513.4181, 175315.4394, 425222.6624, 396736.9844, 290811.7923, 45628.9471, -110941.82, -272356.2149, -303391.2037, -338667.781, -295226.877, -106373.2845, 41186.7333, 274657.8578, 516720.1595, 432742.083 ])) assert np.mean(result_df["value1_trend"]) == 470658.0934271346 assert np.mean(result_df["value1_seasonal"]) == 40229.89887290871 assert np.mean(result_df["value1_residuals"]) == 8493.430776879803
def test_target_column_preparation(self, time_column_name, timeseries_identifiers_names, basic_config): df = pd.DataFrame({ "date": [ "2020-12-31", "2021-12-15", "2022-12-01", ], "id": [1, 1, 1], "target": [1, 2, 3], "invalid_target": ["a", "b", "c"], "missing_target": [1, np.nan, 2], "unformatted_target": ["1", "2", "3"] }) dku_config = DecompositionConfig() basic_config["target_columns"] = ["target"] basic_config["frequency_unit"] = "12M" basic_config["season_length_12M"] = 4 dku_config.add_parameters(basic_config, list(df.columns)) preparator = TimeseriesPreparator(dku_config) df_prepared = preparator.prepare_timeseries_dataframe(df) assert df_prepared.loc[0, "target"] == 1 dku_config = DecompositionConfig() basic_config["target_columns"] = ["unformatted_target"] basic_config["frequency_unit"] = "12M" basic_config["season_length_12M"] = 4 dku_config.add_parameters(basic_config, list(df.columns)) preparator = TimeseriesPreparator(dku_config) df_prepared_unformatted = preparator.prepare_timeseries_dataframe(df) assert df_prepared_unformatted.loc[0, "unformatted_target"] == 1 dku_config = DecompositionConfig() basic_config["target_columns"] = ["invalid_target"] basic_config["frequency_unit"] = "12M" basic_config["season_length_12M"] = 4 dku_config.add_parameters(basic_config, list(df.columns)) preparator = TimeseriesPreparator(dku_config) with pytest.raises(ValueError) as err: _ = preparator.prepare_timeseries_dataframe(df) assert "must be numeric" in str(err.value) dku_config = DecompositionConfig() basic_config["target_columns"] = ["missing_target"] basic_config["frequency_unit"] = "12M" basic_config["season_length_12M"] = 4 dku_config.add_parameters(basic_config, list(df.columns)) preparator = TimeseriesPreparator(dku_config) with pytest.raises(ValueError) as err: _ = preparator.prepare_timeseries_dataframe(df) assert "missing value" in str(err.value)
def test_hour_truncation(): df = pd.DataFrame( { "date": [ "2020-01-07 12:12:00", "2020-01-07 17:35:00", "2020-01-07 14:55:00", "2020-01-07 18:06:00", "2020-01-08 04:40:00", "2020-01-08 06:13:00", "2020-01-08 03:23:00", ], "id": [1, 1, 1, 1, 2, 2, 2], } ) frequency = "2H" time_column_name = "date" timeseries_identifiers_names = ["id"] df[time_column_name] = pd.to_datetime(df[time_column_name]).dt.tz_localize(tz=None) preparator = TimeseriesPreparator( time_column_name=time_column_name, frequency=frequency, timeseries_identifiers_names=timeseries_identifiers_names, max_timeseries_length=2, ) dataframe_prepared = preparator._truncate_dates(df) dataframe_prepared = preparator._sort(dataframe_prepared) preparator._check_regular_frequency(dataframe_prepared) dataframe_prepared = preparator._keep_last_dates(dataframe_prepared) assert dataframe_prepared[time_column_name][0] == pd.Timestamp("2020-01-07 16:00:00") assert dataframe_prepared[time_column_name][3] == pd.Timestamp("2020-01-08 06:00:00")
def test_semester_truncation(): df = pd.DataFrame( { "date": [ "2020-12-15", "2021-06-28", "2021-12-01", ], "id": [1, 1, 1], } ) frequency = "6M" time_column_name = "date" timeseries_identifiers_names = ["id"] df[time_column_name] = pd.to_datetime(df[time_column_name]).dt.tz_localize(tz=None) preparator = TimeseriesPreparator( time_column_name=time_column_name, frequency=frequency, timeseries_identifiers_names=timeseries_identifiers_names, ) dataframe_prepared = preparator._truncate_dates(df) dataframe_prepared = preparator._sort(dataframe_prepared) preparator._check_regular_frequency(dataframe_prepared) assert dataframe_prepared[time_column_name][0] == pd.Timestamp("2020-12-31") assert dataframe_prepared[time_column_name][1] == pd.Timestamp("2021-06-30") assert dataframe_prepared[time_column_name][2] == pd.Timestamp("2021-12-31")
def test_week_sunday_truncation(): df = pd.DataFrame( { "date": [ "2021-01-03 12:12:00", "2021-01-05 17:35:00", "2021-01-15 14:55:00", ], "id": [1, 1, 1], } ) frequency = "W-SUN" time_column_name = "date" timeseries_identifiers_names = ["id"] df[time_column_name] = pd.to_datetime(df[time_column_name]).dt.tz_localize(tz=None) preparator = TimeseriesPreparator( time_column_name=time_column_name, frequency=frequency, timeseries_identifiers_names=timeseries_identifiers_names, max_timeseries_length=2, ) dataframe_prepared = preparator._truncate_dates(df) dataframe_prepared = preparator._sort(dataframe_prepared) preparator._check_regular_frequency(dataframe_prepared) dataframe_prepared = preparator._keep_last_dates(dataframe_prepared) assert dataframe_prepared[time_column_name][0] == pd.Timestamp("2021-01-10") assert dataframe_prepared[time_column_name][1] == pd.Timestamp("2021-01-17")
def test_hour_truncation(self, time_column_name, timeseries_identifiers_names, basic_config): df = pd.DataFrame({ "date": [ "2020-01-07 12:12:00", "2020-01-07 17:35:00", "2020-01-07 14:55:00", "2020-01-07 18:06:00", "2020-01-08 04:40:00", "2020-01-08 06:13:00", "2020-01-08 03:23:00", ], "id": [1, 1, 1, 1, 2, 2, 2], "target": [1, 2, 3, 4, 5, 6, 7] }) df[time_column_name] = pd.to_datetime( df[time_column_name]).dt.tz_localize(tz=None) dku_config = DecompositionConfig() basic_config["frequency_step_hours"] = "2" basic_config["frequency_unit"] = "H" basic_config["season_length_H"] = 12 basic_config["long_format"] = True basic_config["timeseries_identifiers"] = timeseries_identifiers_names dku_config.add_parameters(basic_config, list(df.columns)) preparator = TimeseriesPreparator(dku_config, max_timeseries_length=2) dataframe_prepared = preparator._truncate_dates(df) dataframe_prepared = preparator._sort(dataframe_prepared) preparator._check_regular_frequency(dataframe_prepared) dataframe_prepared = preparator._keep_last_dates(dataframe_prepared) assert dataframe_prepared[time_column_name][0] == pd.Timestamp( "2020-01-07 16:00:00") assert dataframe_prepared[time_column_name][3] == pd.Timestamp( "2020-01-08 06:00:00")
def test_minutes_truncation(self, time_column_name, basic_config): df = pd.DataFrame({ "date": [ "2021-01-01 12:17:42", "2021-01-01 12:30:00", "2021-01-01 12:46:00", ], "id": [1, 1, 1], "target": [1, 2, 3] }) dku_config = DecompositionConfig() basic_config["frequency_step_minutes"] = "15" basic_config["frequency_unit"] = "min" basic_config["season_length_min"] = 4 dku_config.add_parameters(basic_config, list(df.columns)) df[time_column_name] = pd.to_datetime( df[time_column_name]).dt.tz_localize(tz=None) preparator = TimeseriesPreparator(dku_config) dataframe_prepared = preparator._truncate_dates(df) dataframe_prepared = preparator._sort(dataframe_prepared) preparator._check_regular_frequency(dataframe_prepared) assert dataframe_prepared[time_column_name][0] == pd.Timestamp( "2021-01-01 12:15:00") assert dataframe_prepared[time_column_name][2] == pd.Timestamp( "2021-01-01 12:45:00")
def test_year_truncation(self, time_column_name, timeseries_identifiers_names, basic_config): df = pd.DataFrame({ "date": [ "2020-12-31", "2021-12-15", "2022-12-01", ], "id": [1, 1, 1], "target": [1, 2, 3] }) dku_config = DecompositionConfig() basic_config["frequency_unit"] = "12M" basic_config["season_length_12M"] = 4 basic_config["long_format"] = True basic_config["timeseries_identifiers"] = timeseries_identifiers_names basic_config["season_length_12M"] = 4 dku_config.add_parameters(basic_config, list(df.columns)) preparator = TimeseriesPreparator(dku_config) df[time_column_name] = pd.to_datetime( df[time_column_name]).dt.tz_localize(tz=None) dataframe_prepared = preparator._truncate_dates(df) dataframe_prepared = preparator._sort(dataframe_prepared) preparator._check_regular_frequency(dataframe_prepared) assert dataframe_prepared[time_column_name][0] == pd.Timestamp( "2020-12-31") assert dataframe_prepared[time_column_name][1] == pd.Timestamp( "2021-12-31") assert dataframe_prepared[time_column_name][2] == pd.Timestamp( "2022-12-31")
def test_week_sunday_truncation(self, time_column_name, timeseries_identifiers_names, basic_config): df = pd.DataFrame({ "date": [ "2021-01-03 12:12:00", "2021-01-05 17:35:00", "2021-01-15 14:55:00", ], "id": [1, 1, 1], "target": [1, 2, 3] }) dku_config = DecompositionConfig() basic_config["frequency_unit"] = "W" basic_config["frequency_end_of_week"] = "SUN" basic_config["season_length_W"] = 7 basic_config["long_format"] = True basic_config["timeseries_identifiers"] = timeseries_identifiers_names dku_config.add_parameters(basic_config, list(df.columns)) preparator = TimeseriesPreparator(dku_config, max_timeseries_length=2) df[time_column_name] = pd.to_datetime( df[time_column_name]).dt.tz_localize(tz=None) dataframe_prepared = preparator._truncate_dates(df) dataframe_prepared = preparator._sort(dataframe_prepared) preparator._check_regular_frequency(dataframe_prepared) dataframe_prepared = preparator._keep_last_dates(dataframe_prepared) assert dataframe_prepared[time_column_name][0] == pd.Timestamp( "2021-01-10") assert dataframe_prepared[time_column_name][1] == pd.Timestamp( "2021-01-17")
def test_business_day_truncation(self, time_column_name, timeseries_identifiers_names, basic_config): df = pd.DataFrame({ "date": [ "2021-01-04 12:17:42", "2021-01-05 00:00:00", "2021-01-06 12:46:00", ], "id": [1, 1, 1], "target": [1, 2, 3] }) dku_config = DecompositionConfig() basic_config["frequency_unit"] = "B" basic_config["season_length_B"] = 5 basic_config["long_format"] = True basic_config["timeseries_identifiers"] = timeseries_identifiers_names dku_config.add_parameters(basic_config, list(df.columns)) preparator = TimeseriesPreparator(dku_config) df[time_column_name] = pd.to_datetime( df[time_column_name]).dt.tz_localize(tz=None) dataframe_prepared = preparator._truncate_dates(df) dataframe_prepared = preparator._sort(dataframe_prepared) preparator._check_regular_frequency(dataframe_prepared) assert dataframe_prepared[time_column_name][0] == pd.Timestamp( "2021-01-04") assert dataframe_prepared[time_column_name][2] == pd.Timestamp( "2021-01-06")
def test_day_truncation(): df = pd.DataFrame( { "date": [ "2021-01-01 12:17:42", "2021-01-02 00:00:00", "2021-01-03 12:46:00", ], "id": [1, 1, 1], } ) frequency = "D" time_column_name = "date" timeseries_identifiers_names = ["id"] df[time_column_name] = pd.to_datetime(df[time_column_name]).dt.tz_localize(tz=None) preparator = TimeseriesPreparator( time_column_name=time_column_name, frequency=frequency, timeseries_identifiers_names=timeseries_identifiers_names, ) dataframe_prepared = preparator._truncate_dates(df) dataframe_prepared = preparator._sort(dataframe_prepared) preparator._check_regular_frequency(dataframe_prepared) assert dataframe_prepared[time_column_name][0] == pd.Timestamp("2021-01-01") assert dataframe_prepared[time_column_name][2] == pd.Timestamp("2021-01-03")
def test_collision(self, basic_dku_config, input_df): basic_dku_config.target_columns = ["value1"] input_df = input_df.rename(columns={"value2": "value1_trend"}) timeseries_preparator = TimeseriesPreparator(basic_dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = MockDecomposition(basic_dku_config) df_results = decomposition.fit(df_prepared) assert df_results.columns[3] == "value1_trend_0" assert df_results.columns[4] == "value1_seasonal"
def df_from_freq(dku_config): data = [315.58, 316.39, 316.79, 312.09, 321.08, 450.08, 298.79] freq = dku_config.frequency df = pd.DataFrame.from_dict({ "value1": data, "date": pd.date_range("1-1-1959", periods=len(data), freq=freq) }) timeseries_preparator = TimeseriesPreparator(dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe(df) return df_prepared
def test_long_format_multiple_ids(self, basic_dku_config, long_df_multiple_ids): basic_dku_config.long_format = True basic_dku_config.timeseries_identifiers = ["country", "items"] timeseries_preparator = TimeseriesPreparator(basic_dku_config) df_long_prepared = timeseries_preparator.prepare_timeseries_dataframe( long_df_multiple_ids) decomposition = MockDecomposition(basic_dku_config) df_results = decomposition.fit(df_long_prepared) np.testing.assert_equal(df_results["country"].values, np.array([0, 0, 1, 1, 1])) assert np.mean(df_results["value1_trend"]) == 2.2
def add_future_external_features(gluon_train_dataset, external_features_future_df, prediction_length, frequency): """Append the future external features to the 'feat_dynamic_real' arrays of each timeseries of the ListDataset used for training. First check that all timeseries are valid (regular time steps of the chosen frequency and they all have the same start date). Args: gluon_train_dataset (gluonts.dataset.common.ListDataset): ListDataset created with the GluonDataset class. external_features_future_df (DataFrame): Dataframe of future (dated after timeseries of gluon_train_dataset) external features. prediction_length (int): To check that external_features_future_df has the right length. frequency (str): To check that the time column has the right frequency and values. Raises: ValueError: If the length of external_features_future_df is not prediction_length. Returns: gluonts.dataset.common.ListDataset with future external features. """ gluon_dataset = copy.deepcopy(gluon_train_dataset) if isinstance(to_offset(frequency), CUSTOMISABLE_FREQUENCIES_OFFSETS): frequency = gluon_train_dataset.process.trans[0].freq start_date, periods = None, None for i, timeseries in enumerate(gluon_train_dataset): if TIMESERIES_KEYS.IDENTIFIERS in timeseries: # filter the dataframe to only get rows with the right identifiers timeseries_identifiers = timeseries[TIMESERIES_KEYS.IDENTIFIERS] conditions = [external_features_future_df[k] == v for k, v in timeseries_identifiers.items()] timeseries_external_features_future_df = apply_filter_conditions(external_features_future_df, conditions) else: timeseries_external_features_future_df = external_features_future_df feat_dynamic_real_train = timeseries[TIMESERIES_KEYS.FEAT_DYNAMIC_REAL] feat_dynamic_real_columns_names = timeseries[TIMESERIES_KEYS.FEAT_DYNAMIC_REAL_COLUMNS_NAMES] time_column_name = timeseries[TIMESERIES_KEYS.TIME_COLUMN_NAME] timeseries_preparator = TimeseriesPreparator( time_column_name=time_column_name, frequency=frequency, ) timeseries_external_features_future_df = timeseries_preparator.prepare_timeseries_dataframe(timeseries_external_features_future_df) feat_dynamic_real_future = timeseries_external_features_future_df[feat_dynamic_real_columns_names].values.T if feat_dynamic_real_future.shape[1] != prediction_length: raise ValueError(f"Please provide {prediction_length} future values of external features, as this was the forecasting horizon used for training") feat_dynamic_real_appended = np.append(feat_dynamic_real_train, feat_dynamic_real_future, axis=1) gluon_dataset.list_data[i][TIMESERIES_KEYS.FEAT_DYNAMIC_REAL] = feat_dynamic_real_appended return gluon_dataset
def test_multiplicative_model_with_negative_values(self, basic_dku_config, input_df): input_df.loc[0, "value1"] = -2 timeseries_preparator = TimeseriesPreparator(basic_dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) input_validator = DecompositionInputValidator(basic_dku_config) with pytest.raises(ValueError) as err: _ = input_validator.check(df_prepared) assert "multiplicative" in str(err.value) assert "negative" in str(err.value) assert "value1" in str(err.value)
def test_insufficient_samples_2_ts_identifiers(self, basic_dku_config, long_df): basic_dku_config.long_format = True basic_dku_config.timeseries_identifiers = ["country", "item"] timeseries_preparator = TimeseriesPreparator(basic_dku_config) df_too_short = timeseries_preparator.prepare_timeseries_dataframe( long_df) input_validator = DecompositionInputValidator(basic_dku_config) with pytest.raises(ValueError) as err: _ = input_validator.check(df_too_short) assert "need at least" in str(err.value) assert "country" in str(err.value) assert "item" in str(err.value) assert "[1 1 1 1]" in str(err.value)
def test_single_target(self, basic_dku_config, input_df): basic_dku_config.target_columns = ["value1"] timeseries_preparator = TimeseriesPreparator(basic_dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = MockDecomposition(basic_dku_config) df_results = decomposition.fit(df_prepared) size = df_prepared.shape[0] np.testing.assert_equal(df_results["value1_trend"], np.ones(size)) np.testing.assert_equal(df_results["value1_seasonal"], 2 * np.ones(size)) np.testing.assert_equal(df_results["value1_residuals"], 3 * np.ones(size))
def test_long_format(self, basic_dku_config, long_df): basic_dku_config.long_format = True basic_dku_config.timeseries_identifiers = ["country"] timeseries_preparator = TimeseriesPreparator(basic_dku_config) df_long_prepared = timeseries_preparator.prepare_timeseries_dataframe( long_df) decomposition = MockDecomposition(basic_dku_config) df_results = decomposition.fit(df_long_prepared) np.testing.assert_equal(df_results["value1_trend"], np.array([1, 1, 3, 3])) np.testing.assert_equal(df_results["value2_trend"], np.array([2, 2, 4, 4])) np.testing.assert_equal(df_results["value1_seasonal"], np.array([2, 2, 6, 6])) np.testing.assert_equal(df_results["value2_residuals"], np.array([6, 6, 12, 12]))
def test_missing_values_identifiers(): with pytest.raises(ValueError): df = pd.DataFrame( { "date": ["2018-01-06", "2018-01-07", "2018-01-08", "2018-01-06", "2018-01-07", "2018-01-08"], "volume": [2, 4, 2, 5, 2, 5], "item": [1, 1, np.NaN, 2, 2, 2], } ) timeseries_preparator = TimeseriesPreparator( time_column_name="date", frequency="D", target_columns_names=["volume"], timeseries_identifiers_names=["item"], ) training_df_prepared = timeseries_preparator.prepare_timeseries_dataframe(df)
def test_duplicate_dates(self, time_column_name, timeseries_identifiers_names, basic_config): df = pd.DataFrame({ "date": [ "2021-01-01 12:12:00", "2021-01-01 17:35:00", "2021-01-02 14:55:00", ], "id": [1, 1, 1], "target": [1, 2, 3] }) dku_config = DecompositionConfig() basic_config["frequency"] = "D" dku_config.add_parameters(basic_config, list(df.columns)) df[time_column_name] = pd.to_datetime( df[time_column_name]).dt.tz_localize(tz=None) preparator = TimeseriesPreparator(dku_config) with pytest.raises(ValueError): _ = preparator._truncate_dates(df)
def test_STL_additive(self, dku_config, input_df): timeseries_preparator = TimeseriesPreparator(dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = STLDecomposition(dku_config) results = decomposition.fit(df_prepared) expected_array = np.array([ 547017.8314, 537486.722, 528097.1954, 518846.2605, 509728.8989, 500744.2034, 491895.324, 483188.5115, 474630.5299, 466256.2782, 458496.2869, 454985.6935, 453114.0625, 452740.149, 453810.1866, 456404.7768, 463218.9767, 470913.292, 478947.2522, 487217.229, 495684.7824, 504325.6079, 513126.1746, 522081.8564, 531195.1428, 540473.2835 ]) rounded_results = np.round(results["value1_trend"].values, 4) np.testing.assert_equal(rounded_results, expected_array) assert np.mean(results["value1_trend"]) == 492101.0195351211 assert np.mean(results["value1_seasonal"]) == 32625.652227975654 assert np.mean(results["value1_residuals"]) == -5345.248686173698
def test_duplicate_dates(): df = pd.DataFrame( { "date": [ "2021-01-01 12:12:00", "2021-01-01 17:35:00", "2021-01-02 14:55:00", ], "id": [1, 1, 1], } ) frequency = "D" time_column_name = "date" timeseries_identifiers_names = ["id"] df[time_column_name] = pd.to_datetime(df[time_column_name]).dt.tz_localize(tz=None) preparator = TimeseriesPreparator( time_column_name=time_column_name, frequency=frequency, ) with pytest.raises(ValueError): dataframe_prepared = preparator._truncate_dates(df)
def test_STL_multiplicative(self, dku_config, input_df): dku_config.model = "multiplicative" timeseries_preparator = TimeseriesPreparator(dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = STLDecomposition(dku_config) results = decomposition.fit(df_prepared) expected_array = [ 1.87080328, 1.94864198, 1.97546651, 1.47349625, 0.74672304, 0.6552587, 0.5000725, 0.46825876, 0.49417933, 0.86890043, 1.16434155, 1.63725892, 2.17084151, 2.106642, 1.95377386, 1.32400823, 0.92620183, 0.51855162, 0.44493062, 0.35877353, 0.47054681, 0.94481716, 1.30967762, 1.88240591, 2.51946737, 2.28270725 ] rounded_results = np.round(results["value1_seasonal"].values, 8) np.testing.assert_equal(rounded_results, expected_array) assert np.mean(results["value1_trend"]) == 409265.35453951 assert np.mean(results["value1_seasonal"]) == 1.2698748679749627 assert np.mean(results["value1_residuals"]) == 0.9941032097902623
config = get_recipe_config() params = load_training_config(config) mxnet_context = set_mxnet_context(params["gpu_devices"]) models_parameters = get_models_parameters( config, is_training_multivariate=params["is_training_multivariate"]) start = perf_counter() training_df = params["training_dataset"].get_dataframe() timeseries_preparator = TimeseriesPreparator( time_column_name=params["time_column_name"], frequency=params["frequency"], target_columns_names=params["target_columns_names"], timeseries_identifiers_names=params["timeseries_identifiers_names"], external_features_columns_names=params["external_features_columns_names"], max_timeseries_length=params["max_timeseries_length"], ) training_df_prepared = timeseries_preparator.prepare_timeseries_dataframe( training_df) training_session = TrainingSession( target_columns_names=params["target_columns_names"], time_column_name=params["time_column_name"], frequency=params["frequency"], epoch=params["epoch"], models_parameters=models_parameters, prediction_length=params["prediction_length"], training_df=training_df_prepared,
def test_empty_input_dataset(self, dku_config, time_column_name): empty_df = pd.DataFrame(columns=["value1", "target", time_column_name]) timeseries_preparator = TimeseriesPreparator(dku_config) with pytest.raises(ValueError) as err: _ = timeseries_preparator.prepare_timeseries_dataframe(empty_df) assert "empty" in str(err.value)
from io_utils import get_input_output, set_column_description from recipe_config_loading import get_decomposition_params from safe_logger import SafeLogger from timeseries_preparation.preparation import TimeseriesPreparator logger = SafeLogger("Timeseries preparation plugin") (input_dataset, output_dataset) = get_input_output() config = get_recipe_config() input_dataset_columns = [ column["name"] for column in input_dataset.read_schema() ] (dku_config, input_validator, decomposition) = get_decomposition_params(config, input_dataset_columns) timeseries_preparator = TimeseriesPreparator(dku_config) input_df = input_dataset.get_dataframe(infer_with_pandas=False) df_prepared = timeseries_preparator.prepare_timeseries_dataframe(input_df) input_validator.check(df_prepared) start = perf_counter() logger.info("Decomposing time series...") transformed_df = decomposition.fit(df_prepared) logger.info( "Decomposing time series: Done in {:.2f} seconds".format(perf_counter() - start)) transformation_df = output_dataset.write_with_schema(transformed_df) set_column_description(output_dataset, decomposition.columns_descriptions, input_dataset)
def test_advanced_smoothers(self, config, input_df, input_dataset_columns): config["decomposition_model"] = "additive" config["additional_parameters_STL"] = {"trend": "35", "low_pass": "******"} dku_config = STLConfig() dku_config.add_parameters(config, input_dataset_columns) timeseries_preparator = TimeseriesPreparator(dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = STLDecomposition(dku_config) result_df = decomposition.fit(df_prepared) assert result_df.shape == (26, 6) np.testing.assert_array_equal( np.round(result_df.value1_seasonal.values, 4), np.array([ 329279.6394, 360305.5117, 378691.0343, 151319.491, -166075.4661, -206300.4391, -272041.7161, -285356.053, -274969.4078, -125368.4261, -10804.3636, 173084.5489, 421640.9531, 393264.9995, 288207.4229, 42573.3565, -111402.3446, -270267.5348, -299889.3857, -334837.5864, -291850.134, -103986.6224, 42205.6726, 274027.7075, 515335.6499, 429183.6225 ])) assert np.mean(result_df["value1_trend"]) == 482542.4367257319 assert np.mean(result_df["value1_seasonal"]) == 40229.62038767122 assert np.mean(result_df["value1_residuals"]) == -3390.634036480091 config["additional_parameters_STL"] = { "trend": "2999999", "low_pass": "******" } dku_config = STLConfig() dku_config.add_parameters(config, input_dataset_columns) timeseries_preparator = TimeseriesPreparator(dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = STLDecomposition(dku_config) result_df = decomposition.fit(df_prepared) assert result_df.shape == (26, 6) assert np.mean(result_df["value1_trend"]) == 476077.5935197392 assert np.mean(result_df["value1_seasonal"]) == 43303.82955718398 assert np.mean(result_df["value1_residuals"]) == -3.134258664571322e-11 config["additional_parameters_STL"] = {"trend": ""} dku_config = STLConfig() dku_config.add_parameters(config, input_dataset_columns) timeseries_preparator = TimeseriesPreparator(dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = STLDecomposition(dku_config) result_df = decomposition.fit(df_prepared) assert result_df.shape == (26, 6) assert np.mean(result_df["value1_trend"]) == 492101.0195351211 assert np.mean(result_df["value1_seasonal"]) == 32625.652227975654 assert np.mean(result_df["value1_residuals"]) == -5345.248686173698 config["additional_parameters_STL"] = {"trend": "None"} dku_config = STLConfig() dku_config.add_parameters(config, input_dataset_columns) timeseries_preparator = TimeseriesPreparator(dku_config) df_prepared = timeseries_preparator.prepare_timeseries_dataframe( input_df) decomposition = STLDecomposition(dku_config) result_df = decomposition.fit(df_prepared) assert result_df.shape == (26, 6) assert np.mean(result_df["value1_trend"]) == 492101.0195351211 assert np.mean(result_df["value1_seasonal"]) == 32625.652227975654 assert np.mean(result_df["value1_residuals"]) == -5345.248686173698