def test_CanonicalInstanceSplitter( start, target, is_train: bool, use_prediction_features: bool, allow_target_padding: bool, ): train_length = 100 pred_length = 13 t = transform.CanonicalInstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, instance_sampler=(transform.UniformSplitSampler( p=1.0, min_past=train_length, ) if is_train else ( transform.ValidationSplitSampler() if allow_target_padding else transform.TestSplitSampler())), instance_length=train_length, prediction_length=pred_length, time_series_fields=["some_time_feature"], allow_target_padding=allow_target_padding, use_prediction_features=use_prediction_features, ) assert_serializable(t) other_feat = np.arange(len(target) + 100) data = { "start": start, "target": target, "some_time_feature": other_feat, "some_other_col": "ABC", } out = list(t.flatmap_transform(data, is_train=is_train)) min_num_instances = 1 if allow_target_padding and not is_train else 0 if is_train: assert len(out) == max(min_num_instances, len(target) - train_length + 1) else: assert len(out) == 1 for o in out: assert "target" not in o assert "future_target" not in o assert "some_time_feature" not in o assert "some_other_col" in o assert len(o["past_some_time_feature"]) == train_length assert len(o["past_target"]) == train_length if use_prediction_features and not is_train: assert len(o["future_some_time_feature"]) == pred_length
def test_multi_dim_transformation(is_train): train_length = 10 first_dim: list = list(np.arange(1, 11, 1)) first_dim[-1] = "NaN" second_dim: list = list(np.arange(11, 21, 1)) second_dim[0] = "NaN" ds = gluonts.dataset.common.ListDataset( data_iter=[{"start": "2012-01-01", "target": [first_dim, second_dim]}], freq="1D", one_dim_target=False, ) pred_length = 2 # Looks weird - but this is necessary to assert the nan entries correctly. first_dim[-1] = np.nan second_dim[0] = np.nan t = transform.Chain( trans=[ transform.AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field="time_feat", time_features=[ time_feature.DayOfWeek(), time_feature.DayOfMonth(), time_feature.MonthOfYear(), ], pred_length=pred_length, ), transform.AddAgeFeature( target_field=FieldName.TARGET, output_field="age", pred_length=pred_length, log_scale=True, ), transform.AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field="observed_values", imputation_method=None, ), transform.VstackFeatures( output_field="dynamic_feat", input_fields=["age", "time_feat"], drop_inputs=True, ), transform.InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, instance_sampler=( transform.ExpectedNumInstanceSampler( num_instances=4, min_future=pred_length ) if is_train else transform.TestSplitSampler() ), past_length=train_length, future_length=pred_length, time_series_fields=["dynamic_feat", "observed_values"], output_NTC=False, ), ] ) assert_serializable(t) if is_train: for u in t(iter(ds), is_train=True): assert_shape(u["past_target"], (2, 10)) assert_shape(u["past_dynamic_feat"], (4, 10)) assert_shape(u["past_observed_values"], (2, 10)) assert_shape(u["future_target"], (2, 2)) assert_padded_array( u["past_observed_values"], np.array([[1.0] * 9 + [0.0], [0.0] + [1.0] * 9]), u["past_is_pad"], ) assert_padded_array( u["past_target"], np.array([first_dim, second_dim]), u["past_is_pad"], ) else: for u in t(iter(ds), is_train=False): assert_shape(u["past_target"], (2, 10)) assert_shape(u["past_dynamic_feat"], (4, 10)) assert_shape(u["past_observed_values"], (2, 10)) assert_shape(u["future_target"], (2, 0)) assert_padded_array( u["past_observed_values"], np.array([[1.0] * 9 + [0.0], [0.0] + [1.0] * 9]), u["past_is_pad"], ) assert_padded_array( u["past_target"], np.array([first_dim, second_dim]), u["past_is_pad"], )
def test_InstanceSplitter( start, target, lead_time: int, is_train: bool, pick_incomplete: bool ): train_length = 100 pred_length = 13 t = transform.InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, instance_sampler=( transform.UniformSplitSampler( p=1.0, min_past=0 if pick_incomplete else train_length, min_future=lead_time + pred_length, ) if is_train else transform.TestSplitSampler( min_past=0 if pick_incomplete else train_length ) ), past_length=train_length, future_length=pred_length, lead_time=lead_time, time_series_fields=["some_time_feature"], ) assert_serializable(t) other_feat = np.arange(len(target) + 100) data = { "start": start, "target": target, "some_time_feature": other_feat, "some_other_col": "ABC", } if not is_train and not pick_incomplete and len(target) < train_length: with pytest.raises(AssertionError): out = list(t.flatmap_transform(data, is_train=is_train)) return else: out = list(t.flatmap_transform(data, is_train=is_train)) if is_train: assert len(out) == max( 0, len(target) - pred_length - lead_time + 1 - (0 if pick_incomplete else train_length), ) else: assert len(out) == 1 for o in out: assert "target" not in o assert "some_time_feature" not in o assert "some_other_col" in o assert len(o["past_some_time_feature"]) == train_length assert len(o["past_target"]) == train_length if is_train: assert len(o["future_target"]) == pred_length assert len(o["future_some_time_feature"]) == pred_length else: assert len(o["future_target"]) == 0 assert len(o["future_some_time_feature"]) == pred_length