def create_transformation(self) -> Transformation:
return AsNumpyArray(field=FieldName.TARGET,
expected_ndim=2,
dtype=self.dtype) + AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
)
def create_transformation(self) -> Transformation:
remove_field_names = [FieldName.FEAT_DYNAMIC_CAT]
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
if not self.use_feat_static_real:
remove_field_names.append(FieldName.FEAT_STATIC_REAL)
return Chain([RemoveFields(field_names=remove_field_names)] + (
[SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0]
)] if not self.use_feat_static_cat else []
) + ([SetField(
output_field=FieldName.FEAT_STATIC_REAL, value=[0.0]
)] if not self.use_feat_static_real else []) + [
AsNumpyArray(
field=FieldName
.TARGET,
expected_ndim=1 +
len(self.distr_output
.event_shape),
),
# maps the target to (1, T)
# if the target data is uni dimensional
ExpandDimArray(
field=FieldName.TARGET,
axis=0 if self.distr_output.event_shape[0] == 1 else None,
),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
log_scale=True,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] +
([FieldName.FEAT_DYNAMIC_REAL] if self.
use_feat_dynamic_real else []),
),
TargetDimIndicator(
field_name="target_dimension_indicator",
target_field=FieldName.TARGET,
),
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT,
expected_ndim=1,
dtype=np.long),
AsNumpyArray(field=FieldName.FEAT_STATIC_REAL, expected_ndim=1),
])
def test_simple_model():
dsinfo, training_data, test_data = default_synthetic()
freq = dsinfo.metadata.freq
prediction_length = dsinfo.prediction_length
context_length = 2 * prediction_length
hidden_dimensions = [10, 10]
net = LightningFeedForwardNetwork(
freq=freq,
prediction_length=prediction_length,
context_length=context_length,
hidden_dimensions=hidden_dimensions,
distr_output=NormalOutput(),
batch_norm=True,
scaling=mean_abs_scaling,
)
transformation = Chain([
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=ExpectedNumInstanceSampler(num_instances=1),
past_length=context_length,
future_length=prediction_length,
time_series_fields=[FieldName.OBSERVED_VALUES],
),
])
data_loader = TrainDataLoader(
training_data,
batch_size=8,
stack_fn=batchify,
transform=transformation,
num_batches_per_epoch=5,
)
trainer = pl.Trainer(max_epochs=3, callbacks=[], weights_summary=None)
trainer.fit(net, train_dataloader=data_loader)
predictor = net.get_predictor(transformation)
forecast_it, ts_it = make_evaluation_predictions(
dataset=test_data,
predictor=predictor,
num_samples=100,
)
evaluator = Evaluator(quantiles=[0.5, 0.9], num_workers=None)
agg_metrics, _ = evaluator(ts_it, forecast_it)
def create_transformation(self) -> Transformation:
remove_field_names = [FieldName.FEAT_DYNAMIC_CAT]
if not self.use_feat_static_real:
remove_field_names.append(FieldName.FEAT_STATIC_REAL)
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
return Chain(
[RemoveFields(field_names=remove_field_names)] +
([SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0.0]
)] if not self.use_feat_static_cat else []) +
([SetField(output_field=FieldName.FEAT_STATIC_REAL, value=[0.0]
)] if not self.use_feat_static_real else []) +
[
AsNumpyArray(
field=FieldName.FEAT_STATIC_CAT,
expected_ndim=1,
dtype=self.dtype,
),
AsNumpyArray(
field=FieldName.FEAT_STATIC_REAL,
expected_ndim=1,
dtype=self.dtype,
),
AsNumpyArray(
field=FieldName.TARGET,
# in the following line, we add 1 for the time dimension
expected_ndim=1 + len(self.distr_output.event_shape),
dtype=self.dtype,
),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
imputation_method=self.imputation_method,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
log_scale=True,
dtype=self.dtype,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] +
([FieldName.FEAT_DYNAMIC_REAL] if self.
use_feat_dynamic_real else []),
),
])
def create_transformation(self) -> Transformation:
remove_field_names = [
FieldName.FEAT_DYNAMIC_CAT,
FieldName.FEAT_STATIC_REAL,
]
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
return Chain(
[RemoveFields(field_names=remove_field_names)] +
([SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0.0]
)] if not self.use_feat_static_cat else []) +
[
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
AsNumpyArray(
field=FieldName.TARGET,
# in the following line, we add 1 for the time dimension
expected_ndim=1 + len(self.distr_output.event_shape),
),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
log_scale=True,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] +
([FieldName.FEAT_DYNAMIC_REAL] if self.
use_feat_dynamic_real else []),
),
InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=ExpectedNumInstanceSampler(num_instances=1),
past_length=self.history_length,
future_length=self.prediction_length,
time_series_fields=[
FieldName.FEAT_TIME,
FieldName.OBSERVED_VALUES,
],
),
])
def create_transformation(self) -> Transformation:
remove_field_names = [
FieldName.FEAT_DYNAMIC_CAT,
FieldName.FEAT_STATIC_REAL,
]
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
return Chain(
[RemoveFields(field_names=remove_field_names)]
+ (
[SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0.0])]
if not self.use_feat_static_cat
else []
)
+ [
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
AsNumpyArray(field=FieldName.TARGET, expected_ndim=1),
# gives target the (1, T) layout
ExpandDimArray(field=FieldName.TARGET, axis=0),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
# Unnormalized seasonal features
AddTimeFeatures(
time_features=self.issm.time_features(),
pred_length=self.prediction_length,
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=SEASON_INDICATORS_FIELD,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
log_scale=True,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE]
+ (
[FieldName.FEAT_DYNAMIC_REAL]
if self.use_feat_dynamic_real
else []
),
),
]
)
def create_transformation(self) -> Transformation:
return Chain([
RemoveFields(field_names=[
FieldName.FEAT_STATIC_REAL,
FieldName.FEAT_DYNAMIC_REAL,
FieldName.FEAT_DYNAMIC_CAT,
]),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
),
])
def create_transformation(
self, bin_edges: np.ndarray, pred_length: int
) -> transform.Transformation:
return Chain(
[
AsNumpyArray(field=FieldName.TARGET, expected_ndim=1),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=time_features_from_frequency_str(self.freq),
pred_length=self.prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE],
),
SetFieldIfNotPresent(
field=FieldName.FEAT_STATIC_CAT, value=[0.0]
),
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=ExpectedNumInstanceSampler(num_instances=1),
past_length=self.context_length,
future_length=pred_length,
output_NTC=False,
time_series_fields=[
FieldName.FEAT_TIME,
FieldName.OBSERVED_VALUES,
],
),
QuantizeScaled(
bin_edges=bin_edges.tolist(),
future_target="future_target",
past_target="past_target",
),
]
)
def create_transformation(self) -> Transformation:
mask_unobserved = AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
)
add_age_feature = AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
log_scale=True,
)
concat_feature = ConcatFeatures(output_field="exog",
input_fields=[FieldName.FEAT_AGE],
drop_inputs=True)
return mask_unobserved + add_age_feature + concat_feature
def create_transformation(self) -> Transformation:
return Chain([
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
),
InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=self.train_sampler,
past_length=self.context_length,
future_length=self.prediction_length,
time_series_fields=[FieldName.OBSERVED_VALUES],
),
])
def create_transformation(self) -> Transformation:
remove_field_names = [FieldName.FEAT_DYNAMIC_CAT]
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
return Chain([
RemoveFields(field_names=remove_field_names),
AsNumpyArray(
field=FieldName.TARGET,
expected_ndim=2,
),
# maps the target to (1, T)
# if the target data is uni dimensional
ExpandDimArray(
field=FieldName.TARGET,
axis=None,
),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME] +
([FieldName.FEAT_DYNAMIC_REAL]
if self.use_feat_dynamic_real else []),
),
SetFieldIfNotPresent(field=FieldName.FEAT_STATIC_CAT, value=[0]),
TargetDimIndicator(
field_name="target_dimension_indicator",
target_field=FieldName.TARGET,
),
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
])
def create_transformation(self) -> Transformation:
return Chain(trans=[
AsNumpyArray(
field=FieldName.TARGET, expected_ndim=2, dtype=self.dtype),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
),
InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=ExpectedNumInstanceSampler(num_instances=1),
time_series_fields=[FieldName.OBSERVED_VALUES],
past_length=self.context_length,
future_length=self.future_length,
output_NTC=False, # output NCT for first layer conv1d
),
])
def create_transformation(self) -> Transformation:
return Chain(
[
AsNumpyArray(
field=FieldName.TARGET,
expected_ndim=1 + len(self.distr_output.event_shape),
),
# maps the target to (1, T) if the target data is uni
# dimensional
ExpandDimArray(
field=FieldName.TARGET,
axis=0 if self.distr_output.event_shape[0] == 1 else None,
),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME],
),
SetFieldIfNotPresent(
field=FieldName.FEAT_STATIC_CAT, value=[0.0]
),
TargetDimIndicator(
field_name=FieldName.TARGET_DIM_INDICATOR,
target_field=FieldName.TARGET,
),
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
]
)
def create_transformation(self) -> Transformation:
return AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
)
def create_transformation(self) -> Transformation:
chain = []
dynamic_feat_fields = []
remove_field_names = [
FieldName.FEAT_DYNAMIC_CAT,
FieldName.FEAT_STATIC_REAL,
]
# --- GENERAL TRANSFORMATION CHAIN ---
# determine unused input
if not self.use_past_feat_dynamic_real:
remove_field_names.append(FieldName.PAST_FEAT_DYNAMIC_REAL)
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
if not self.use_feat_static_cat:
remove_field_names.append(FieldName.FEAT_STATIC_CAT)
chain.extend(
[
RemoveFields(field_names=remove_field_names),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
),
]
)
# --- TRANSFORMATION CHAIN FOR DYNAMIC FEATURES ---
if self.add_time_feature:
chain.append(
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=time_features_from_frequency_str(self.freq),
pred_length=self.prediction_length,
dtype=self.dtype,
)
)
dynamic_feat_fields.append(FieldName.FEAT_TIME)
if self.add_age_feature:
chain.append(
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
dtype=self.dtype,
)
)
dynamic_feat_fields.append(FieldName.FEAT_AGE)
if self.use_feat_dynamic_real:
# Backwards compatibility:
chain.append(
RenameFields({"dynamic_feat": FieldName.FEAT_DYNAMIC_REAL})
)
dynamic_feat_fields.append(FieldName.FEAT_DYNAMIC_REAL)
# we need to make sure that there is always some dynamic input
# we will however disregard it in the hybrid forward.
# the time feature is empty for yearly freq so also adding a dummy feature
# in the case that the time feature is the only one on
if len(dynamic_feat_fields) == 0 or (
not self.add_age_feature
and not self.use_feat_dynamic_real
and self.freq == "Y"
):
chain.append(
AddConstFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_CONST,
pred_length=self.prediction_length,
const=0.0, # For consistency in case with no dynamic features
dtype=self.dtype,
)
)
dynamic_feat_fields.append(FieldName.FEAT_CONST)
# now we map all the dynamic input of length context_length + prediction_length onto FieldName.FEAT_DYNAMIC
# we exclude past_feat_dynamic_real since its length is only context_length
if len(dynamic_feat_fields) > 1:
chain.append(
VstackFeatures(
output_field=FieldName.FEAT_DYNAMIC,
input_fields=dynamic_feat_fields,
)
)
elif len(dynamic_feat_fields) == 1:
chain.append(
RenameFields({dynamic_feat_fields[0]: FieldName.FEAT_DYNAMIC})
)
# --- TRANSFORMATION CHAIN FOR STATIC FEATURES ---
if not self.use_feat_static_cat:
chain.append(
SetField(
output_field=FieldName.FEAT_STATIC_CAT,
value=np.array([0], dtype=np.int32),
)
)
# --- SAMPLE AND CUT THE TIME-SERIES ---
chain.append(
# because of how the forking decoder works, every time step
# in context is used for splitting, which is why we use the TestSplitSampler
ForkingSequenceSplitter(
train_sampler=TestSplitSampler(),
enc_len=self.context_length,
dec_len=self.prediction_length,
num_forking=self.num_forking,
encoder_series_fields=[
FieldName.OBSERVED_VALUES,
# RTS with past and future values which is never empty because added dummy constant variable
FieldName.FEAT_DYNAMIC,
]
+ (
# RTS with only past values are only used by the encoder
[FieldName.PAST_FEAT_DYNAMIC_REAL]
if self.use_past_feat_dynamic_real
else []
),
encoder_disabled_fields=(
[FieldName.FEAT_DYNAMIC]
if not self.enable_encoder_dynamic_feature
else []
)
+ (
[FieldName.PAST_FEAT_DYNAMIC_REAL]
if not self.enable_encoder_dynamic_feature
and self.use_past_feat_dynamic_real
else []
),
decoder_series_fields=[
FieldName.OBSERVED_VALUES,
# Decoder will use all fields under FEAT_DYNAMIC which are the RTS with past and future values
FieldName.FEAT_DYNAMIC,
],
decoder_disabled_fields=(
[FieldName.FEAT_DYNAMIC]
if not self.enable_decoder_dynamic_feature
else []
),
prediction_time_decoder_exclude=[FieldName.OBSERVED_VALUES],
)
)
# past_feat_dynamic features generated above in ForkingSequenceSplitter from those under feat_dynamic - we need
# to stack with the other short related time series from the system labeled as past_past_feat_dynamic_real.
# The system labels them as past_feat_dynamic_real and the additional past_ is added to the string
# in the ForkingSequenceSplitter
if self.use_past_feat_dynamic_real:
# Stack features from ForkingSequenceSplitter horizontally since they were transposed
# so shape is now (enc_len, num_past_feature_dynamic)
chain.append(
VstackFeatures(
output_field=FieldName.PAST_FEAT_DYNAMIC,
input_fields=[
"past_" + FieldName.PAST_FEAT_DYNAMIC_REAL,
FieldName.PAST_FEAT_DYNAMIC,
],
h_stack=True,
)
)
return Chain(chain)
def create_transformation(self) -> Transformation:
chain = []
dynamic_feat_fields = []
remove_field_names = [
FieldName.FEAT_DYNAMIC_CAT,
FieldName.FEAT_STATIC_REAL,
]
# --- GENERAL TRANSFORMATION CHAIN ---
# determine unused input
if not self.use_past_feat_dynamic_real:
remove_field_names.append(FieldName.PAST_FEAT_DYNAMIC_REAL)
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
if not self.use_feat_static_cat:
remove_field_names.append(FieldName.FEAT_STATIC_CAT)
chain.extend(
[
RemoveFields(field_names=remove_field_names),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
),
]
)
# --- TRANSFORMATION CHAIN FOR DYNAMIC FEATURES ---
if self.add_time_feature:
chain.append(
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=time_features_from_frequency_str(self.freq),
pred_length=self.prediction_length,
dtype=self.dtype,
)
)
dynamic_feat_fields.append(FieldName.FEAT_TIME)
if self.add_age_feature:
chain.append(
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
dtype=self.dtype,
)
)
dynamic_feat_fields.append(FieldName.FEAT_AGE)
if self.use_feat_dynamic_real:
# Backwards compatibility:
chain.append(
RenameFields({"dynamic_feat": FieldName.FEAT_DYNAMIC_REAL})
)
dynamic_feat_fields.append(FieldName.FEAT_DYNAMIC_REAL)
# we need to make sure that there is always some dynamic input
# we will however disregard it in the hybrid forward.
# the time feature is empty for yearly freq so also adding a dummy feature
# in the case that the time feature is the only one on
if len(dynamic_feat_fields) == 0 or (
not self.add_age_feature
and not self.use_feat_dynamic_real
and self.freq == "Y"
):
chain.append(
AddConstFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_CONST,
pred_length=self.prediction_length,
const=0.0, # For consistency in case with no dynamic features
dtype=self.dtype,
)
)
dynamic_feat_fields.append(FieldName.FEAT_CONST)
# now we map all the dynamic input of length context_length + prediction_length onto FieldName.FEAT_DYNAMIC
# we exclude past_feat_dynamic_real since its length is only context_length
if len(dynamic_feat_fields) > 1:
chain.append(
VstackFeatures(
output_field=FieldName.FEAT_DYNAMIC,
input_fields=dynamic_feat_fields,
)
)
elif len(dynamic_feat_fields) == 1:
chain.append(
RenameFields({dynamic_feat_fields[0]: FieldName.FEAT_DYNAMIC})
)
# --- TRANSFORMATION CHAIN FOR STATIC FEATURES ---
if not self.use_feat_static_cat:
chain.append(
SetField(
output_field=FieldName.FEAT_STATIC_CAT,
value=np.array([0], dtype=np.int32),
)
)
return Chain(chain)
def create_transformation(self) -> Transformation:
def use_marginal_transformation(
marginal_transformation: bool, ) -> Transformation:
if marginal_transformation:
return CDFtoGaussianTransform(
target_field=FieldName.TARGET,
observed_values_field=FieldName.OBSERVED_VALUES,
max_context_length=self.conditioning_length,
target_dim=self.target_dim,
)
else:
return RenameFields({
f"past_{FieldName.TARGET}":
f"past_{FieldName.TARGET}_cdf",
f"future_{FieldName.TARGET}":
f"future_{FieldName.TARGET}_cdf",
})
return Chain([
AsNumpyArray(
field=FieldName.TARGET,
expected_ndim=1 + len(self.distr_output.event_shape),
),
# maps the target to (1, T) if the target data is uni
# dimensional
ExpandDimArray(
field=FieldName.TARGET,
axis=0 if self.distr_output.event_shape[0] == 1 else None,
),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME],
),
SetFieldIfNotPresent(field=FieldName.FEAT_STATIC_CAT, value=[0.0]),
TargetDimIndicator(
field_name=FieldName.TARGET_DIM_INDICATOR,
target_field=FieldName.TARGET,
),
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=ExpectedNumInstanceSampler(num_instances=1),
past_length=self.history_length,
future_length=self.prediction_length,
time_series_fields=[
FieldName.FEAT_TIME,
FieldName.OBSERVED_VALUES,
],
pick_incomplete=self.pick_incomplete,
),
use_marginal_transformation(self.use_marginal_transformation),
SampleTargetDim(
field_name=FieldName.TARGET_DIM_INDICATOR,
target_field=FieldName.TARGET + "_cdf",
observed_values_field=FieldName.OBSERVED_VALUES,
num_samples=self.target_dim_sample,
shuffle=self.shuffle_target_dim,
),
])
def create_transformation(self) -> Transformation:
transforms = []
if self.use_feat_dynamic_real:
transforms.append(
AsNumpyArray(
field=FieldName.FEAT_DYNAMIC_REAL,
expected_ndim=2,
))
else:
transforms.extend([
SetField(
output_field=FieldName.FEAT_DYNAMIC_REAL,
value=[[]] *
(self.context_length + self.prediction_length),
),
AsNumpyArray(
field=FieldName.FEAT_DYNAMIC_REAL,
expected_ndim=2,
),
# SwapAxes(input_fields=[FieldName.FEAT_DYNAMIC_REAL], axes=(0,1)),
])
if self.use_feat_dynamic_cat:
transforms.append(
AsNumpyArray(
field=FieldName.FEAT_DYNAMIC_CAT,
expected_ndim=2,
))
else:
# Manually set dummy dynamic categorical features and split by time
# Unknown issue in dataloader if leave splitting to InstanceSplitter
transforms.extend([
SetField(
output_field="past_" + FieldName.FEAT_DYNAMIC_CAT,
value=[[]] * self.context_length,
),
AsNumpyArray(
field="past_" + FieldName.FEAT_DYNAMIC_CAT,
expected_ndim=2,
),
SetField(
output_field="future_" + FieldName.FEAT_DYNAMIC_CAT,
value=[[]] * self.prediction_length,
),
AsNumpyArray(
field="future_" + FieldName.FEAT_DYNAMIC_CAT,
expected_ndim=2,
),
])
if self.use_feat_static_real:
transforms.append(
AsNumpyArray(
field=FieldName.FEAT_STATIC_REAL,
expected_ndim=1,
))
else:
transforms.extend([
SetField(
output_field=FieldName.FEAT_STATIC_REAL,
value=[],
),
AsNumpyArray(
field=FieldName.FEAT_STATIC_REAL,
expected_ndim=1,
),
])
if self.use_feat_static_cat:
transforms.append(
AsNumpyArray(
field=FieldName.FEAT_STATIC_CAT,
expected_ndim=1,
))
time_series_fields = [FieldName.OBSERVED_VALUES]
if self.use_feat_dynamic_cat:
time_series_fields.append(FieldName.FEAT_DYNAMIC_CAT)
if self.use_feat_dynamic_real or (self.time_features is not None):
time_series_fields.append(FieldName.FEAT_DYNAMIC_REAL)
transforms.extend([
AsNumpyArray(field=FieldName.TARGET, expected_ndim=1),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
log_scale=True,
),
VstackFeatures(
output_field=FieldName.FEAT_DYNAMIC_REAL,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] +
([FieldName.FEAT_DYNAMIC_REAL]
if self.use_feat_dynamic_real else []),
),
InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=self.train_sampler,
past_length=self.context_length,
future_length=self.prediction_length,
time_series_fields=time_series_fields,
pick_incomplete=True,
),
])
return Chain(transforms)
def create_input_transform(
is_train,
prediction_length,
past_length,
use_feat_static_cat,
use_feat_dynamic_real,
freq,
time_features,
extract_tail_chunks_for_train: bool = False,
):
SEASON_INDICATORS_FIELD = "seasonal_indicators"
remove_field_names = [
FieldName.FEAT_DYNAMIC_CAT,
FieldName.FEAT_STATIC_REAL,
]
if not use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
time_features = (
time_features
if time_features is not None
else time_features_from_frequency_str(freq)
)
transform = Chain(
[RemoveFields(field_names=remove_field_names)]
+ (
[SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0.0])]
if not use_feat_static_cat
else []
)
+ [
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
AsNumpyArray(field=FieldName.TARGET, expected_ndim=1),
# gives target the (1, T) layout
ExpandDimArray(field=FieldName.TARGET, axis=0),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
# Unnormalized seasonal features
AddTimeFeatures(
time_features=CompositeISSM.seasonal_features(freq),
pred_length=prediction_length,
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=SEASON_INDICATORS_FIELD,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=time_features,
pred_length=prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=prediction_length,
log_scale=True,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE]
+ (
[FieldName.FEAT_DYNAMIC_REAL]
if use_feat_dynamic_real
else []
),
),
CanonicalInstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
instance_sampler=ExpectedNumInstanceSampler(num_instances=1),
time_series_fields=[
FieldName.FEAT_TIME,
SEASON_INDICATORS_FIELD,
FieldName.OBSERVED_VALUES,
],
allow_target_padding=True,
instance_length=past_length,
use_prediction_features=True,
prediction_length=prediction_length,
)
if (is_train and not extract_tail_chunks_for_train)
else CanonicalInstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
instance_sampler=TestSplitSampler(),
time_series_fields=[
FieldName.FEAT_TIME,
SEASON_INDICATORS_FIELD,
FieldName.OBSERVED_VALUES,
],
allow_target_padding=True,
instance_length=past_length,
use_prediction_features=True,
prediction_length=prediction_length,
),
]
)
return transform
def create_transformation(self) -> Transformation:
remove_field_names = [FieldName.FEAT_DYNAMIC_CAT]
if not self.use_feat_static_real:
remove_field_names.append(FieldName.FEAT_STATIC_REAL)
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
return Chain(
[RemoveFields(field_names=remove_field_names)]
+ (
[SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0.0])]
if not self.use_feat_static_cat
else []
)
+ (
[SetField(output_field=FieldName.FEAT_STATIC_REAL, value=[0.0])]
if not self.use_feat_static_real
else []
)
+ [
AsNumpyArray(
field=FieldName.FEAT_STATIC_CAT,
expected_ndim=1,
dtype=self.dtype,
),
AsNumpyArray(
field=FieldName.FEAT_STATIC_REAL,
expected_ndim=1,
dtype=self.dtype,
),
AsNumpyArray(
field=FieldName.TARGET,
# in the following line, we add 1 for the time dimension
expected_ndim=1 + len(self.distr_output_m.event_shape),
dtype=self.dtype,
),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
AddInterDemandPeriodFeature(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.TARGET, # FieldName.FEAT_TIME FieldName.TARGET #if we want to append to feat time,
pred_length=self.prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
log_scale=True,
dtype=self.dtype,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE]
+ (
[FieldName.FEAT_DYNAMIC_REAL]
if self.use_feat_dynamic_real
else []
),
),
# DropNonZeroTarget(
# input_fields=[FieldName.FEAT_TIME, FieldName.OBSERVED_VALUES],
# target_field=FieldName.TARGET,
# pred_length=self.prediction_length,
# ),
RenewalInstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=ExpectedNumInstanceSampler(num_instances=1),
past_length=self.history_length,
future_length=self.prediction_length,
time_series_fields=[FieldName.FEAT_TIME, FieldName.OBSERVED_VALUES],
# dummy_value=self.distr_output_m.value_in_support,
# pick_incomplete=False
),
]
)
def create_transformation(self) -> Transformation:
remove_field_names = [
FieldName.FEAT_DYNAMIC_CAT,
FieldName.FEAT_STATIC_REAL,
]
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
return Chain(
[RemoveFields(field_names=remove_field_names)]
+ (
[SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0.0])]
if not self.use_feat_static_cat
else []
)
+ [
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
AsNumpyArray(field=FieldName.TARGET, expected_ndim=1),
# gives target the (1, T) layout
ExpandDimArray(field=FieldName.TARGET, axis=0),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
# Unnormalized seasonal features
AddTimeFeatures(
time_features=CompositeISSM.seasonal_features(self.freq),
pred_length=self.prediction_length,
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=SEASON_INDICATORS_FIELD,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
log_scale=True,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE]
+ (
[FieldName.FEAT_DYNAMIC_REAL]
if self.use_feat_dynamic_real
else []
),
),
CanonicalInstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
instance_sampler=TestSplitSampler(),
time_series_fields=[
FieldName.FEAT_TIME,
SEASON_INDICATORS_FIELD,
FieldName.OBSERVED_VALUES,
],
allow_target_padding=True,
instance_length=self.past_length,
use_prediction_features=True,
prediction_length=self.prediction_length,
),
]
)
def create_transformation(self): # Feature transformation that the model uses for input. return AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, )
def create_transformation(self) -> Transformation:
chain = []
dynamic_feat_fields = []
remove_field_names = [
FieldName.FEAT_DYNAMIC_CAT,
FieldName.FEAT_STATIC_REAL,
]
# --- GENERAL TRANSFORMATION CHAIN ---
# determine unused input
if not self.use_feat_dynamic_real:
remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL)
if not self.use_feat_static_cat:
remove_field_names.append(FieldName.FEAT_STATIC_CAT)
chain.extend([
RemoveFields(field_names=remove_field_names),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
dtype=self.dtype,
),
])
# --- TRANSFORMATION CHAIN FOR DYNAMIC FEATURES ---
if self.add_time_feature:
chain.append(
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=time_features_from_frequency_str(self.freq),
pred_length=self.prediction_length,
), )
dynamic_feat_fields.append(FieldName.FEAT_TIME)
if self.add_age_feature:
chain.append(
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.prediction_length,
dtype=self.dtype,
), )
dynamic_feat_fields.append(FieldName.FEAT_AGE)
if self.use_feat_dynamic_real:
dynamic_feat_fields.append(FieldName.FEAT_DYNAMIC_REAL)
# we need to make sure that there is always some dynamic input
# we will however disregard it in the hybrid forward
if len(dynamic_feat_fields) == 0:
chain.append(
AddConstFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_CONST,
pred_length=self.prediction_length,
dtype=self.dtype,
), )
dynamic_feat_fields.append(FieldName.FEAT_CONST)
# now we map all the dynamic input onto FieldName.FEAT_DYNAMIC
if len(dynamic_feat_fields) > 1:
chain.append(
VstackFeatures(
output_field=FieldName.FEAT_DYNAMIC,
input_fields=dynamic_feat_fields,
))
elif len(dynamic_feat_fields) == 1:
chain.append(
RenameFields({dynamic_feat_fields[0]: FieldName.FEAT_DYNAMIC}))
# --- TRANSFORMATION CHAIN FOR STATIC FEATURES ---
if not self.use_feat_static_cat:
chain.append(
SetField(
output_field=FieldName.FEAT_STATIC_CAT,
value=np.array([0.0]),
), )
# --- SAMPLE AND CUT THE TIME-SERIES ---
chain.append(
# because of how the forking decoder works, every time step
# in context is used for splitting, which is why we use the TestSplitSampler
ForkingSequenceSplitter(
train_sampler=TestSplitSampler(),
enc_len=self.context_length,
dec_len=self.prediction_length,
encoder_series_fields=[
FieldName.OBSERVED_VALUES,
FieldName.FEAT_DYNAMIC,
],
decoder_series_fields=[FieldName.OBSERVED_VALUES] +
([FieldName.FEAT_DYNAMIC]
if self.enable_decoder_dynamic_feature else []),
prediction_time_decoder_exclude=[FieldName.OBSERVED_VALUES],
), )
return Chain(chain)
def create_transformation(self) -> Transformation:
transforms = ([AsNumpyArray(field=FieldName.TARGET, expected_ndim=1)] +
([
AsNumpyArray(field=name, expected_ndim=1)
for name in self.static_cardinalities.keys()
]) + [
AsNumpyArray(field=name, expected_ndim=1)
for name in chain(
self.static_feature_dims.keys(),
self.dynamic_cardinalities.keys(),
)
] + [
AsNumpyArray(field=name, expected_ndim=2)
for name in self.dynamic_feature_dims.keys()
] + [
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
])
if self.static_cardinalities:
transforms.append(
VstackFeatures(
output_field=FieldName.FEAT_STATIC_CAT,
input_fields=list(self.static_cardinalities.keys()),
h_stack=True,
))
else:
transforms.extend([
SetField(
output_field=FieldName.FEAT_STATIC_CAT,
value=[0.0],
),
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
])
if self.static_feature_dims:
transforms.append(
VstackFeatures(
output_field=FieldName.FEAT_STATIC_REAL,
input_fields=list(self.static_feature_dims.keys()),
h_stack=True,
))
else:
transforms.extend([
SetField(
output_field=FieldName.FEAT_STATIC_REAL,
value=[0.0],
),
AsNumpyArray(field=FieldName.FEAT_STATIC_REAL,
expected_ndim=1),
])
if self.dynamic_cardinalities:
transforms.append(
VstackFeatures(
output_field=FieldName.FEAT_DYNAMIC_CAT,
input_fields=list(self.dynamic_cardinalities.keys()),
))
else:
transforms.extend([
SetField(
output_field=FieldName.FEAT_DYNAMIC_CAT,
value=[[0.0]],
),
AsNumpyArray(
field=FieldName.FEAT_DYNAMIC_CAT,
expected_ndim=2,
),
BroadcastTo(
field=FieldName.FEAT_DYNAMIC_CAT,
ext_length=self.prediction_length,
),
])
input_fields = [FieldName.FEAT_TIME]
if self.dynamic_feature_dims:
input_fields += list(self.dynamic_feature_dims.keys())
transforms.append(
VstackFeatures(
input_fields=input_fields,
output_field=FieldName.FEAT_DYNAMIC_REAL,
))
if self.past_dynamic_cardinalities:
transforms.append(
VstackFeatures(
output_field=FieldName.PAST_FEAT_DYNAMIC + "_cat",
input_fields=list(self.past_dynamic_cardinalities.keys()),
))
else:
transforms.extend([
SetField(
output_field=FieldName.PAST_FEAT_DYNAMIC + "_cat",
value=[[0.0]],
),
AsNumpyArray(
field=FieldName.PAST_FEAT_DYNAMIC + "_cat",
expected_ndim=2,
),
BroadcastTo(field=FieldName.PAST_FEAT_DYNAMIC + "_cat"),
])
if self.past_dynamic_feature_dims:
transforms.append(
VstackFeatures(
output_field=FieldName.PAST_FEAT_DYNAMIC_REAL,
input_fields=list(self.past_dynamic_feature_dims.keys()),
))
else:
transforms.extend([
SetField(
output_field=FieldName.PAST_FEAT_DYNAMIC_REAL,
value=[[0.0]],
),
AsNumpyArray(field=FieldName.PAST_FEAT_DYNAMIC_REAL,
expected_ndim=2),
BroadcastTo(field=FieldName.PAST_FEAT_DYNAMIC_REAL),
])
return Chain(transforms)
def transform_data(self):
# 首先需要把 target
time_features = time_features_from_frequency_str(self.config.freq)
self.time_dim = len(time_features) + 1 # 考虑多加了一个 age_features
seasonal = CompositeISSM.seasonal_features(self.freq)
self.seasonal_dim = len(seasonal)
transformation = Chain([
SwapAxes(
input_fields=[FieldName.TARGET],
axes=(0, 1),
),
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
# Unnormalized seasonal features
AddTimeFeatures(
time_features=seasonal,
pred_length=self.pred_length,
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field="seasonal_indicators",
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=time_features,
pred_length=self.pred_length,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_AGE,
pred_length=self.pred_length,
log_scale=True,
),
VstackFeatures(
output_field=FieldName.FEAT_TIME,
input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] +
([FieldName.FEAT_DYNAMIC_REAL]
if self.use_feat_dynamic_real else []),
),
InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=TestSplitSampler(),
past_length=self.config.past_length,
future_length=self.config.pred_length,
output_NTC=True,
time_series_fields=[
FieldName.FEAT_TIME, FieldName.OBSERVED_VALUES,
"seasonal_indicators"
],
pick_incomplete=False,
)
])
print('已设置时间特征~~')
# 设置环境变量的 dataloader
target_train_iters = [
iter(
TrainDataLoader_OnlyPast(
dataset=self.target_data[i].train,
transform=transformation,
batch_size=self.config.batch_size,
num_batches_per_epoch=self.config.num_batches_per_epoch,
)) for i in range(len(self.target_data))
]
target_test_iters = [
iter(
InferenceDataLoader_WithFuture(
dataset=self.target_data[i].test,
transform=transformation,
batch_size=self.config.batch_size,
)) for i in range(len(self.target_data))
]
self.target_train_loader = stackIterOut(
target_train_iters,
fields=[FieldName.OBSERVED_VALUES, FieldName.TARGET],
dim=0,
include_future=False)
self.target_test_loader = stackIterOut(
target_test_iters,
fields=[FieldName.OBSERVED_VALUES, FieldName.TARGET],
dim=0,
include_future=True)
}]
ds2 = ListDataset(
[{
"start": "2020/01/01",
"target": [1, 2, 3, np.nan, 5, np.nan, 7, np.nan, np.nan, 10],
}],
freq="1D",
)
@pytest.mark.parametrize("ds", [ds1, ds2])
@pytest.mark.parametrize(
"transform",
[
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddAgeFeature(
target_field=FieldName.TARGET,
output_field="age_feature",
pred_length=1,
),
AddConstFeature(
target_field=FieldName.TARGET,
output_field="constant",
pred_length=1,
),
],
)
def test_dataset_imutability(ds, transform):
ds_c = deepcopy(ds)
def create_transformation(self) -> Transformation:
transforms = ([AsNumpyArray(field=FieldName.TARGET, expected_ndim=1)] +
([
AsNumpyArray(field=name, expected_ndim=1)
for name in self.static_cardinalities.keys()
]) + [
AsNumpyArray(field=name, expected_ndim=1)
for name in chain(
self.static_feature_dims.keys(),
self.dynamic_cardinalities.keys(),
)
] + [
AsNumpyArray(field=name, expected_ndim=2)
for name in self.dynamic_feature_dims.keys()
] + [
AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
),
AddTimeFeatures(
start_field=FieldName.START,
target_field=FieldName.TARGET,
output_field=FieldName.FEAT_TIME,
time_features=self.time_features,
pred_length=self.prediction_length,
),
])
ts_fields = []
past_ts_fields = []
if self.static_cardinalities:
transforms.append(
VstackFeatures(
output_field=FieldName.FEAT_STATIC_CAT,
input_fields=list(self.static_cardinalities.keys()),
))
else:
transforms.extend([
SetField(
output_field=FieldName.FEAT_STATIC_CAT,
value=[0.0],
),
AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1),
])
if self.static_feature_dims:
transforms.append(
VstackFeatures(
output_field=FieldName.FEAT_STATIC_REAL,
input_fields=list(self.static_feature_dims.keys()),
))
else:
transforms.extend([
SetField(
output_field=FieldName.FEAT_STATIC_REAL,
value=[0.0],
),
AsNumpyArray(field=FieldName.FEAT_STATIC_REAL,
expected_ndim=1),
])
if self.dynamic_cardinalities:
transforms.append(
VstackFeatures(
output_field=FieldName.FEAT_DYNAMIC_CAT,
input_fields=list(self.dynamic_cardinalities.keys()),
))
ts_fields.append(FieldName.FEAT_DYNAMIC_CAT)
else:
transforms.extend([
SetField(
output_field=FieldName.FEAT_DYNAMIC_CAT,
value=[0.0],
),
AsNumpyArray(field=FieldName.FEAT_DYNAMIC_CAT,
expected_ndim=1),
])
input_fields = [FieldName.FEAT_TIME]
if self.dynamic_feature_dims:
input_fields += list(self.dynamic_feature_dims.keys())
transforms.append(
VstackFeatures(
input_fields=input_fields,
output_field=FieldName.FEAT_DYNAMIC_REAL,
))
ts_fields.append(FieldName.FEAT_DYNAMIC_REAL)
if self.past_dynamic_cardinalities:
transforms.append(
VstackFeatures(
output_field=FieldName.PAST_FEAT_DYNAMIC + "_cat",
input_fields=list(self.past_dynamic_cardinalities.keys()),
))
past_ts_fields.append(FieldName.PAST_FEAT_DYNAMIC + "_cat")
else:
transforms.extend([
SetField(
output_field=FieldName.PAST_FEAT_DYNAMIC + "_cat",
value=[0.0],
),
AsNumpyArray(
field=FieldName.PAST_FEAT_DYNAMIC + "_cat",
expected_ndim=1,
),
])
if self.past_dynamic_feature_dims:
transforms.append(
VstackFeatures(
output_field=FieldName.PAST_FEAT_DYNAMIC_REAL,
input_fields=list(self.past_dynamic_feature_dims.keys()),
))
past_ts_fields.append(FieldName.PAST_FEAT_DYNAMIC_REAL)
else:
transforms.extend([
SetField(
output_field=FieldName.PAST_FEAT_DYNAMIC_REAL,
value=[[0.0]],
),
AsNumpyArray(field=FieldName.PAST_FEAT_DYNAMIC_REAL,
expected_ndim=2),
])
transforms.append(
TFTInstanceSplitter(
train_sampler=ExpectedNumInstanceSampler(
num_instances=self.num_instance_per_series, ),
past_length=self.context_length,
future_length=self.prediction_length,
time_series_fields=ts_fields,
past_time_series_fields=past_ts_fields,
))
return Chain(transforms)
def create_transformation(self) -> Transformation:
mask_unobserved = AddObservedValuesIndicator(
target_field=FieldName.TARGET,
output_field=FieldName.OBSERVED_VALUES,
)
return mask_unobserved
