def create_transformation(self) -> transform.Transformation: return Chain( [ AsNumpyArray(field=FieldName.TARGET, expected_ndim=1), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=time_features_from_frequency_str(self.freq), pred_length=self.prediction_length, ), AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=self.prediction_length, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE], ), SetFieldIfNotPresent( field=FieldName.FEAT_STATIC_CAT, value=[0.0] ), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), ] )
def create_transformation(self) -> Transformation: return Chain( trans=[ AsNumpyArray( field=FieldName.TARGET, expected_ndim=2, dtype=self.dtype ), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, dtype=self.dtype, ), InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=self.train_sampler, time_series_fields=[FieldName.OBSERVED_VALUES], past_length=self.context_length, future_length=self.prediction_length, lead_time=self.lead_time, output_NTC=False, # output NCT for first layer conv2d ), ] )
def create_transformation(self) -> Transformation: return Chain( [ AsNumpyArray(field=FieldName.TARGET, expected_ndim=1), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), SetFieldIfNotPresent( field=FieldName.FEAT_STATIC_CAT, value=[0.0] ), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), CanonicalInstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, instance_sampler=TestSplitSampler(), time_series_fields=[FieldName.FEAT_TIME], instance_length=self.context_length, use_prediction_features=True, prediction_length=self.prediction_length, ), ] )
def create_transformation(freq, context_length, prediction_length): return Chain( [ AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=prediction_length, log_scale=True, ), InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, instance_sampler=ExpectedNumInstanceSampler( num_instances=1, min_future=prediction_length, ), past_length=context_length, future_length=prediction_length, time_series_fields=[ FieldName.FEAT_AGE, FieldName.FEAT_DYNAMIC_REAL, FieldName.OBSERVED_VALUES, ], ), ] )
def create_transformation(self) -> Transformation: return Chain([ AsNumpyArray( field=FieldName.TARGET, expected_ndim=1 + len(self.distr_output.event_shape), ), # maps the target to (1, T) if the target data is uni # dimensional ExpandDimArray( field=FieldName.TARGET, axis=0 if self.distr_output.event_shape[0] == 1 else None, ), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME], ), SetFieldIfNotPresent(field=FieldName.FEAT_STATIC_CAT, value=[0.0]), TargetDimIndicator( field_name=FieldName.TARGET_DIM_INDICATOR, target_field=FieldName.TARGET, ), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), ])
def create_transformation(self) -> Transformation: return Chain( trans=[ AsNumpyArray(field=FieldName.TARGET, expected_ndim=1), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=time_features_from_frequency_str(self.freq), pred_length=self.prediction_length, ), SetFieldIfNotPresent( field=FieldName.FEAT_STATIC_CAT, value=[0.0] ), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), transform.InstanceSplitter( target_field=transform.FieldName.TARGET, is_pad_field=transform.FieldName.IS_PAD, start_field=transform.FieldName.START, forecast_start_field=transform.FieldName.FORECAST_START, train_sampler=TestSplitSampler(), time_series_fields=[FieldName.FEAT_TIME], past_length=self.context_length, future_length=self.prediction_length, ), ] )
def create_transformation(self) -> Transformation: remove_field_names = [] if not self.use_feat_static_real: remove_field_names.append(FieldName.FEAT_STATIC_REAL) if not self.use_feat_dynamic_real: remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL) if not self.use_feat_dynamic_cat: remove_field_names.append(FieldName.FEAT_DYNAMIC_CAT) return Chain( [RemoveFields(field_names=remove_field_names)] + ([SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0] )] if not self.use_feat_static_cat else []) + ([SetField(output_field=FieldName.FEAT_STATIC_REAL, value=[0.0] )] if not self.use_feat_static_real else []) + [ AsNumpyArray( field=FieldName.FEAT_STATIC_CAT, expected_ndim=1, dtype=np.long, ), AsNumpyArray( field=FieldName.FEAT_STATIC_REAL, expected_ndim=1, dtype=self.dtype, ), AsNumpyArray( field=FieldName.TARGET, # in the following line, we add 1 for the time dimension expected_ndim=1 + len(self.distr_output.event_shape), dtype=self.dtype, ), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, dtype=self.dtype, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=self.prediction_length, log_scale=True, dtype=self.dtype, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] + ([FieldName.FEAT_DYNAMIC_REAL] if self. use_feat_dynamic_real else []) + ([FieldName.FEAT_DYNAMIC_CAT] if self. use_feat_dynamic_cat else []), ), ])
def create_transformation(self) -> Transformation: remove_field_names = [FieldName.FEAT_DYNAMIC_CAT] if not self.use_feat_static_real: remove_field_names.append(FieldName.FEAT_STATIC_REAL) if not self.use_feat_dynamic_real: remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL) return Chain( [RemoveFields(field_names=remove_field_names)] + ([SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0.0] )] if not self.use_feat_static_cat else []) + ([SetField(output_field=FieldName.FEAT_STATIC_REAL, value=[0.0] )] if not self.use_feat_static_real else []) + [ AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), AsNumpyArray(field=FieldName.FEAT_STATIC_REAL, expected_ndim=1), AsNumpyArray( field=FieldName.TARGET, # in the following line, we add 1 for the time dimension expected_ndim=1 + len(self.distr_output.event_shape), ), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=self.prediction_length, log_scale=True, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] + ([FieldName.FEAT_DYNAMIC_REAL] if self. use_feat_dynamic_real else []), ), InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=ExpectedNumInstanceSampler(num_instances=1), past_length=self.history_length, future_length=self.prediction_length, time_series_fields=[ FieldName.FEAT_TIME, FieldName.OBSERVED_VALUES, ], ), ])
def create_transformation(self) -> Transformation: remove_field_names = [FieldName.FEAT_DYNAMIC_CAT] if not self.use_feat_dynamic_real: remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL) if not self.use_feat_static_real: remove_field_names.append(FieldName.FEAT_STATIC_REAL) return Chain([RemoveFields(field_names=remove_field_names)] + ( [SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0] )] if not self.use_feat_static_cat else [] ) + ([SetField( output_field=FieldName.FEAT_STATIC_REAL, value=[0.0] )] if not self.use_feat_static_real else []) + [ AsNumpyArray( field=FieldName .TARGET, expected_ndim=1 + len(self.distr_output .event_shape), ), # maps the target to (1, T) # if the target data is uni dimensional ExpandDimArray( field=FieldName.TARGET, axis=0 if self.distr_output.event_shape[0] == 1 else None, ), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=self.prediction_length, log_scale=True, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] + ([FieldName.FEAT_DYNAMIC_REAL] if self. use_feat_dynamic_real else []), ), TargetDimIndicator( field_name="target_dimension_indicator", target_field=FieldName.TARGET, ), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1, dtype=np.long), AsNumpyArray(field=FieldName.FEAT_STATIC_REAL, expected_ndim=1), ])
def get_dataset_and_transformation(): # dont recompute, since expensive global _data_cache if _data_cache is not None: return _data_cache # create constant dataset with each time series having # variable length and unique constant integer entries dataset = ConstantDataset( num_steps=CD_NUM_STEPS, num_timeseries=CD_NUM_TIME_SERIES ) list_dataset = list(dataset.train) for i, ts in enumerate(list_dataset): ts["start"] = pd.Timestamp(ts_input=ts["start"], freq=dataset.freq) # get randomness in the ts lengths ts["target"] = np.array( ts["target"] * random.randint(1, CD_MAX_LEN_MULTIPLICATION_FACTOR) ) list_dataset = ListDataset(data_iter=list_dataset, freq=dataset.freq) list_dataset_pred_length = dataset.prediction_length # use every possible time point to split the time series transformation = Chain( [ InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=UniformSplitSampler( p=SPLITTING_SAMPLE_PROBABILITY # THIS IS IMPORTANT FOR THE TEST ), past_length=CONTEXT_LEN, future_length=list_dataset_pred_length, dummy_value=1.0, ), ] ) # original no multiprocessing processed validation dataset train_data_transformed_original = list( ValidationDataLoader( dataset=list_dataset, transform=transformation, batch_size=BATCH_SIZE, num_workers=0, # This is the crucial difference ctx=current_context(), ) ) _data_cache = ( list_dataset, transformation, list_dataset_pred_length, train_data_transformed_original, ) return _data_cache
def create_transformation(self) -> Transformation: return Chain(trans=[ AsNumpyArray(field=FieldName.TARGET, expected_ndim=1), ForkingSequenceSplitter( train_sampler=TestSplitSampler(), enc_len=self.context_length, dec_len=self.prediction_length, ), ])
def test_simple_model(): dsinfo, training_data, test_data = default_synthetic() freq = dsinfo.metadata.freq prediction_length = dsinfo.prediction_length context_length = 2 * prediction_length hidden_dimensions = [10, 10] net = LightningFeedForwardNetwork( freq=freq, prediction_length=prediction_length, context_length=context_length, hidden_dimensions=hidden_dimensions, distr_output=NormalOutput(), batch_norm=True, scaling=mean_abs_scaling, ) transformation = Chain([ AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=ExpectedNumInstanceSampler(num_instances=1), past_length=context_length, future_length=prediction_length, time_series_fields=[FieldName.OBSERVED_VALUES], ), ]) data_loader = TrainDataLoader( training_data, batch_size=8, stack_fn=batchify, transform=transformation, num_batches_per_epoch=5, ) trainer = pl.Trainer(max_epochs=3, callbacks=[], weights_summary=None) trainer.fit(net, train_dataloader=data_loader) predictor = net.get_predictor(transformation) forecast_it, ts_it = make_evaluation_predictions( dataset=test_data, predictor=predictor, num_samples=100, ) evaluator = Evaluator(quantiles=[0.5, 0.9], num_workers=None) agg_metrics, _ = evaluator(ts_it, forecast_it)
def create_transformation(self) -> Transformation: remove_field_names = [ FieldName.FEAT_DYNAMIC_CAT, FieldName.FEAT_STATIC_REAL, ] if not self.use_feat_dynamic_real: remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL) return Chain( [RemoveFields(field_names=remove_field_names)] + ( [SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0.0])] if not self.use_feat_static_cat else [] ) + [ AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), AsNumpyArray(field=FieldName.TARGET, expected_ndim=1), # gives target the (1, T) layout ExpandDimArray(field=FieldName.TARGET, axis=0), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), # Unnormalized seasonal features AddTimeFeatures( time_features=self.issm.time_features(), pred_length=self.prediction_length, start_field=FieldName.START, target_field=FieldName.TARGET, output_field=SEASON_INDICATORS_FIELD, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=self.prediction_length, log_scale=True, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] + ( [FieldName.FEAT_DYNAMIC_REAL] if self.use_feat_dynamic_real else [] ), ), ] )
def create_transformation(self) -> Transformation: return Chain( trans=[ AsNumpyArray(field=FieldName.TARGET, expected_ndim=2, dtype=self.dtype), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, dtype=self.dtype, ), ] )
def create_transformation(self) -> Transformation: return Chain([ InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=self.train_sampler, past_length=self.context_length, future_length=self.prediction_length, time_series_fields=[], ) ])
def create_transformation(self) -> Transformation: return Chain([ InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=ExpectedNumInstanceSampler(num_instances=1), past_length=self.context_length, future_length=self.prediction_length, time_series_fields=[], # [FieldName.FEAT_DYNAMIC_REAL] ) ])
def create_transformation(self) -> Transformation: return Chain(trans=[ AsNumpyArray(field=FieldName.TARGET, expected_ndim=1), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=time_features_from_frequency_str(self.freq), pred_length=self.prediction_length, ), SetFieldIfNotPresent(field=FieldName.FEAT_STATIC_CAT, value=[0.0]), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), ])
def create_transformation(self) -> Transformation: return Chain([ ContinuousTimeInstanceSplitter( past_interval_length=self.context_interval_length, future_interval_length=self.prediction_interval_length, train_sampler=ContinuousTimeUniformSampler( num_instances=self.num_training_instances), ), RenameFields({ "past_target": "target", "past_valid_length": "valid_length", }), ])
def create_transformation(self) -> Transformation: return Chain([ RemoveFields(field_names=[ FieldName.FEAT_STATIC_REAL, FieldName.FEAT_DYNAMIC_REAL, FieldName.FEAT_DYNAMIC_CAT, ]), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, dtype=self.dtype, ), ])
def create_transformation( self, bin_edges: np.ndarray, pred_length: int ) -> transform.Transformation: return Chain( [ AsNumpyArray(field=FieldName.TARGET, expected_ndim=1), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=time_features_from_frequency_str(self.freq), pred_length=self.prediction_length, ), AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=self.prediction_length, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE], ), SetFieldIfNotPresent( field=FieldName.FEAT_STATIC_CAT, value=[0.0] ), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=ExpectedNumInstanceSampler(num_instances=1), past_length=self.context_length, future_length=pred_length, output_NTC=False, time_series_fields=[ FieldName.FEAT_TIME, FieldName.OBSERVED_VALUES, ], ), QuantizeScaled( bin_edges=bin_edges.tolist(), future_target="future_target", past_target="past_target", ), ] )
def _create_post_split_transform(): return Chain([ CountTrailingZeros( new_field="time_remaining", target_field="past_target", as_array=True, ), ToIntervalSizeFormat(target_field="past_target", discard_first=True), RenameFields({"future_target": "sparse_future"}), AsNumpyArray(field="past_target", expected_ndim=2), SwapAxes(input_fields=["past_target"], axes=(0, 1)), AddAxisLength(target_field="past_target", axis=0), ])
def create_transformation(self) -> Transformation: return Chain([ AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, dtype=self.dtype, ), InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=self.train_sampler, past_length=self.context_length, future_length=self.prediction_length, time_series_fields=[FieldName.OBSERVED_VALUES], ), ])
def create_transformation(self) -> Transformation: return Chain([ AsNumpyArray( field=FieldName.TARGET, # in the following line, we add 1 for the time dimension expected_ndim=1 + len(self.distr_output.event_shape), ), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=self.prediction_length, log_scale=True, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE], ), SetFieldIfNotPresent(field=FieldName.FEAT_STATIC_CAT, value=[0.0]), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), InstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, train_sampler=ExpectedNumInstanceSampler(num_instances=1), past_length=self.history_length, future_length=self.prediction_length, time_series_fields=[ FieldName.FEAT_TIME, FieldName.OBSERVED_VALUES, ], ), ])
def create_transformation(self) -> Transformation: remove_field_names = [FieldName.FEAT_DYNAMIC_CAT] if not self.use_feat_dynamic_real: remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL) return Chain([ RemoveFields(field_names=remove_field_names), AsNumpyArray( field=FieldName.TARGET, expected_ndim=2, ), # maps the target to (1, T) # if the target data is uni dimensional ExpandDimArray( field=FieldName.TARGET, axis=None, ), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME] + ([FieldName.FEAT_DYNAMIC_REAL] if self.use_feat_dynamic_real else []), ), SetFieldIfNotPresent(field=FieldName.FEAT_STATIC_CAT, value=[0]), TargetDimIndicator( field_name="target_dimension_indicator", target_field=FieldName.TARGET, ), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), ])
def _create_instance_splitter(self, mode: str): assert mode in ["training", "validation", "test"] instance_sampler = { "training": ContinuousTimeUniformSampler( num_instances=self.num_training_instances, min_past=self.context_interval_length, min_future=self.prediction_interval_length, ), "validation": ContinuousTimePredictionSampler( allow_empty_interval=True, min_past=self.context_interval_length, min_future=self.prediction_interval_length, ), "test": ContinuousTimePredictionSampler( min_past=self.context_interval_length, allow_empty_interval=False, ), }[mode] assert isinstance(instance_sampler, ContinuousTimePointSampler) return Chain( [ ContinuousTimeInstanceSplitter( past_interval_length=self.context_interval_length, future_interval_length=self.prediction_interval_length, instance_sampler=instance_sampler, ), RenameFields( { "past_target": "target", "past_valid_length": "valid_length", } ), ] )
def create_transformation(self) -> Transformation: return Chain([])
def create_transformation(self) -> Transformation: remove_field_names = [ FieldName.FEAT_DYNAMIC_CAT, FieldName.FEAT_STATIC_REAL, ] if not self.use_feat_dynamic_real: remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL) return Chain( [RemoveFields(field_names=remove_field_names)] + ( [SetField(output_field=FieldName.FEAT_STATIC_CAT, value=[0.0])] if not self.use_feat_static_cat else [] ) + [ AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), AsNumpyArray(field=FieldName.TARGET, expected_ndim=1), # gives target the (1, T) layout ExpandDimArray(field=FieldName.TARGET, axis=0), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), # Unnormalized seasonal features AddTimeFeatures( time_features=CompositeISSM.seasonal_features(self.freq), pred_length=self.prediction_length, start_field=FieldName.START, target_field=FieldName.TARGET, output_field=SEASON_INDICATORS_FIELD, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=self.prediction_length, log_scale=True, ), VstackFeatures( output_field=FieldName.FEAT_TIME, input_fields=[FieldName.FEAT_TIME, FieldName.FEAT_AGE] + ( [FieldName.FEAT_DYNAMIC_REAL] if self.use_feat_dynamic_real else [] ), ), CanonicalInstanceSplitter( target_field=FieldName.TARGET, is_pad_field=FieldName.IS_PAD, start_field=FieldName.START, forecast_start_field=FieldName.FORECAST_START, instance_sampler=TestSplitSampler(), time_series_fields=[ FieldName.FEAT_TIME, SEASON_INDICATORS_FIELD, FieldName.OBSERVED_VALUES, ], allow_target_padding=True, instance_length=self.past_length, use_prediction_features=True, prediction_length=self.prediction_length, ), ] )
def create_transformation(self) -> Transformation: transforms = ([AsNumpyArray(field=FieldName.TARGET, expected_ndim=1)] + ([ AsNumpyArray(field=name, expected_ndim=1) for name in self.static_cardinalities.keys() ]) + [ AsNumpyArray(field=name, expected_ndim=1) for name in chain( self.static_feature_dims.keys(), self.dynamic_cardinalities.keys(), ) ] + [ AsNumpyArray(field=name, expected_ndim=2) for name in self.dynamic_feature_dims.keys() ] + [ AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, ), AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=self.time_features, pred_length=self.prediction_length, ), ]) if self.static_cardinalities: transforms.append( VstackFeatures( output_field=FieldName.FEAT_STATIC_CAT, input_fields=list(self.static_cardinalities.keys()), h_stack=True, )) else: transforms.extend([ SetField( output_field=FieldName.FEAT_STATIC_CAT, value=[0.0], ), AsNumpyArray(field=FieldName.FEAT_STATIC_CAT, expected_ndim=1), ]) if self.static_feature_dims: transforms.append( VstackFeatures( output_field=FieldName.FEAT_STATIC_REAL, input_fields=list(self.static_feature_dims.keys()), h_stack=True, )) else: transforms.extend([ SetField( output_field=FieldName.FEAT_STATIC_REAL, value=[0.0], ), AsNumpyArray(field=FieldName.FEAT_STATIC_REAL, expected_ndim=1), ]) if self.dynamic_cardinalities: transforms.append( VstackFeatures( output_field=FieldName.FEAT_DYNAMIC_CAT, input_fields=list(self.dynamic_cardinalities.keys()), )) else: transforms.extend([ SetField( output_field=FieldName.FEAT_DYNAMIC_CAT, value=[[0.0]], ), AsNumpyArray( field=FieldName.FEAT_DYNAMIC_CAT, expected_ndim=2, ), BroadcastTo( field=FieldName.FEAT_DYNAMIC_CAT, ext_length=self.prediction_length, ), ]) input_fields = [FieldName.FEAT_TIME] if self.dynamic_feature_dims: input_fields += list(self.dynamic_feature_dims.keys()) transforms.append( VstackFeatures( input_fields=input_fields, output_field=FieldName.FEAT_DYNAMIC_REAL, )) if self.past_dynamic_cardinalities: transforms.append( VstackFeatures( output_field=FieldName.PAST_FEAT_DYNAMIC + "_cat", input_fields=list(self.past_dynamic_cardinalities.keys()), )) else: transforms.extend([ SetField( output_field=FieldName.PAST_FEAT_DYNAMIC + "_cat", value=[[0.0]], ), AsNumpyArray( field=FieldName.PAST_FEAT_DYNAMIC + "_cat", expected_ndim=2, ), BroadcastTo(field=FieldName.PAST_FEAT_DYNAMIC + "_cat"), ]) if self.past_dynamic_feature_dims: transforms.append( VstackFeatures( output_field=FieldName.PAST_FEAT_DYNAMIC_REAL, input_fields=list(self.past_dynamic_feature_dims.keys()), )) else: transforms.extend([ SetField( output_field=FieldName.PAST_FEAT_DYNAMIC_REAL, value=[[0.0]], ), AsNumpyArray(field=FieldName.PAST_FEAT_DYNAMIC_REAL, expected_ndim=2), BroadcastTo(field=FieldName.PAST_FEAT_DYNAMIC_REAL), ]) return Chain(transforms)
def _create_instance_splitter(self, mode: str): assert mode in ["training", "validation", "test"] instance_sampler = { "training": self.train_sampler, "validation": self.validation_sampler, "test": TestSplitSampler(), }[mode] chain = [] chain.append( # because of how the forking decoder works, every time step # in context is used for splitting, which is why we use the TestSplitSampler ForkingSequenceSplitter( instance_sampler=instance_sampler, enc_len=self.context_length, dec_len=self.prediction_length, num_forking=self.num_forking, encoder_series_fields=[ FieldName.OBSERVED_VALUES, # RTS with past and future values which is never empty because added dummy constant variable FieldName.FEAT_DYNAMIC, ] + ( # RTS with only past values are only used by the encoder [FieldName.PAST_FEAT_DYNAMIC_REAL] if self.use_past_feat_dynamic_real else [] ), encoder_disabled_fields=( [FieldName.FEAT_DYNAMIC] if not self.enable_encoder_dynamic_feature else [] ) + ( [FieldName.PAST_FEAT_DYNAMIC_REAL] if not self.enable_encoder_dynamic_feature and self.use_past_feat_dynamic_real else [] ), decoder_series_fields=[ # Decoder will use all fields under FEAT_DYNAMIC which are the RTS with past and future values FieldName.FEAT_DYNAMIC, ] + ([FieldName.OBSERVED_VALUES] if mode is not "test" else []), decoder_disabled_fields=( [FieldName.FEAT_DYNAMIC] if not self.enable_decoder_dynamic_feature else [] ), ) ) # past_feat_dynamic features generated above in ForkingSequenceSplitter from those under feat_dynamic - we need # to stack with the other short related time series from the system labeled as past_past_feat_dynamic_real. # The system labels them as past_feat_dynamic_real and the additional past_ is added to the string # in the ForkingSequenceSplitter if self.use_past_feat_dynamic_real: # Stack features from ForkingSequenceSplitter horizontally since they were transposed # so shape is now (enc_len, num_past_feature_dynamic) chain.append( VstackFeatures( output_field=FieldName.PAST_FEAT_DYNAMIC, input_fields=[ "past_" + FieldName.PAST_FEAT_DYNAMIC_REAL, FieldName.PAST_FEAT_DYNAMIC, ], h_stack=True, ) ) return Chain(chain)
def create_transformation(self) -> Transformation: chain = [] dynamic_feat_fields = [] remove_field_names = [ FieldName.FEAT_DYNAMIC_CAT, FieldName.FEAT_STATIC_REAL, ] # --- GENERAL TRANSFORMATION CHAIN --- # determine unused input if not self.use_past_feat_dynamic_real: remove_field_names.append(FieldName.PAST_FEAT_DYNAMIC_REAL) if not self.use_feat_dynamic_real: remove_field_names.append(FieldName.FEAT_DYNAMIC_REAL) if not self.use_feat_static_cat: remove_field_names.append(FieldName.FEAT_STATIC_CAT) chain.extend( [ RemoveFields(field_names=remove_field_names), AddObservedValuesIndicator( target_field=FieldName.TARGET, output_field=FieldName.OBSERVED_VALUES, dtype=self.dtype, ), ] ) # --- TRANSFORMATION CHAIN FOR DYNAMIC FEATURES --- if self.add_time_feature: chain.append( AddTimeFeatures( start_field=FieldName.START, target_field=FieldName.TARGET, output_field=FieldName.FEAT_TIME, time_features=time_features_from_frequency_str(self.freq), pred_length=self.prediction_length, dtype=self.dtype, ) ) dynamic_feat_fields.append(FieldName.FEAT_TIME) if self.add_age_feature: chain.append( AddAgeFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_AGE, pred_length=self.prediction_length, dtype=self.dtype, ) ) dynamic_feat_fields.append(FieldName.FEAT_AGE) if self.use_feat_dynamic_real: # Backwards compatibility: chain.append( RenameFields({"dynamic_feat": FieldName.FEAT_DYNAMIC_REAL}) ) dynamic_feat_fields.append(FieldName.FEAT_DYNAMIC_REAL) # we need to make sure that there is always some dynamic input # we will however disregard it in the hybrid forward. # the time feature is empty for yearly freq so also adding a dummy feature # in the case that the time feature is the only one on if len(dynamic_feat_fields) == 0 or ( not self.add_age_feature and not self.use_feat_dynamic_real and self.freq == "Y" ): chain.append( AddConstFeature( target_field=FieldName.TARGET, output_field=FieldName.FEAT_CONST, pred_length=self.prediction_length, const=0.0, # For consistency in case with no dynamic features dtype=self.dtype, ) ) dynamic_feat_fields.append(FieldName.FEAT_CONST) # now we map all the dynamic input of length context_length + prediction_length onto FieldName.FEAT_DYNAMIC # we exclude past_feat_dynamic_real since its length is only context_length if len(dynamic_feat_fields) > 1: chain.append( VstackFeatures( output_field=FieldName.FEAT_DYNAMIC, input_fields=dynamic_feat_fields, ) ) elif len(dynamic_feat_fields) == 1: chain.append( RenameFields({dynamic_feat_fields[0]: FieldName.FEAT_DYNAMIC}) ) # --- TRANSFORMATION CHAIN FOR STATIC FEATURES --- if not self.use_feat_static_cat: chain.append( SetField( output_field=FieldName.FEAT_STATIC_CAT, value=np.array([0], dtype=np.int32), ) ) # --- SAMPLE AND CUT THE TIME-SERIES --- chain.append( # because of how the forking decoder works, every time step # in context is used for splitting, which is why we use the TestSplitSampler ForkingSequenceSplitter( train_sampler=TestSplitSampler(), enc_len=self.context_length, dec_len=self.prediction_length, num_forking=self.num_forking, encoder_series_fields=[ FieldName.OBSERVED_VALUES, # RTS with past and future values which is never empty because added dummy constant variable FieldName.FEAT_DYNAMIC, ] + ( # RTS with only past values are only used by the encoder [FieldName.PAST_FEAT_DYNAMIC_REAL] if self.use_past_feat_dynamic_real else [] ), encoder_disabled_fields=( [FieldName.FEAT_DYNAMIC] if not self.enable_encoder_dynamic_feature else [] ) + ( [FieldName.PAST_FEAT_DYNAMIC_REAL] if not self.enable_encoder_dynamic_feature and self.use_past_feat_dynamic_real else [] ), decoder_series_fields=[ FieldName.OBSERVED_VALUES, # Decoder will use all fields under FEAT_DYNAMIC which are the RTS with past and future values FieldName.FEAT_DYNAMIC, ], decoder_disabled_fields=( [FieldName.FEAT_DYNAMIC] if not self.enable_decoder_dynamic_feature else [] ), prediction_time_decoder_exclude=[FieldName.OBSERVED_VALUES], ) ) # past_feat_dynamic features generated above in ForkingSequenceSplitter from those under feat_dynamic - we need # to stack with the other short related time series from the system labeled as past_past_feat_dynamic_real. # The system labels them as past_feat_dynamic_real and the additional past_ is added to the string # in the ForkingSequenceSplitter if self.use_past_feat_dynamic_real: # Stack features from ForkingSequenceSplitter horizontally since they were transposed # so shape is now (enc_len, num_past_feature_dynamic) chain.append( VstackFeatures( output_field=FieldName.PAST_FEAT_DYNAMIC, input_fields=[ "past_" + FieldName.PAST_FEAT_DYNAMIC_REAL, FieldName.PAST_FEAT_DYNAMIC, ], h_stack=True, ) ) return Chain(chain)