def test_covariates_as_tensors_for_location_filters_nones(self): ts_data = { "temperature": { "US": [70.5, 73.0], "CH": [72.5, 75.3], }, "mobility": { "US": [98.4, 70.1], "CH": [73.5, 65.3], "IT": [83.5, 65.0], }, "humidity": { "US": [34.3, 38.2], "CH": [44.2, 42.4], "IT": None, } } covariate_feature_specs = [ model_spec.FeatureSpec(name="temperature", initializer=None), model_spec.FeatureSpec(name="mobility", initializer=None), model_spec.FeatureSpec(name="humidity", initializer=None), ] expected = [ np.array([[70.5, 98.4, 34.3], [72.5, 73.5, 44.2]]), np.array([[73.0, 70.1, 38.2], [75.3, 65.3, 42.4]]) ] actual = feature_utils.covariate_features_to_dense( ts_data, covariate_feature_specs, ["US", "CH"], 2) self.assertAllClose(actual, expected)
def __init__( self, chosen_locations, num_known_timesteps, forecast_window_size, output_window_size, static_features=None, static_scalers=None, static_overrides=None, covariates=None, ts_scalers=None, forecasted_covariates=None, covariate_overrides=None, static_feature_specs=None, covariate_feature_specs=None, ts_categorical_features=None, static_categorical_features=None, covariate_feature_time_offset=0, covariate_feature_window=1, use_fixed_covariate_mask=True, initial_bias=0.0, lower_bound=-5.0, upper_bound=5.0, location_dependent_bias=True, override_raw_features=True, random_seed=0, name="", link_fn="identity", distribution="linear", trainable=True, ): """Initialize GAM Encoder.""" # This GAM encoder supports overriding the covariates for counterfactual # simulations. Covariate overrides are one of: # 1. A multiplicative factor for continuous features (not listed in # ts_categorical_features). # 2. An absolute value for categorical features when they are to be # overridden. # 3. A value of '-1' for categorical features when they are not to be # overridden and the original value from the covariates tensor is to be # used. # 4. The raw, unnormalized feature values for ground-truth overrides. self.name = name self.override_raw_features = override_raw_features np.random.seed(random_seed) self.random_seed = random_seed self.link_fn = link_fn self.distribution = distribution self.num_known_timesteps = num_known_timesteps if covariate_feature_time_offset > forecast_window_size: raise ValueError( "Covariate_feature_time_offset cannot be greater than forecast window" " size as there would be no weights to offset to.") self.covariate_feature_time_offset = covariate_feature_time_offset self.covariate_temporal_dim = covariate_feature_window self.covariate_feature_window = covariate_feature_window self.use_fixed_covariate_mask = use_fixed_covariate_mask # save scalers within GamEncoder object so that member methods can call # them. This is memory-inefficient since each encoder object instance will # then have its own copy of the scalers, but makes it easier to access them # within each method. self.static_scalers = static_scalers self.ts_scalers = ts_scalers self.covariate_feature_specs = covariate_feature_specs self.static_feature_specs = static_feature_specs self.static_feature_kernel = _init_feature_kernel( static_feature_specs or {}, kernel_name=name + "StaticFeatureKernel", trainable=trainable, ) self.covariate_feature_kernel = _init_feature_kernel( covariate_feature_specs or {}, tile_size=self.covariate_temporal_dim, kernel_name=name + "CovariateFeatureKernel", trainable=trainable, ) if static_overrides is not None: if self.override_raw_features and not self.static_scalers: raise ValueError( "You cannot override raw static features without static_scalers." ) overridden_static_features = dict() for feature in static_features: if feature not in static_overrides: overridden_static_features[feature] = static_features[ feature] else: overridden_static_features[feature] = dict() for location in static_features[feature]: if location in static_overrides[feature]: overridden_static_features[feature][ location] = self._apply_static_override( feature, static_features[feature][location], static_overrides[feature][location]) logging.info( "Overriding %s %s static feature %s at %s with %s (old: %s)", type(self).__name__, self.name, feature, location, overridden_static_features[feature][location], static_features[feature][location]) else: overridden_static_features[feature][ location] = static_features[feature][location] self.static_feature_values = feature_utils.static_feature_to_dense( overridden_static_features or {}, static_feature_specs or {}, chosen_locations) else: self.static_feature_values = feature_utils.static_feature_to_dense( static_features or {}, static_feature_specs or {}, chosen_locations) self.static_feature_values = tf.constant(self.static_feature_values, dtype=tf.float32) if self.covariate_feature_kernel is not None: self.covariate_feature_values = feature_utils.covariate_features_to_dense( covariates or {}, covariate_feature_specs or {}, chosen_locations, num_known_timesteps) if covariate_overrides is not None: if self.override_raw_features and not self.ts_scalers: raise ValueError( "You cannot override raw ts features without ts_scalers." ) self.covariate_feature_overrides = feature_utils.covariate_overrides_to_dense( covariate_overrides, covariate_feature_specs or {}, chosen_locations, num_known_timesteps + forecast_window_size) else: self.covariate_feature_overrides = None self.covariate_feature_values = tf.constant( self.covariate_feature_values, dtype=tf.float32) else: self.covariate_feature_values = tf.constant([], dtype=tf.float32) # ts_categorical_features = constants.TS_CATEGORICAL_COUNTY_FEATURES # for the first version of the What-If Tool, we are filtering categorical # features by hardcoding a substring pattern as a filter. # for this we need to know the *positional* name of each feature override. # we get this by constructing a mask over the features tensor of shape # (num_locations)X(num_all_features). self.ts_categorical_mask = None self.ts_continuous_mask = None self.static_categorical_mask = None self.static_continuous_mask = None self.covariate_lasso_mask = None num_locations = len(chosen_locations) if covariate_overrides is not None: if ts_categorical_features and covariate_feature_specs: self.ts_categorical_mask = feature_utils.get_categorical_features_mask( covariate_feature_specs or {}, ts_categorical_features, num_locations, is_static=False) self.ts_categorical_mask = self.ts_categorical_mask.astype( np.float32) self.ts_continuous_mask = np.ones_like( self.ts_categorical_mask) - self.ts_categorical_mask if static_categorical_features and static_feature_specs: self.static_categorical_mask = feature_utils.get_categorical_features_mask( static_feature_specs or {}, static_categorical_features, num_locations, is_static=True) self.static_categorical_mask = self.static_categorical_mask.astype( np.float32) self.static_continuous_mask = np.ones_like( self.static_categorical_mask ) - self.static_categorical_mask if self.covariate_feature_kernel is not None: # Extract covariates name for each encoder self.forecasted_feature_values = ( feature_utils.extract_forecasted_features( forecasted_covariates, covariate_feature_specs)) self.forecasted_feature_values = tf.constant(np.array( self.forecasted_feature_values), dtype=tf.float32) self.covariate_lasso_mask = feature_utils.get_lasso_feature_mask( covariate_feature_specs) # Apply location dependent biasing. if location_dependent_bias: initial_location_bias = initial_bias * np.ones( len(chosen_locations)) else: initial_location_bias = initial_bias self.location_bias = tf.Variable(initial_location_bias, dtype=tf.float32, name=name + "LocationBias", trainable=trainable)