def test_covariates_as_tensors_for_location_filters_nones(self):
ts_data = {
"temperature": {
"US": [70.5, 73.0],
"CH": [72.5, 75.3],
},
"mobility": {
"US": [98.4, 70.1],
"CH": [73.5, 65.3],
"IT": [83.5, 65.0],
},
"humidity": {
"US": [34.3, 38.2],
"CH": [44.2, 42.4],
"IT": None,
}
}
covariate_feature_specs = [
model_spec.FeatureSpec(name="temperature", initializer=None),
model_spec.FeatureSpec(name="mobility", initializer=None),
model_spec.FeatureSpec(name="humidity", initializer=None),
]
expected = [
np.array([[70.5, 98.4, 34.3], [72.5, 73.5, 44.2]]),
np.array([[73.0, 70.1, 38.2], [75.3, 65.3, 42.4]])
]
actual = feature_utils.covariate_features_to_dense(
ts_data, covariate_feature_specs, ["US", "CH"], 2)
self.assertAllClose(actual, expected)
def __init__(
self,
chosen_locations,
num_known_timesteps,
forecast_window_size,
output_window_size,
static_features=None,
static_scalers=None,
static_overrides=None,
covariates=None,
ts_scalers=None,
forecasted_covariates=None,
covariate_overrides=None,
static_feature_specs=None,
covariate_feature_specs=None,
ts_categorical_features=None,
static_categorical_features=None,
covariate_feature_time_offset=0,
covariate_feature_window=1,
use_fixed_covariate_mask=True,
initial_bias=0.0,
lower_bound=-5.0,
upper_bound=5.0,
location_dependent_bias=True,
override_raw_features=True,
random_seed=0,
name="",
link_fn="identity",
distribution="linear",
trainable=True,
):
"""Initialize GAM Encoder."""
# This GAM encoder supports overriding the covariates for counterfactual
# simulations. Covariate overrides are one of:
# 1. A multiplicative factor for continuous features (not listed in
# ts_categorical_features).
# 2. An absolute value for categorical features when they are to be
# overridden.
# 3. A value of '-1' for categorical features when they are not to be
# overridden and the original value from the covariates tensor is to be
# used.
# 4. The raw, unnormalized feature values for ground-truth overrides.
self.name = name
self.override_raw_features = override_raw_features
np.random.seed(random_seed)
self.random_seed = random_seed
self.link_fn = link_fn
self.distribution = distribution
self.num_known_timesteps = num_known_timesteps
if covariate_feature_time_offset > forecast_window_size:
raise ValueError(
"Covariate_feature_time_offset cannot be greater than forecast window"
" size as there would be no weights to offset to.")
self.covariate_feature_time_offset = covariate_feature_time_offset
self.covariate_temporal_dim = covariate_feature_window
self.covariate_feature_window = covariate_feature_window
self.use_fixed_covariate_mask = use_fixed_covariate_mask
# save scalers within GamEncoder object so that member methods can call
# them. This is memory-inefficient since each encoder object instance will
# then have its own copy of the scalers, but makes it easier to access them
# within each method.
self.static_scalers = static_scalers
self.ts_scalers = ts_scalers
self.covariate_feature_specs = covariate_feature_specs
self.static_feature_specs = static_feature_specs
self.static_feature_kernel = _init_feature_kernel(
static_feature_specs or {},
kernel_name=name + "StaticFeatureKernel",
trainable=trainable,
)
self.covariate_feature_kernel = _init_feature_kernel(
covariate_feature_specs or {},
tile_size=self.covariate_temporal_dim,
kernel_name=name + "CovariateFeatureKernel",
trainable=trainable,
)
if static_overrides is not None:
if self.override_raw_features and not self.static_scalers:
raise ValueError(
"You cannot override raw static features without static_scalers."
)
overridden_static_features = dict()
for feature in static_features:
if feature not in static_overrides:
overridden_static_features[feature] = static_features[
feature]
else:
overridden_static_features[feature] = dict()
for location in static_features[feature]:
if location in static_overrides[feature]:
overridden_static_features[feature][
location] = self._apply_static_override(
feature,
static_features[feature][location],
static_overrides[feature][location])
logging.info(
"Overriding %s %s static feature %s at %s with %s (old: %s)",
type(self).__name__, self.name, feature,
location,
overridden_static_features[feature][location],
static_features[feature][location])
else:
overridden_static_features[feature][
location] = static_features[feature][location]
self.static_feature_values = feature_utils.static_feature_to_dense(
overridden_static_features or {}, static_feature_specs or {},
chosen_locations)
else:
self.static_feature_values = feature_utils.static_feature_to_dense(
static_features or {}, static_feature_specs or {},
chosen_locations)
self.static_feature_values = tf.constant(self.static_feature_values,
dtype=tf.float32)
if self.covariate_feature_kernel is not None:
self.covariate_feature_values = feature_utils.covariate_features_to_dense(
covariates or {}, covariate_feature_specs or {},
chosen_locations, num_known_timesteps)
if covariate_overrides is not None:
if self.override_raw_features and not self.ts_scalers:
raise ValueError(
"You cannot override raw ts features without ts_scalers."
)
self.covariate_feature_overrides = feature_utils.covariate_overrides_to_dense(
covariate_overrides, covariate_feature_specs or {},
chosen_locations,
num_known_timesteps + forecast_window_size)
else:
self.covariate_feature_overrides = None
self.covariate_feature_values = tf.constant(
self.covariate_feature_values, dtype=tf.float32)
else:
self.covariate_feature_values = tf.constant([], dtype=tf.float32)
# ts_categorical_features = constants.TS_CATEGORICAL_COUNTY_FEATURES
# for the first version of the What-If Tool, we are filtering categorical
# features by hardcoding a substring pattern as a filter.
# for this we need to know the *positional* name of each feature override.
# we get this by constructing a mask over the features tensor of shape
# (num_locations)X(num_all_features).
self.ts_categorical_mask = None
self.ts_continuous_mask = None
self.static_categorical_mask = None
self.static_continuous_mask = None
self.covariate_lasso_mask = None
num_locations = len(chosen_locations)
if covariate_overrides is not None:
if ts_categorical_features and covariate_feature_specs:
self.ts_categorical_mask = feature_utils.get_categorical_features_mask(
covariate_feature_specs or {},
ts_categorical_features,
num_locations,
is_static=False)
self.ts_categorical_mask = self.ts_categorical_mask.astype(
np.float32)
self.ts_continuous_mask = np.ones_like(
self.ts_categorical_mask) - self.ts_categorical_mask
if static_categorical_features and static_feature_specs:
self.static_categorical_mask = feature_utils.get_categorical_features_mask(
static_feature_specs or {},
static_categorical_features,
num_locations,
is_static=True)
self.static_categorical_mask = self.static_categorical_mask.astype(
np.float32)
self.static_continuous_mask = np.ones_like(
self.static_categorical_mask
) - self.static_categorical_mask
if self.covariate_feature_kernel is not None:
# Extract covariates name for each encoder
self.forecasted_feature_values = (
feature_utils.extract_forecasted_features(
forecasted_covariates, covariate_feature_specs))
self.forecasted_feature_values = tf.constant(np.array(
self.forecasted_feature_values),
dtype=tf.float32)
self.covariate_lasso_mask = feature_utils.get_lasso_feature_mask(
covariate_feature_specs)
# Apply location dependent biasing.
if location_dependent_bias:
initial_location_bias = initial_bias * np.ones(
len(chosen_locations))
else:
initial_location_bias = initial_bias
self.location_bias = tf.Variable(initial_location_bias,
dtype=tf.float32,
name=name + "LocationBias",
trainable=trainable)