def create_training_network(self) -> DeepFactorTrainingNetwork:
return DeepFactorTrainingNetwork(
embedder=FeatureEmbedder(
cardinalities=self.cardinality,
embedding_dims=self.embedding_dimensions,
),
global_model=self.global_factor,
local_model=self.noise_process,
)
def create_training_network(self) -> CanonicalTrainingNetwork:
return CanonicalTrainingNetwork(
embedder=FeatureEmbedder(
cardinalities=self.cardinality,
embedding_dims=self.embedding_dimensions,
),
model=self.model,
distr_output=self.distr_output,
is_sequential=self.is_sequential,
)
def __init__(
self,
num_layers: int,
num_cells: int,
cell_type: str,
past_length: int,
prediction_length: int,
issm: ISSM,
dropout_rate: float,
cardinality: List[int],
embedding_dimension: int,
scaling: bool = True,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.num_layers = num_layers
self.num_cells = num_cells
self.cell_type = cell_type
self.past_length = past_length
self.prediction_length = prediction_length
self.issm = issm
self.dropout_rate = dropout_rate
self.cardinality = cardinality
self.embedding_dimension = embedding_dimension
self.num_cat = len(cardinality)
self.scaling = scaling
self.univariate = self.issm.output_dim() == 1
with self.name_scope():
self.prior_mean_model = mx.gluon.nn.Dense(
units=self.issm.latent_dim(), flatten=False)
self.prior_cov_diag_model = mx.gluon.nn.Dense(
units=self.issm.latent_dim(),
activation="sigmoid", # TODO: puot explicit upper bound
flatten=False,
)
self.lstm = mx.gluon.rnn.HybridSequentialRNNCell()
self.lds_proj = LDSArgsProj(output_dim=self.issm.output_dim())
for k in range(num_layers):
cell = mx.gluon.rnn.LSTMCell(hidden_size=num_cells)
cell = mx.gluon.rnn.ResidualCell(cell) if k > 0 else cell
cell = (mx.gluon.rnn.ZoneoutCell(cell,
zoneout_states=dropout_rate)
if dropout_rate > 0.0 else cell)
self.lstm.add(cell)
self.embedder = FeatureEmbedder(
cardinalities=cardinality,
embedding_dims=[embedding_dimension for _ in cardinality],
)
if scaling:
self.scaler = MeanScaler(keepdims=False)
else:
self.scaler = NOPScaler(keepdims=False)
def test_feature_embedder(config, hybridize):
out_shape = config['shape'][:-1] + (sum(
config['kwargs']['embedding_dims']), )
embed_feature = FeatureEmbedder(prefix='embed_feature_',
**config['kwargs'])
embed_feature.collect_params().initialize(mx.initializer.One())
if hybridize:
embed_feature.hybridize()
def test_parameters_length():
exp_params_len = len(embed_feature.collect_params().keys())
act_params_len = len(config['kwargs']['embedding_dims'])
assert exp_params_len == act_params_len
def test_parameter_names():
for param in embed_feature.collect_params():
assert param.startswith('embed_feature_')
def test_forward_pass():
act_output = embed_feature(mx.nd.ones(shape=config['shape']))
exp_output = mx.nd.ones(shape=out_shape)
assert act_output.shape == exp_output.shape
assert mx.nd.sum(act_output - exp_output) < 1e-20
test_parameters_length()
test_parameter_names()
test_forward_pass()
def __init__(
self,
encoder: TransformerEncoder,
decoder: TransformerDecoder,
history_length: int,
context_length: int,
prediction_length: int,
distr_output: DistributionOutput,
cardinality: List[int],
embedding_dimension: int,
lags_seq: List[int],
scaling: bool = True,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.history_length = history_length
self.context_length = context_length
self.prediction_length = prediction_length
self.scaling = scaling
self.cardinality = cardinality
self.embedding_dimension = embedding_dimension
self.distr_output = distr_output
assert len(
set(lags_seq)) == len(lags_seq), "no duplicated lags allowed!"
lags_seq.sort()
self.lags_seq = lags_seq
self.target_shape = distr_output.event_shape
with self.name_scope():
self.proj_dist_args = distr_output.get_args_proj()
self.encoder = encoder
self.decoder = decoder
self.embedder = FeatureEmbedder(
cardinalities=cardinality,
embedding_dims=[embedding_dimension for _ in cardinality],
)
if scaling:
self.scaler = MeanScaler(keepdims=True)
else:
self.scaler = NOPScaler(keepdims=True)
def __init__(
self,
freq: str,
prediction_length: int,
cardinality: List[int],
embedding_dimension: int,
encoder: Seq2SeqEncoder,
decoder_mlp_layer: List[int],
decoder_mlp_static_dim: int,
scaler: Scaler = NOPScaler(),
context_length: Optional[int] = None,
quantiles: Optional[List[float]] = None,
trainer: Trainer = Trainer(),
num_parallel_samples: int = 100,
) -> None:
assert (prediction_length >
0), "The value of `prediction_length` should be > 0"
assert (context_length is None or context_length > 0
), "The value of `context_length` should be > 0"
assert quantiles is None or all(
0 <= d <= 1 for d in
quantiles), "Elements of `quantiles` should be >= 0 and <= 1"
super().__init__(trainer=trainer)
self.context_length = (context_length if context_length is not None
else prediction_length)
self.prediction_length = prediction_length
self.freq = freq
self.quantiles = (quantiles
if quantiles is not None else [0.1, 0.5, 0.9])
self.encoder = encoder
self.decoder_mlp_layer = decoder_mlp_layer
self.decoder_mlp_static_dim = decoder_mlp_static_dim
self.scaler = scaler
self.embedder = FeatureEmbedder(
cardinalities=cardinality,
embedding_dims=[embedding_dimension for _ in cardinality],
)
self.num_parallel_samples = num_parallel_samples
def __init__(
self,
encoder: Seq2SeqEncoder,
enc2dec: Seq2SeqEnc2Dec,
decoder: Seq2SeqDecoder,
quantile_output: QuantileOutput,
context_length: int,
cardinality: List[int],
embedding_dimension: List[int],
scaling: bool = True,
dtype: DType = np.float32,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.encoder = encoder
self.enc2dec = enc2dec
self.decoder = decoder
self.quantile_output = quantile_output
self.context_length = context_length
self.cardinality = cardinality
self.embedding_dimension = embedding_dimension
self.scaling = scaling
self.dtype = dtype
if self.scaling:
self.scaler = MeanScaler(keepdims=True)
else:
self.scaler = NOPScaler(keepdims=True)
with self.name_scope():
self.quantile_proj = quantile_output.get_quantile_proj()
self.loss = quantile_output.get_loss()
self.embedder = FeatureEmbedder(
cardinalities=cardinality,
embedding_dims=embedding_dimension,
dtype=self.dtype,
)
def __init__(
self,
bin_values: mx.nd.NDArray,
n_residue: int,
n_skip: int,
dilation_depth: int,
n_stacks: int,
act_type: str,
cardinality: List[int],
embedding_dimension: int,
pred_length: int,
**kwargs,
):
super().__init__(**kwargs)
self.dilation_depth = dilation_depth
self.pred_length = pred_length
self.mu = len(bin_values)
self.dilations = WaveNet._get_dilations(
dilation_depth=dilation_depth, n_stacks=n_stacks
)
self.receptive_field = WaveNet.get_receptive_field(
dilation_depth=dilation_depth, n_stacks=n_stacks
)
self.trim_lengths = [
sum(self.dilations) - sum(self.dilations[: i + 1])
for i, _ in enumerate(self.dilations)
]
with self.name_scope():
self.feature_embedder = FeatureEmbedder(
cardinalities=cardinality,
embedding_dims=[embedding_dimension for _ in cardinality],
)
self.post_transform = LookupValues(bin_values)
self.target_embed = nn.Embedding(
input_dim=self.mu, output_dim=n_residue
)
self.residuals = nn.HybridSequential()
for i, d in enumerate(self.dilations):
is_not_last = i + 1 < len(self.dilations)
self.residuals.add(
CausalDilatedResidue(
n_residue=n_residue,
n_skip=n_skip,
dilation=d,
return_dense_out=is_not_last,
kernel_size=2,
)
)
# heuristic assuming ~5 features
std = 1.0 / math.sqrt(n_residue + 5)
self.conv_project = nn.Conv1D(
channels=n_residue,
kernel_size=1,
use_bias=True,
weight_initializer=mx.init.Uniform(std),
bias_initializer="zero",
)
self.conv1 = conv1d(
in_channels=n_skip, channels=n_skip, kernel_size=1
)
self.conv2 = conv1d(
in_channels=n_skip, channels=self.mu, kernel_size=1
)
self.output_act = (
nn.ELU()
if act_type == "elu"
else nn.Activation(act_type=act_type)
)
self.cross_entropy_loss = gluon.loss.SoftmaxCrossEntropyLoss()
def __init__(
self,
num_layers: int,
num_cells: int,
cell_type: str,
history_length: int,
context_length: int,
prediction_length: int,
distr_output_m: DistributionOutput,
distr_output_q: DistributionOutput,
dropout_rate: float,
cardinality: List[int],
embedding_dimension: List[int],
lags_seq: List[int],
scaling: bool = True,
dtype: DType = np.float32,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.num_layers = num_layers
self.num_cells = num_cells
self.cell_type = cell_type
self.history_length = history_length
self.context_length = context_length
self.prediction_length = prediction_length
self.dropout_rate = dropout_rate
self.cardinality = cardinality
self.embedding_dimension = embedding_dimension
self.num_cat = len(cardinality)
self.scaling = scaling
self.dtype = dtype
assert len(cardinality) == len(
embedding_dimension
), "embedding_dimension should be a list with the same size as cardinality"
assert len(
set(lags_seq)) == len(lags_seq), "no duplicated lags allowed!"
lags_seq.sort()
self.lags_seq = lags_seq
# 2 separate distributions
self.distr_output_m = distr_output_m
self.distr_output_q = distr_output_q
RnnCell = {
"lstm": mx.gluon.rnn.LSTMCell,
"gru": mx.gluon.rnn.GRUCell
}[self.cell_type]
self.target_shape = distr_output_m.event_shape
# TODO: is the following restriction needed?
assert (
len(self.target_shape) <= 1
), "Argument `target_shape` should be a tuple with 1 element at most"
with self.name_scope():
self.proj_distr_args_m = self.distr_output_m.get_args_proj(
prefix="m")
self.proj_distr_args_q = self.distr_output_q.get_args_proj(
prefix="q")
self.rnn = mx.gluon.rnn.HybridSequentialRNNCell()
for k in range(num_layers):
cell = RnnCell(hidden_size=num_cells)
cell = mx.gluon.rnn.ResidualCell(cell) if k > 0 else cell
cell = (mx.gluon.rnn.ZoneoutCell(cell,
zoneout_states=dropout_rate)
if dropout_rate > 0.0 else cell)
self.rnn.add(cell)
self.rnn.cast(dtype=dtype)
self.embedder = FeatureEmbedder(
cardinalities=cardinality,
embedding_dims=embedding_dimension,
dtype=self.dtype,
)
if scaling:
self.scaler = MeanScaler(keepdims=True)
else:
self.scaler = NOPScaler(keepdims=True)
def test_feature_assembler(config, hybridize):
# iterate over the power-set of all possible feature types, excluding the empty set
feature_types = {
'static_cat',
'static_real',
'dynamic_cat',
'dynamic_real',
}
feature_combs = chain.from_iterable(
combinations(feature_types, r) for r in range(1,
len(feature_types) + 1))
# iterate over the power-set of all possible feature types, including the empty set
embedder_types = {'embed_static', 'embed_dynamic'}
embedder_combs = chain.from_iterable(
combinations(embedder_types, r)
for r in range(0,
len(embedder_types) + 1))
for enabled_embedders in embedder_combs:
embed_static = (FeatureEmbedder(**config['embed_static'])
if 'embed_static' in enabled_embedders else None)
embed_dynamic = (FeatureEmbedder(**config['embed_dynamic'])
if 'embed_dynamic' in enabled_embedders else None)
for enabled_features in feature_combs:
assemble_feature = FeatureAssembler(
T=config['T'],
# use_static_cat='static_cat' in enabled_features,
# use_static_real='static_real' in enabled_features,
# use_dynamic_cat='dynamic_cat' in enabled_features,
# use_dynamic_real='dynamic_real' in enabled_features,
embed_static=embed_static,
embed_dynamic=embed_dynamic,
)
assemble_feature.collect_params().initialize(mx.initializer.One())
if hybridize:
assemble_feature.hybridize()
def test_parameters_length():
exp_params_len = sum([
len(config[k]['embedding_dims'])
for k in ['embed_static', 'embed_dynamic']
if k in enabled_embedders
])
act_params_len = len(assemble_feature.collect_params().keys())
assert exp_params_len == act_params_len
def test_parameter_names():
if embed_static:
for param in embed_static.collect_params():
assert param.startswith('static_cat_')
if embed_dynamic:
for param in embed_dynamic.collect_params():
assert param.startswith('dynamic_cat_')
def test_forward_pass():
N, T = config['N'], config['T']
inp_features = []
out_features = []
if 'static_cat' not in enabled_features:
inp_features.append(mx.nd.zeros(shape=(N, 1)))
out_features.append(mx.nd.zeros(shape=(N, T, 1)))
elif embed_static: # and 'static_cat' in enabled_features
C = config['static_cat']['C']
inp_features.append(
mx.nd.concat(
*[
mx.nd.random.uniform(
0,
config['embed_static']['cardinalities'][c],
shape=(N, 1),
).floor() for c in range(C)
],
dim=1,
))
out_features.append(
mx.nd.ones(shape=(
N,
T,
sum(config['embed_static']['embedding_dims']),
)))
else: # not embed_static and 'static_cat' in enabled_features
C = config['static_cat']['C']
inp_features.append(
mx.nd.concat(
*[
mx.nd.random.uniform(
0,
config['embed_static']['cardinalities'][c],
shape=(N, 1),
).floor() for c in range(C)
],
dim=1,
))
out_features.append(
mx.nd.tile(
mx.nd.expand_dims(inp_features[-1], axis=1),
reps=(1, T, 1),
))
if 'static_real' not in enabled_features:
inp_features.append(mx.nd.zeros(shape=(N, 1)))
out_features.append(mx.nd.zeros(shape=(N, T, 1)))
else:
C = config['static_real']['C']
static_real = mx.nd.random.uniform(0, 100, shape=(N, C))
inp_features.append(static_real)
out_features.append(
mx.nd.tile(static_real.expand_dims(axis=-2),
reps=(1, T, 1)))
if 'dynamic_cat' not in enabled_features:
inp_features.append(mx.nd.zeros(shape=(N, T, 1)))
out_features.append(mx.nd.zeros(shape=(N, T, 1)))
elif embed_dynamic: # and 'static_cat' in enabled_features
C = config['dynamic_cat']['C']
inp_features.append(
mx.nd.concat(
*[
mx.nd.random.uniform(
0,
config['embed_dynamic']['cardinalities']
[c],
shape=(N, T, 1),
).floor() for c in range(C)
],
dim=2,
))
out_features.append(
mx.nd.ones(shape=(
N,
T,
sum(config['embed_dynamic']['embedding_dims']),
)))
else: # not embed_dynamic and 'dynamic_cat' in enabled_features
C = config['dynamic_cat']['C']
inp_features.append(
mx.nd.concat(
*[
mx.nd.random.uniform(
0,
config['embed_dynamic']['cardinalities']
[c],
shape=(N, T, 1),
).floor() for c in range(C)
],
dim=2,
))
out_features.append(inp_features[-1])
if 'dynamic_real' not in enabled_features:
inp_features.append(mx.nd.zeros(shape=(N, T, 1)))
out_features.append(mx.nd.zeros(shape=(N, T, 1)))
else:
C = config['dynamic_real']['C']
dynamic_real = mx.nd.random.uniform(0,
100,
shape=(N, T, C))
inp_features.append(dynamic_real)
out_features.append(dynamic_real)
exp_output = mx.nd.concat(*out_features, dim=2)
act_output = assemble_feature(*inp_features)
assert exp_output.shape == act_output.shape
assert mx.nd.sum(exp_output - act_output) < 1e-20
test_parameters_length()
test_parameter_names()
test_forward_pass()
def __init__(
self,
num_layers: int,
num_cells: int,
cell_type: str,
past_length: int,
prediction_length: int,
issm: ISSM,
dropout_rate: float,
cardinality: List[int],
embedding_dimension: List[int],
scaling: bool = True,
noise_std_bounds: ParameterBounds = ParameterBounds(1e-6, 1.0),
prior_cov_bounds: ParameterBounds = ParameterBounds(1e-6, 1.0),
innovation_bounds: ParameterBounds = ParameterBounds(1e-6, 0.01),
**kwargs,
) -> None:
super().__init__(**kwargs)
self.num_layers = num_layers
self.num_cells = num_cells
self.cell_type = cell_type
self.past_length = past_length
self.prediction_length = prediction_length
self.issm = issm
self.dropout_rate = dropout_rate
self.cardinality = cardinality
self.embedding_dimension = embedding_dimension
self.num_cat = len(cardinality)
self.scaling = scaling
assert len(cardinality) == len(
embedding_dimension
), "embedding_dimension should be a list with the same size as cardinality"
self.univariate = self.issm.output_dim() == 1
self.noise_std_bounds = noise_std_bounds
self.prior_cov_bounds = prior_cov_bounds
self.innovation_bounds = innovation_bounds
with self.name_scope():
self.prior_mean_model = mx.gluon.nn.Dense(
units=self.issm.latent_dim(), flatten=False
)
self.prior_cov_diag_model = mx.gluon.nn.Dense(
units=self.issm.latent_dim(),
activation="sigmoid",
flatten=False,
)
self.lstm = mx.gluon.rnn.HybridSequentialRNNCell()
self.lds_proj = LDSArgsProj(
output_dim=self.issm.output_dim(),
noise_std_bounds=self.noise_std_bounds,
innovation_bounds=self.innovation_bounds,
)
for k in range(num_layers):
cell = mx.gluon.rnn.LSTMCell(hidden_size=num_cells)
cell = mx.gluon.rnn.ResidualCell(cell) if k > 0 else cell
cell = (
mx.gluon.rnn.ZoneoutCell(cell, zoneout_states=dropout_rate)
if dropout_rate > 0.0
else cell
)
self.lstm.add(cell)
self.embedder = FeatureEmbedder(
cardinalities=cardinality, embedding_dims=embedding_dimension
)
if scaling:
self.scaler = MeanScaler(keepdims=False)
else:
self.scaler = NOPScaler(keepdims=False)