def __init__(self,
hidden_layers,
num_outputs=None,
activation_fn="relu",
name=None,
**kwargs):
super(DenseNetwork, self).__init__(name=name)
self.layers = [
Dense(
size,
name=f"fc_{k}",
activation=activation_fn,
kernel_initializer=initializers.glorot_normal(),
bias_initializer=initializers.constant(0.1),
) for k, size in enumerate(hidden_layers)
]
if num_outputs:
self.layers.append(
Dense(
num_outputs,
name="fc_out",
activation=None,
kernel_initializer=initializers.glorot_normal(),
bias_initializer=initializers.constant(0.1),
))
def __init__(self, num_classes=6, L1=128, L2=256,
cell_units=128, num_linear=768, p=10, time_step=100,
F1=64, dropout_keep_prob=0.8):
super().__init__()
# strides = [1,1,1,1]
self.conv1 = Conv2D(filters=L1, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.dropout_keep_prob = dropout_keep_prob
self.max_pool = MaxPool2D(pool_size=2, strides=2, padding='valid')
self.conv2 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.conv3 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.conv4 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.conv5 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.conv6 = Conv2D(filters=L2, kernel_size=(5, 3), padding="same", use_bias=True,
bias_initializer=initializers.constant(0.1))
self.dropout = Dropout(dropout_keep_prob)
self.reshape1 = Reshape((-1, time_step, L2 * p))
self.reshape2 = Reshape((-1, L2 * p))
# self.reshape3 = Reshape((-1, time_step, num_linear))
self.flatten = Flatten()
self.d1 = Dense(num_linear, use_bias=True, bias_initializer=initializers.constant(0.1),
kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.1))
self.bn = BatchNormalization()
self.d2 = Dense(F1, use_bias=True, bias_initializer=initializers.constant(0.1),
kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.1))
self.d3 = Dense(num_classes, use_bias=True, bias_initializer=initializers.constant(0.1),
kernel_initializer=initializers.TruncatedNormal(mean=0., stddev=0.1), activation='softmax')
self.bilstm = Bidirectional(LSTM(cell_units, return_sequences=True), merge_mode='concat')
self.attention = attention()
def __init__(self,
tau,
bias=1.0,
nonlinearity='linear',
tau_learn='True',
**kwargs):
self.tau_ = constant(tau)
self.tau_learn_ = tau_learn
self.bias_ = constant(bias)
self.nonlinearity = nonlinearity
super(TauLayer, self).__init__(**kwargs)
def _create_model_predictor(self, lr):
y = self.y
Z0_start = y[:, 0]
V0_start = (np.argmax(np.cumsum(y > 0, axis=1) == 2, axis=1)
- np.argmax(np.cumsum(y > 0, axis=1) == 1, axis=1)).reshape((y.shape[0], 1))
inp_y = Input(shape=(None, 1))
inp_emb_id = Input(shape=(1,)) # Dummy ID for embeddings
inp_y_decoded = inp_y
if self.seasonality:
for seasonality in self.seasonality:
inp_y_decoded = seasonality.apply_decoder(inp_y_decoded)
# (Ab)using embeddings here for initial value variables
init_Z0 = Embedding(y.shape[0], 1, embeddings_initializer=constant(Z0_start), name="Z0")(inp_emb_id)[:, 0, :]
init_V0 = Embedding(y.shape[0], 1, embeddings_initializer=constant(V0_start), name="V0")(inp_emb_id)[:, 0, :]
rnncell = CrostonsRNN(y.shape[0], self.sba)
rnn = RNN(rnncell, return_sequences=True, return_state=True)
out_rnn = rnn((inp_y_decoded, tf.cast(inp_emb_id[:, None, :] * tf.ones_like(inp_y_decoded), tf.int32)),
initial_state=[init_Z0, init_V0, tf.ones_like(inp_emb_id)])
if self.sba:
initial_out = ((init_Z0 / init_V0) * (1 - rnncell.alpha / 2))
else:
initial_out = (init_Z0 / init_V0)
out = tf.keras.layers.concatenate([
initial_out[:, :, None],
out_rnn[0]
], axis=1)
if self.seasonality:
for seasonality in self.seasonality:
out = seasonality.apply_encoder(out)
model = Model(inputs=[inp_y, inp_emb_id]
+ ([input
for seasonality in self.seasonality
for input in seasonality.get_model_inputs()] if self.seasonality else []),
outputs=out)
model.compile(Adam(lr), self.loss)
# predictor also outputs final state for predicting out-of-sample
predictor = Model(inputs=[inp_y, inp_emb_id]
+ ([input
for seasonality in self.seasonality
for input in seasonality.get_model_inputs()] if self.seasonality else []),
outputs=[out, out_rnn[1:]]
)
return model, predictor
def build(self, input_shape):
# Typical value from literature
init_alpha, init_beta = 0.05, 0.05
# Smoothing for sales
self.alpha = self.add_weight(shape=(self.n_time_series, 1),
initializer=constant(init_alpha), name='alpha',
constraint=AbsoluteMinMax(0.0, 1.0))
# Smoothing for intervals
self.beta = self.add_weight(shape=(self.n_time_series, 1),
initializer=constant(init_beta), name='beta',
constraint=AbsoluteMinMax(0.0, 1.0))
self.built = True
def build_mlp_clf(input_shape):
x_in = Input(shape=(input_shape, ))
def dense_block(h, units):
h = Dense(units=units,
use_bias=True,
activation=None,
kernel_initializer=he_normal(),
bias_initializer=constant(0.0))(h)
h = BatchNormalization()(h)
h = LeakyReLU(0.2)(h)
h = Dropout(rate=0.5)(h)
return h
h = dense_block(x_in, units=32)
h = dense_block(h, units=16)
h = Dense(units=1,
use_bias=False,
activation='sigmoid',
kernel_initializer='normal',
bias_initializer=constant(0.0))(h)
mlp_clf = Model(inputs=x_in, outputs=h)
mlp_clf.compile(loss='binary_crossentropy',
optimizer=Adam(5e-4),
metrics=['accuracy'])
return mlp_clf
def build(self, input_shape):
self.W = self.add_weight(name='W',
shape=(input_shape[1], input_shape[2]),
initializer=constant(1.0),
trainable=True,
regularizer=reg)
self.b = self.add_weight(name="b",
shape=(1, input_shape[2]),
initializer=constant(0.0),
trainable=True,
regularizer=reg)
# Build the layer from the base class
# This also sets self.built = True
super(FusionLayer, self).build(input_shape)
def build(self, input_shape):
assert len(input_shape) == 5, "The input Tensor should have shape=[None, input_height, input_width," \
" input_num_capsule, input_num_atoms]"
self.input_height = input_shape[1]
self.input_width = input_shape[2]
self.input_num_capsule = input_shape[3]
self.input_num_atoms = input_shape[4]
# Transform matrix
if self.upsamp_type == 'subpix':
self.W = self.add_weight(shape=[self.kernel_size, self.kernel_size,
self.input_num_atoms,
self.num_capsule * self.num_atoms * self.scaling * self.scaling],
initializer=self.kernel_initializer,
name='W')
elif self.upsamp_type == 'resize':
self.W = self.add_weight(shape=[self.kernel_size, self.kernel_size,
self.input_num_atoms, self.num_capsule * self.num_atoms],
initializer=self.kernel_initializer, name='W')
elif self.upsamp_type == 'deconv':
self.W = self.add_weight(shape=[self.kernel_size, self.kernel_size,
self.num_capsule * self.num_atoms, self.input_num_atoms],
initializer=self.kernel_initializer, name='W')
else:
raise NotImplementedError('Upsampling must be one of: "deconv", "resize", or "subpix"')
self.b = self.add_weight(shape=[1, 1, self.num_capsule, self.num_atoms],
initializer=initializers.constant(0.1),
name='b')
self.built = True
def _create_embedding(self):
self._embedding = Embedding(
self.seasonal_period,
output_dim=1,
name="seas_emb_%s" % self.seasonality_type,
embeddings_initializer=constant(0),
embeddings_regularizer=l1_l2(
l2=self.l2_reg) if self.l2_reg else None)
def dense_block(h, units):
h = Dense(units=units,
use_bias=True,
activation=None,
kernel_initializer=he_normal(),
bias_initializer=constant(0.0))(h)
h = BatchNormalization()(h)
h = LeakyReLU(0.2)(h)
h = Dropout(rate=0.5)(h)
return h
def build(self, input_shape):
"""Build the layer."""
super().build(input_shape)
self.q, self.p = self.kernel.shape
self.sigma = self.add_weight('sigma',
shape=[self.p + self.q],
initializer=initializers.constant(
self.initial_sigma),
dtype=self.dtype,
trainable=True)
def build(self, input_shape):
# Initial values copied from statsmodels implementation
init_alpha = 0.5 / max(self.state_size[-1], 1)
# Smoothing
self.alpha = self.add_weight(shape=(input_shape[-1], 1),
initializer=constant(init_alpha), name='alpha',
constraint=AbsoluteMinMax(0.0, 1.0))
if self.trend:
self.beta = self.add_weight(shape=(input_shape[-1], 1),
initializer=constant(0.1 * init_alpha), name='beta',
constraint=AbsoluteMinMax(0.0, 1.0))
# Trend damping
self.phi = self.add_weight(shape=(input_shape[-1], 1), initializer=constant(0.99), name='phi',
constraint=AbsoluteMinMax(0.8, 1.0))
if self.seasonal:
self.gamma = self.add_weight(shape=(input_shape[-1], 1), initializer=constant(0.5),
name='gamma', constraint=AbsoluteMinMax(0.0, 1.0))
self.built = True
def _create_embedding(self):
# First embedding for time series ID
self._embedding = Embedding(
self.n_time_series,
output_dim=self.n_dim,
name="seas_emb_%s" % self.seasonality_type,
embeddings_initializer=constant(0),
embeddings_regularizer=l1_l2(
l2=self.l2_reg) if self.l2_reg else None)
self._seasonality_weights = Dense(self.seasonal_period,
activation='linear')
def create_mdrnn(self, direction):
kernel_initializer = initializers.zeros()
recurrent_initializer = initializers.constant(2)
bias_initializer = initializers.zeros()
return MDRNN(units=1, input_shape=(None, 1),
kernel_initializer=kernel_initializer,
recurrent_initializer=recurrent_initializer,
bias_initializer=bias_initializer,
activation=None,
return_sequences=True,
direction=direction)
def build(self, input_shape):
self.kernel_mu = self.add_weight(
name='kernel_mu',
shape=(input_shape[1], self.units),
initializer=tf.keras.initializers.random_normal(
stddev=self.init_sigma),
trainable=True)
self.bias_mu = self.add_weight(
name='bias_mu',
shape=(self.units, ),
initializer=initializers.random_normal(stddev=self.init_sigma),
trainable=True)
self.kernel_rho = self.add_weight(
name='kernel_rho',
shape=(input_shape[1], self.units),
initializer=initializers.constant(0.0),
trainable=True)
self.bias_rho = self.add_weight(name='bias_rho',
shape=(self.units, ),
initializer=initializers.constant(0.0),
trainable=True)
super().build(input_shape)
def createModel(initWeight=0):
model = ks.Sequential()
#model.add(ks.layers.Dense(units=1, input_shape=[1])
#model.add(ks.layers.Dense(units=1, input_shape=[1],kernel_initializer='zeros')) #initializers
#model.add(ks.layers.Dense(units=1, input_shape=[1], kernel_initializer='ones')) #bias_initializer
model.add(
ks.layers.Dense(units=1,
input_shape=[1],
kernel_initializer=initializers.constant(initWeight)))
model.compile(optimizer='sgd', loss='mean_squared_error')
model.summary()
return model
def build(self, input_shape):
assert len(input_shape) == 5, "The input Tensor should have shape=[None, input_height, input_width," \
" input_num_capsule, input_num_atoms]"
self.input_height = input_shape[1].value
self.input_width = input_shape[2].value
self.input_num_capsule = input_shape[3].value
self.input_num_atoms = input_shape[4].value
'''
# Transform matrix
self.W = self.add_weight(shape=[self.kernel_size, self.kernel_size,
self.input_num_atoms, self.num_capsule * self.num_atoms],
initializer=self.kernel_initializer,
name='W')
'''
ones_kernel = tf.ones([1, 1, 1, 1])
W_shape = [
self.kernel_size, self.kernel_size, self.input_num_atoms,
self.num_capsule * self.num_atoms
]
self.W = tf.layers.conv2d(ones_kernel,
self.kernel_size * self.kernel_size *
self.input_num_atoms * self.num_capsule *
self.num_atoms,
1,
1,
kernel_initializer=self.kernel_initializer,
trainable=self.is_training,
use_bias=False,
reuse=tf.AUTO_REUSE,
name='W' + self.namey)
self.W = tf.reshape(self.W, W_shape)
b_shape = [1, 1, self.num_capsule, self.num_atoms]
ones_kernel = tf.ones([1, 1, 1, 1])
self.b = tf.layers.conv2d(
ones_kernel,
self.num_atoms * self.num_capsule,
1,
1,
kernel_initializer=initializers.constant(0.1),
trainable=self.is_training,
use_bias=False,
reuse=tf.AUTO_REUSE,
name='b' + self.namey)
self.b = tf.reshape(self.b, b_shape)
'''
self.b = self.add_weight(shape=[1, 1, self.num_capsule, self.num_atoms],
initializer=initializers.constant(0.1),
name='b')
'''
self.built = True
def _add_binom_m(num_classes,
tau,
tau_mode,
extra_depth=None,
extra_depth_nonlinearity='relu'):
"""Añade el bloque final de la red que contiene la implementación de la distribución Binomial.
:param end_nonlinearity: La no linealidad que se desea aplicar en la primera capa Dense de un nodo.
:param tau: Valor inicial de tau
:param tau_mode:
non_learnable = No se aprende el valor de tau, se escoge el valor pasado como parámetro.
sigm_learnable = Se aprende como un peso mediante entrenamiento el valor de tau dentro de una función sigmoide (estabiliza el entrenamiento.)
"""
assert tau_mode in ["non_learnable", "sigm_learnable"]
# NOTE: Weird Bug. This is numerically unstable when
# deterministic=True with the article's resnet
# so: Added eps, and added clip
model = tf.keras.Sequential()
k = num_classes
if extra_depth != None:
model.add(Dense(units=extra_depth,
activation=extra_depth_nonlinearity))
model.add(
Dense(units=1,
activation='sigmoid',
kernel_initializer='he_normal',
bias_initializer=constant(0.)))
model.add(
Dense(units=k,
activation='linear',
kernel_initializer='ones',
trainable=False))
c = np.asarray([[(i) for i in range(0, k)]], dtype="float32")
from scipy.special import binom
binom_coef = binom(k - 1, c).astype("float32")
### NOTE: NUMERICALLY UNSTABLE ###
eps = 1e-6
model.add(
Lambda(lambda px: (K.log(binom_coef) + (c * K.log(px + eps)) +
((k - 1 - c) * K.log(1. - px + eps)))))
if tau_mode == "non_learnable":
if tau != 1:
model.add(Lambda(lambda px: px / tau))
else:
model.add(TauLayer(tau, bias=0., nonlinearity='sigmoid'))
model.add(Activation('softmax'))
# return model
return model
def __init__(self,
n_states,
n_actions,
fc1_dims,
fc2_dims,
actor_activation,
directory='tmp/PPO'):
super(ActorNetwork, self).__init__()
self.checkpoint_file = os.path.join(directory, 'Actor_PPO')
if not os.path.exists(directory):
os.makedirs(directory)
self.checkpoint_dir = directory
# Orthogonal Weight initialization + Zero Bias initialization (Implementation Details)
self.fc1 = Dense(fc1_dims,
input_shape=(n_states, ),
activation='tanh',
kernel_initializer=initializers.orthogonal(
np.sqrt(2)),
bias_initializer=initializers.constant(0.0))
self.fc2 = Dense(fc2_dims,
activation='tanh',
kernel_initializer=initializers.orthogonal(
np.sqrt(2)),
bias_initializer=initializers.constant(0.0))
# Last Layer weights ~100 times smaller (Implementation Details)
self.fc3 = Dense(n_actions,
activation=actor_activation,
kernel_initializer=initializers.orthogonal(0.01),
bias_initializer=initializers.constant(0.0))
# log_std weights learnable & independent of states (Implementation Detail)
self.log_std = tf.Variable(tf.zeros((1, n_actions), dtype=tf.float32),
trainable=True)
def build(self, input_shape):
if self.shared:
p_shape = 1
else:
if self.data_format == "channels_last":
p_shape = input_shape[-1]
else:
p_shape = input_shape[1]
self.p = self.add_weight(
name="p",
shape=[p_shape],
initializer=initializers.constant(self._p_init),
trainable=self.trainable,
)
def __init__(self, n_states, fc1_dims, fc2_dims, directory='tmp/PPO'):
super(CriticNetwork, self).__init__()
self.checkpoint_file = os.path.join(directory, 'Critic_PPO')
if not os.path.exists(directory):
os.makedirs(directory)
self.checkpoint_dir = directory
self.fc1 = Dense(fc1_dims,
input_shape=(n_states, ),
activation='tanh',
kernel_initializer=initializers.orthogonal(
np.sqrt(2)),
bias_initializer=initializers.constant(0.0))
self.fc2 = Dense(fc2_dims,
activation='tanh',
kernel_initializer=initializers.orthogonal(
np.sqrt(2)),
bias_initializer=initializers.constant(0.0))
self.fc3 = Dense(1,
kernel_initializer=initializers.orthogonal(1.0),
bias_initializer=initializers.constant(0.0))
def build(self, input_shape):
if self.gated:
self.num_filters = self.num_filters * 2
# size of V is L, Cin, Cout
self.V = self.add_weight(
shape=(self.filter_size, input_shape[-1], self.num_filters),
dtype=tf.float32,
initializer=initializers.RandomNormal(0, 0.01),
trainable=True)
self.g = self.add_weight(shape=(self.num_filters, ),
dtype=tf.float32,
initializer=initializers.constant(1.),
trainable=True)
self.b = self.add_weight(shape=(self.num_filters, ),
dtype=tf.float32,
initializer=None,
trainable=True)
def build_Embedded(self, action_space=6, dueling=True):
self.network_size = 256
X_input = Input(shape=(self.REM_STEP*99) )
# X_input = Input(shape=(self.REM_STEP*7,))
input_reshape=(self.REM_STEP,99)
X = X_input
truncatedn_init = initializers.TruncatedNormal(0, 1e-2)
x_init = "he_uniform"
y_init = initializers.glorot_uniform()
const_init = initializers.constant(1e-2)
X_reshaped = Reshape(input_reshape)(X_input)
X = LayerNormalization(axis=2)(X_reshaped)
X = LayerNormalization(axis=1)(X)
X = Reshape((2,-1))(X)
X = Dense(self.network_size*2, kernel_initializer=y_init, activation ="relu")(X)
X = Dense(256, kernel_initializer=y_init, activation ="relu")(X)
X = TimeDistributed(Dense(self.network_size/4, kernel_initializer=y_init,activation ="relu"))(X)
X = Flatten()(X)
if dueling:
state_value = Dense(
1, activation ="softmax")(X)
state_value = Lambda(lambda s: K.expand_dims(
s[:, 0], -1), output_shape=(action_space,))(state_value)
action_advantage = Dense(
action_space, activation ="linear") (X)
action_advantage = Lambda(lambda a: a[:, :] - K.mean(
a[:, :], keepdims=True), output_shape=(action_space,))(action_advantage)
X = Add()([state_value, action_advantage])
else:
# Output Layer with # of actions: 2 nodes (left, right)
X = Dense(action_space, activation="relu",
kernel_initializer='he_uniform')(X)
model = Model(inputs=X_input, outputs=X, name='build_Embedded')
model.compile(loss=huber_loss, optimizer=Adam(
lr=self.learning_rate), metrics=["accuracy"])
# model.compile(loss="mean_squared_error", optimizer=Adam(lr=0.00025,epsilon=0.01), metrics=["accuracy"])
model.summary()
return model
def build(self, input_shape=None):
channels = input_shape[-1]
if channels is None:
raise ValueError('Channel dimension of the inputs should be defined. Found `None`.')
self.input_spec = layers.InputSpec(ndim=max(2, len(input_shape)), axes={-1: channels})
kernel_initializer = initializers.random_normal(mean=0., stddev=np.sqrt(1. / channels))
self.carry = layers.Dense(
channels,
kernel_initializer=kernel_initializer,
bias_initializer=initializers.constant(-2.),
activation='sigmoid')
self.transform = layers.Dense(
channels,
kernel_initializer=kernel_initializer,
activation='relu')
super().build(input_shape)
def build(self, input_shape):
assert len(input_shape) == 5, "The input Tensor should have shape=[None, input_height, input_width," \
" input_num_capsule, input_num_atoms]"
self.input_height = input_shape[1]
self.input_width = input_shape[2]
self.input_num_capsule = input_shape[3]
self.input_num_atoms = input_shape[4]
# Transform matrix
self.W = self.add_weight(shape=[self.kernel_size, self.kernel_size,
self.input_num_atoms, self.num_capsule * self.num_atoms],
initializer=self.kernel_initializer,
name='W')
self.b = self.add_weight(shape=[1, 1, self.num_capsule, self.num_atoms],
initializer=initializers.constant(0.1),
name='b')
self.built = True
def build(self, input_shapes):
"""
Builds the layer
Args:
input_shapes (list of int): shapes of the layer's inputs (node features and adjacency matrix)
"""
n_nodes = input_shapes[-1]
t_steps = input_shapes[-2]
self.units = t_steps
self.A = self.add_weight(
name="A",
shape=(n_nodes, n_nodes),
trainable=False,
initializer=initializers.constant(self.adj),
)
self.kernel = self.add_weight(
shape=(t_steps, self.units),
initializer=self.kernel_initializer,
name="kernel",
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
)
if self.use_bias:
self.bias = self.add_weight(
shape=(n_nodes, ),
initializer=self.bias_initializer,
name="bias",
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
)
else:
self.bias = None
self.built = True
def build(self, input_shapes):
"""
Builds the layer
Args:
input_shapes (list of int): shapes of the layer's inputs (the batches of node features)
"""
_batch_dim, n_nodes, features = input_shapes
self.A = self.add_weight(
name="A",
shape=(n_nodes, n_nodes),
trainable=False,
initializer=initializers.constant(self.adj),
)
self.kernel = self.add_weight(
shape=(features, self.units),
initializer=self.kernel_initializer,
name="kernel",
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
)
if self.use_bias:
self.bias = self.add_weight(
# ensure the per-node bias can be broadcast across each feature
shape=(n_nodes, 1),
initializer=self.bias_initializer,
name="bias",
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
)
else:
self.bias = None
self.built = True
def get_initializers(activation):
"""Returns the kernel and bias initializers for the given activation
function.
Parameters
----------
activation: str
Activation function for a given layer
Returns
-------
kernel_initializer: tf.keras.initializers
Kernel initializer for the given activation function
bias_initializer: tf.keras.initializers
Bias initializer for the given activation function
"""
if activation == "relu":
kernel_initializer = he_normal()
bias_initializer = constant(value=0.01)
else:
kernel_initializer = glorot_normal()
bias_initializer = zeros()
return kernel_initializer, bias_initializer
def _create_model_predictor(self, lr):
y = self.y.ravel()
# Initial values from statsmodels implementation
if self.seasonal:
l0_start = y[np.arange(y.size) % self.seasonal_period == 0].mean()
lead, lag = y[self.seasonal_period: 2 * self.seasonal_period], y[:self.seasonal_period]
if self.trend == "multiplicative":
b0_start = ((lead - lag) / self.seasonal_period).mean()
elif self.trend == "additive":
b0_start = np.exp((np.log(lead.mean()) - np.log(lag.mean())) / self.seasonal_period)
if self.seasonal == "multiplicative":
s0_start = y[:self.seasonal_period] / l0_start
elif self.seasonal == "additive":
s0_start = y[:self.seasonal_period] - l0_start
elif self.trend:
l0_start = y[0]
if self.trend == "multiplicative":
b0_start = y[1] / l0_start
elif self.trend == "additive":
b0_start = y[1] - l0_start
else:
l0_start = y[0]
inp_y = Input(shape=(None, 1))
inp_emb_id = Input(shape=(1,)) # Dummy ID for embeddings
# (Ab)using embeddings here for initial value variables
init_l0 = [Embedding(1, 1, embeddings_initializer=constant(l0_start), name="l0")(inp_emb_id)[:, 0, :]]
if self.trend:
init_b0 = [Embedding(1, 1, embeddings_initializer=constant(b0_start), name="b0")(inp_emb_id)[:, 0, :]]
else:
init_b0 = []
if self.seasonal:
init_seas_emb = Embedding(1, self.seasonal_period, embeddings_initializer=RandomUniform(0.8, 1.2),
name="s0")
init_seas = [init_seas_emb(inp_emb_id)[:, 0, :]]
else:
init_seas = []
rnncell = ESRNN(self.trend, self.seasonal, self.seasonal_period)
rnn = RNN(rnncell, return_sequences=True, return_state=True)
out_rnn = rnn(inp_y, initial_state=init_l0 + init_b0 + init_seas)
if self.trend == "multiplicative":
l0_b0 = init_l0[0] * init_b0[0]
elif self.trend == "additive":
l0_b0 = init_l0[0] + init_b0[0]
else:
l0_b0 = init_l0[0]
if self.seasonal == "multiplicative":
l0_b0_s0 = l0_b0 * init_seas[0][:, :1]
elif self.seasonal == "additive":
l0_b0_s0 = l0_b0 + init_seas[0][:, :1]
else:
l0_b0_s0 = l0_b0
out = tf.keras.layers.concatenate([
tf.math.reduce_sum(l0_b0_s0, axis=1)[:, None, None],
out_rnn[0]
], axis=1)
model = Model(inputs=[inp_y, inp_emb_id], outputs=out)
model.compile(Adam(lr), "mse")
# predictor also outputs final state for predicting out-of-sample
predictor = Model(inputs=[inp_y, inp_emb_id], outputs=[out, out_rnn[1:]])
# Assign initial seasonality weights
if self.seasonal:
init_seas_emb.set_weights([s0_start.reshape(init_seas_emb.get_weights()[0].shape)])
return model, predictor
def main():
dataset = load_dataset()
train_data = np.asarray(dataset['train']['data'])
train_labels = dataset['train']['label']
num_classes = len(np.unique(train_labels))
test_data = np.asarray(dataset['test']['data'])
test_labels = dataset['test']['label']
train_labels = to_categorical(train_labels, num_classes=num_classes)
test_labels = to_categorical(test_labels, num_classes=num_classes)
generator = dataset['generator']
fs_generator = dataset['fs_generator']
generator_kwargs = {
'batch_size': batch_size
}
print('reps : ', reps)
name = 'mnist_' + fs_network + '_r_' + str(regularization)
print(name)
model_kwargs = {
'nclasses': num_classes,
'regularization': regularization
}
total_features = int(np.prod(train_data.shape[1:]))
fs_filename = directory + fs_network + '_trained_model.h5'
classifier_filename = directory + classifier_network + '_trained_model.h5'
if not os.path.isdir(directory):
os.makedirs(directory)
if not os.path.exists(fs_filename) and warming_up:
np.random.seed(1001)
tf.set_random_seed(1001)
model = getattr(network_models, fs_network)(input_shape=train_data.shape[1:], **model_kwargs)
print('training_model')
model.fit_generator(
generator.flow(train_data, train_labels, **generator_kwargs),
steps_per_epoch=train_data.shape[0] // batch_size, epochs=110,
callbacks=[
callbacks.LearningRateScheduler(scheduler())
],
validation_data=(test_data, test_labels),
validation_steps=test_data.shape[0] // batch_size,
verbose=verbose
)
model.save(fs_filename)
del model
K.clear_session()
for e2efs_class in e2efs_classes:
nfeats = []
accuracies = []
times = []
cont_seed = 0
for factor in [.05, .1, .25, .5]:
n_features = int(total_features * factor)
n_accuracies = []
n_times = []
for r in range(reps):
print('factor : ', factor, ' , rep : ', r)
np.random.seed(cont_seed)
tf.set_random_seed(cont_seed)
cont_seed += 1
mask = (np.std(train_data, axis=0) > 1e-3).astype(int).flatten()
classifier = load_model(fs_filename) if warming_up else getattr(network_models, fs_network)(input_shape=train_data.shape[1:], **model_kwargs)
e2efs_layer = e2efs_class(n_features, input_shape=train_data.shape[1:], kernel_initializer=initializers.constant(mask))
model = e2efs_layer.add_to_model(classifier, input_shape=train_data.shape[1:])
optimizer = custom_optimizers.E2EFS_Adam(e2efs_layer=e2efs_layer, lr=1e-3) # optimizers.adam(lr=1e-2)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc'])
model.fs_layer = e2efs_layer
model.classifier = classifier
model.summary()
start_time = time.time()
model.fit_generator(
fs_generator.flow(train_data, train_labels, **generator_kwargs),
steps_per_epoch=train_data.shape[0] // batch_size, epochs=20000,
callbacks=[
E2EFSCallback(verbose=verbose)
],
validation_data=(test_data, test_labels),
validation_steps=test_data.shape[0] // batch_size,
verbose=verbose
)
fs_rank = np.argsort(K.eval(model.heatmap))[::-1]
mask = np.zeros(train_data.shape[1:])
mask.flat[fs_rank[:n_features]] = 1.
# mask = K.eval(model.fs_kernel).reshape(train_data.shape[1:])
n_times.append(time.time() - start_time)
print('nnz : ', np.count_nonzero(mask))
del model
K.clear_session()
model = load_model(classifier_filename) if warming_up else getattr(network_models, classifier_network)(
input_shape=train_data.shape[1:], **model_kwargs)
optimizer = optimizers.Adam(lr=1e-2) # optimizers.adam(lr=1e-2)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc'])
model.fit_generator(
generator.flow(mask * train_data, train_labels, **generator_kwargs),
steps_per_epoch=train_data.shape[0] // batch_size, epochs=80,
callbacks=[
callbacks.LearningRateScheduler(scheduler()),
],
validation_data=(mask * test_data, test_labels),
validation_steps=test_data.shape[0] // batch_size,
verbose=verbose
)
n_accuracies.append(model.evaluate(mask * test_data, test_labels, verbose=0)[-1])
del model
K.clear_session()
print(
'n_features : ', n_features, ', acc : ', n_accuracies, ', time : ', n_times
)
accuracies.append(n_accuracies)
nfeats.append(n_features)
times.append(n_times)
output_filename = directory + fs_network + '_' + classifier_network + '_' + e2efs_class.__name__ + \
'_results_warming_' + str(warming_up) + '.json'
try:
with open(output_filename) as outfile:
info_data = json.load(outfile)
except:
info_data = {}
if name not in info_data:
info_data[name] = []
info_data[name].append(
{
'regularization': regularization,
'reps': reps,
'classification': {
'n_features': nfeats,
'accuracy': accuracies,
'times': times
}
}
)
with open(output_filename, 'w') as outfile:
json.dump(info_data, outfile)