def __init__(self,
gpus=0,
batch_size=100,
segment_size=12,
num_features=121,
num_layers=2,
hidden_size=100,
learning_rate=0.0001,
output_dim=1,
create_tensorboard=False):
lstm_cell = CuDNNLSTM if is_gpu_available() else CpuLSTM
with tf.device('/cpu:0'):
self.model = Sequential()
input_shape = (segment_size, num_features)
for i in range(num_layers - 1):
self.model.add(
lstm_cell(hidden_size,
input_shape=input_shape,
return_sequences=True))
self.model.add(lstm_cell(hidden_size, input_shape=input_shape))
self.model.add(Dense(output_dim))
self.model = model_device_adapter.get_device_specific_model(
self.model, gpus)
optimizer = Adam(lr=learning_rate)
self.model.compile(loss='mse', optimizer=optimizer)
print(self.model.summary())
super(LSTM, self).__init__(batch_size=batch_size,
create_tensorboard=create_tensorboard)
def __init__(self, gpus=1, batch_size=50, segment_size=12, output_size=12, window_size=11, encoder_filters=[50], decoder_filters=[50], learning_rate=0.0001, learning_rate_decay=0, create_tensorboard=False): self.segment_size = segment_size self.output_size = output_size self.gpus = gpus # Define an input sequence. # 1 refers to a single channel of the input encoder_inputs = Input(shape=(segment_size, window_size, window_size, 1)) out = encoder_inputs # encoder for filters in encoder_filters[:-1]: out = ConvLSTM2D(filters=filters, kernel_size=3, return_sequences=True, activation='tanh', padding='same')(out) encoder_outputs, state_h, state_c = ConvLSTM2D(filters=encoder_filters[-1], kernel_size=3, activation='tanh', padding='same', return_state=True, return_sequences=True)(out) # decoder self.decoder_input_shape = (window_size, window_size, encoder_filters[-1]) decoder_inputs = Input(shape=(output_size,) + self.decoder_input_shape) # first decoder layer gets the encoder states out = ConvLSTM2D(filters=decoder_filters[0], kernel_size=3, return_sequences=True, activation='tanh', padding='same')([decoder_inputs, state_h, state_c]) for filters in decoder_filters[1:]: out = ConvLSTM2D(filters=filters, kernel_size=3, return_sequences=True, activation='tanh', padding='same')(out) out = TimeDistributed(Flatten())(out) num_output_features = 1 out = TimeDistributed(Dense(100, activation='relu', kernel_regularizer=regularizers.l2(0.002)))(out) out = TimeDistributed(Dense(num_output_features, activation='linear'))(out) self.model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=out) self.model = model_device_adapter.get_device_specific_model(self.model, gpus) optimizer = Adam(lr=learning_rate, decay=learning_rate_decay) self.model.compile(loss='mse', optimizer=optimizer) print(self.model.summary()) super(WindowedConvLSTMSeq2Seq, self).__init__(batch_size=batch_size, create_tensorboard=create_tensorboard)
def __init__(self, gpus=1, batch_size=50, segment_size=12, output_size=12, window_size=15, cnn_filters=[2,3,4], hidden_sizes=[10,10], learning_rate=0.0001, learning_rate_decay=0, create_tensorboard=False): self.segment_size = segment_size self.output_size = output_size self.gpus = gpus # Define an input sequence. # 1 refers to a single channel of the input inputs = Input(shape=(segment_size, window_size, window_size, 1), name="input") # cnns out = TimeDistributed(Conv2D(cnn_filters[0], kernel_size=5, activation='relu', padding='same'), name="cnn_1")(inputs) out = TimeDistributed(MaxPooling2D(), name=f"max_pool")(out) out = TimeDistributed(Conv2D(cnn_filters[1], kernel_size=5, activation='relu', padding='same'), name="cnn_2")(out) out = TimeDistributed(AveragePooling2D(), name=f"avg_pool_1")(out) out = TimeDistributed(Conv2D(cnn_filters[2], kernel_size=5, activation='relu', padding='same'), name="cnn_3")(out) out = TimeDistributed(AveragePooling2D(), name=f"avg_pool_2")(out) out = TimeDistributed(Flatten(), name="flatten_before_lstm")(out) cells = [LSTMCell(hidden_sizes[0]), LSTMCell(hidden_sizes[1])] out = RNN(cells)(out) # out = Flatten(name="flatten_after_lstm")(out) out = Dense(100, activation='relu', name=f"mlp_relu")(out) out = Dense(output_size, activation='linear', name=f"mlp_linear")(out) self.model = Model(inputs=inputs, outputs=out) self.model = model_device_adapter.get_device_specific_model(self.model, gpus) optimizer = Adam(lr=learning_rate, decay=learning_rate_decay) self.model.compile(loss='mse', optimizer=optimizer) print(self.model.summary()) super(CnnLSTM, self).__init__(batch_size=batch_size, create_tensorboard=create_tensorboard)
def __init__(self, gpus=1, batch_size=50, segment_size=12, window_size=15, learning_rate=0.0001, create_tensorboard=False): self.segment_size = segment_size # Define an input sequence. # 1 refers to a single channel of the input inputs = Input(shape=(segment_size, window_size, window_size, 1)) out = TimeDistributed(Conv2D(25, kernel_size=3, activation='tanh', padding='same'))(inputs) out = TimeDistributed(AveragePooling2D())(out) out = TimeDistributed(Conv2D(50, kernel_size=3, activation='tanh', padding='same'))(out) out = TimeDistributed(AveragePooling2D())(out) out = TimeDistributed(Conv2D(50, kernel_size=3, activation='tanh', padding='same'))(out) out = ConvLSTM2D(filters=50, kernel_size=3, return_sequences=True, activation='tanh', padding='same')(out) out = ConvLSTM2D(filters=50, kernel_size=3, return_sequences=True, activation='tanh', padding='same')(out) out = ConvLSTM2D(filters=25, kernel_size=3, activation='tanh', padding='same')(out) out = Flatten()(out) out = Dense(1)(out) # out = Conv2DTranspose(50, kernel_size=3, activation='tanh', padding='same')(out) # out = UpSampling2D()(out) # out = Conv2DTranspose(50, kernel_size=3, activation='tanh', padding='same')(out) # out = UpSampling2D()(out) # out = Conv2DTranspose(25, kernel_size=3, activation='tanh', padding='same')(out) # out = UpSampling2D()(out) # out = Conv2DTranspose(10, kernel_size=3, activation='tanh')(out) # out = Conv2DTranspose(1, kernel_size=3, activation='tanh')(out) self.model = Model(inputs=inputs, outputs=out) self.model = model_device_adapter.get_device_specific_model(self.model, gpus) optimizer = Adam(lr=learning_rate) self.model.compile(loss='mse', optimizer=optimizer) print(self.model.summary()) super(WindowedCnnConvLSTM, self).__init__(batch_size=batch_size, create_tensorboard=create_tensorboard)
def __init__(self,
gpus=1,
batch_size=50,
segment_size=12,
output_size=12,
grid_size=100,
encoder_filters=[50],
decoder_filters=[50, 1],
dropout=0,
kernel_size=3,
learning_rate=0.0001,
learning_rate_decay=0,
create_tensorboard=False):
print(f"!!!kernel_size: {kernel_size}")
self.segment_size = segment_size
self.output_size = output_size
self.gpus = gpus
# Define an input sequence.
# 1 refers to a single channel of the input
encoder_inputs = Input(shape=(segment_size, grid_size, grid_size, 1))
out = encoder_inputs
# encoder
for i, filters in enumerate(encoder_filters[:-1]):
out = Dropout(dropout)(out)
out = ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
return_sequences=True,
activation='tanh',
padding='same',
name=f"encoder_{i+1}")(out)
out = Dropout(dropout)(out)
encoder_outputs, state_h, state_c = ConvLSTM2D(
filters=encoder_filters[-1],
kernel_size=kernel_size,
activation='tanh',
padding='same',
return_state=True,
return_sequences=True,
name=f"encoder_{len(encoder_filters)}")(out)
# decoder
self.decoder_input_shape = (grid_size, grid_size, encoder_filters[-1])
decoder_inputs = Input(shape=(output_size, ) +
self.decoder_input_shape,
name="decoder_input")
# first decoder layer gets the encoder states
out = ConvLSTM2D(filters=decoder_filters[0],
kernel_size=kernel_size,
return_sequences=True,
activation='tanh',
padding='same',
name="decoder_1")([decoder_inputs, state_h, state_c])
for i, filters in enumerate(decoder_filters[1:]):
out = Dropout(dropout)(out)
out = ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
return_sequences=True,
activation='tanh',
padding='same',
name=f"decoder_{i+2}")(out)
# cnn forming the final outputs, so that predictions can be outside the range of tanh activation
out = TimeDistributed(Conv2D(1,
kernel_size=1,
activation='linear',
padding='same'),
name=f"cnn_final")(out)
self.model = Model(inputs=[encoder_inputs, decoder_inputs],
outputs=out)
self.model = model_device_adapter.get_device_specific_model(
self.model, gpus)
optimizer = Adam(lr=learning_rate, decay=learning_rate_decay)
self.model.compile(loss='mse', optimizer=optimizer)
print(self.model.summary())
super(ConvLSTMSeq2Seq,
self).__init__(batch_size=batch_size,
create_tensorboard=create_tensorboard)
def __init__(self,
gpus=1,
batch_size=100,
segment_size=12,
num_features=121,
num_layers=2,
hidden_size=10,
learning_rate=0.0001,
dropout=0,
output_size=12,
create_tensorboard=False):
self.segment_size = segment_size
self.output_size = output_size
# create encoder and decoder LSTM towers/stacks
encoder = self.create_stacked_lstms(hidden_size=hidden_size,
num_layers=num_layers,
return_sequences=False,
return_state=True)
decoder = self.create_stacked_lstms(hidden_size=hidden_size,
num_layers=num_layers,
return_sequences=True,
return_state=True)
# Define an input sequence.
encoder_inputs = Input(shape=(None, num_features))
encoder_outputs_and_states = encoder(encoder_inputs)
# Discard encoder outputs and only keep the states.
# The outputs are of no interest to us, the encoder's
# job is to create a state describing the input sequence.
# encoder_states = [state_h, state_c]
encoder_states = encoder_outputs_and_states[1:]
# The decoder input will be set to zero as decoder only relies on the encoder state
decoder_inputs = Input(shape=(None, 1))
# Set the initial state of the decoder to be the ouput state of the encoder.
# This is the fundamental part of the encoder-decoder.
decoder_outputs_and_states = decoder(decoder_inputs,
initial_state=encoder_states)
# Only select the output of the decoder (not the states)
decoder_outputs = decoder_outputs_and_states[0]
# TODO: try with and without this dense layer
# Apply a dense layer with linear activation to set output to correct dimension
# and scale
num_output_features = 1
decoder_dense = Dense(num_output_features,
activation='linear') # TODO: try regularizers
decoder_outputs = decoder_dense(decoder_outputs)
self.model = Model(inputs=[encoder_inputs, decoder_inputs],
outputs=decoder_outputs)
self.model = model_device_adapter.get_device_specific_model(
self.model, gpus)
optimizer = Adam(lr=learning_rate)
self.model.compile(loss='mse', optimizer=optimizer)
print(self.model.summary())
super(KerasSeq2Seq,
self).__init__(batch_size=batch_size,
create_tensorboard=create_tensorboard)
def __init__(self,
gpus=1,
batch_size=50,
segment_size=12,
output_size=12,
window_size=11,
cnn_filters=[2, 3, 4],
encoder_filters=[5, 6],
decoder_filters=[6, 7],
mlp_hidden_sizes=[50, 1],
decoder_padding='same',
learning_rate=0.0001,
learning_rate_decay=0,
create_tensorboard=False):
self.segment_size = segment_size
self.output_size = output_size
self.gpus = gpus
# Define an input sequence.
# 1 refers to a single channel of the input
encoder_inputs = Input(shape=(segment_size, window_size, window_size,
1),
name="encoder_input")
# cnns
out = TimeDistributed(Conv2D(cnn_filters[0],
kernel_size=3,
activation='tanh',
padding='same'),
name="cnn_1")(encoder_inputs)
for i, filters in enumerate(cnn_filters[1:]):
out = TimeDistributed(AveragePooling2D(),
name=f"avg_pool_{i + 1}")(out)
out = TimeDistributed(Conv2D(filters,
kernel_size=3,
activation='tanh',
padding='same'),
name=f"cnn_{i+2}")(out)
# encoder
for i, filters in enumerate(encoder_filters[:-1]):
out = ConvLSTM2D(filters=filters,
kernel_size=3,
return_sequences=True,
activation='tanh',
padding='same',
name=f"encoder_convlstm_{i+1}")(out)
encoder_outputs, state_h, state_c = ConvLSTM2D(
filters=encoder_filters[-1],
kernel_size=3,
activation='tanh',
padding='same',
return_state=True,
return_sequences=True,
name=f"encoder_convlstm_{len(encoder_filters)}")(out)
# decoder
latent_dim = window_size // 2**(len(cnn_filters) - 1)
self.decoder_input_shape = (latent_dim, latent_dim,
encoder_filters[-1])
decoder_inputs = Input(shape=(output_size, ) +
self.decoder_input_shape,
name="decoder_input")
out = ConvLSTM2D(
filters=decoder_filters[0],
kernel_size=3,
return_sequences=True,
activation='tanh',
padding='same',
name="decoder_convlstm_1")([decoder_inputs, state_h, state_c])
for i, filters in enumerate(decoder_filters[1:]):
out = ConvLSTM2D(filters=filters,
kernel_size=3,
return_sequences=True,
activation='tanh',
padding=decoder_padding,
name=f"decoder_convlstm_{i+2}")(out)
# predictor mlp
out = TimeDistributed(Flatten(), name="flatten")(out)
for i, hidden_size in enumerate(mlp_hidden_sizes[:-1]):
out = TimeDistributed(Dense(
hidden_size,
activation='relu',
kernel_regularizer=regularizers.l2(0.002)),
name=f"mlp_{i}")(out)
out = TimeDistributed(Dense(mlp_hidden_sizes[-1],
activation='linear',
kernel_regularizer=regularizers.l2(0.002)),
name="mlp_final")(out)
self.model = Model(inputs=[encoder_inputs, decoder_inputs],
outputs=out)
self.model = model_device_adapter.get_device_specific_model(
self.model, gpus)
optimizer = Adam(lr=learning_rate, decay=learning_rate_decay)
self.model.compile(loss='mse', optimizer=optimizer)
print(self.model.summary())
super(CnnConvLSTMSeq2Seq,
self).__init__(batch_size=batch_size,
create_tensorboard=create_tensorboard)
def __init__(self,
gpus=1,
batch_size=50,
segment_size=12,
output_size=12,
window_size=11,
cnn_filters=[25, 50, 50],
encoder_filters=[50],
decoder_filters=[50],
pass_state=True,
learning_rate=0.0001,
learning_rate_decay=0,
create_tensorboard=False):
self.segment_size = segment_size
self.output_size = output_size
self.gpus = gpus
# Define an input sequence.
# 1 refers to a single channel of the input
encoder_inputs = Input(shape=(segment_size, window_size, window_size,
1))
print(f"encoder_inputs: {encoder_inputs}")
out = TimeDistributed(
Conv2D(cnn_filters[0],
kernel_size=3,
activation='tanh',
padding='same'))(encoder_inputs)
out = TimeDistributed(AveragePooling2D())(out)
out = TimeDistributed(
Conv2D(cnn_filters[1],
kernel_size=3,
activation='tanh',
padding='same'))(out)
out = TimeDistributed(AveragePooling2D())(out)
out = TimeDistributed(
Conv2D(cnn_filters[2],
kernel_size=3,
activation='tanh',
padding='same'))(out)
# encoder
for filters in encoder_filters[:-1]:
out = ConvLSTM2D(filters=filters,
kernel_size=3,
return_sequences=True,
activation='tanh',
padding='same')(out)
encoder_outputs, state_h, state_c = ConvLSTM2D(
filters=encoder_filters[-1],
kernel_size=3,
activation='tanh',
padding='same',
return_state=True,
return_sequences=True)(out)
# encoder_outputs shape: (batch_size, segment_size, window_size, window_size, num_filters)
# decoder
latent_dim = window_size // 2**(len(cnn_filters) - 1)
self.decoder_input_shape = (latent_dim, latent_dim,
encoder_filters[-1])
decoder_inputs = Input(shape=(output_size, ) +
self.decoder_input_shape,
name="decoder_input")
attention_layer = ConvRNN2D(ConvLSTMAttentionCell(decoder_filters[0],
kernel_size=3,
padding='same'),
return_sequences=True)
attention_layer._num_constants = 1
# the second encoder_outputs is given as 'constants' to the layer,
# so that it can be fully transferred to the cell
if pass_state:
out = attention_layer(
[decoder_inputs, state_h, state_c, encoder_outputs])
else:
out = attention_layer([decoder_inputs, encoder_outputs])
for filters in decoder_filters[1:]:
out = ConvLSTM2D(filters=filters,
kernel_size=3,
return_sequences=True,
activation='tanh',
padding='same')(out)
out = TimeDistributed(Flatten())(out)
num_output_features = 1
out = TimeDistributed(
Dense(100,
activation='relu',
kernel_regularizer=regularizers.l2(0.002)))(out)
out = TimeDistributed(Dense(num_output_features,
activation='linear'))(out)
self.model = Model(inputs=[encoder_inputs, decoder_inputs],
outputs=out)
self.model = model_device_adapter.get_device_specific_model(
self.model, gpus)
optimizer = Adam(lr=learning_rate, decay=learning_rate_decay)
self.model.compile(loss='mse', optimizer=optimizer)
print(self.model.summary())
super(CnnConvLSTMAttention,
self).__init__(batch_size=batch_size,
create_tensorboard=create_tensorboard)