def img_model():
img_input_shape = (config['seq_len'], config['crop_size'],
config['crop_size'], len(config['channels']))
img = Input(shape=img_input_shape)
m = ZeroPadding3D(padding=(0, (80 - config['crop_size']) // 2,
(80 - config['crop_size']) // 2),
data_format=None)(img)
m = TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2)))(m)
m = self.ConvLSTM_block(
model=m,
filters=32,
kernel_size=(5, 5),
activation=activation,
padding='same',
return_sequences=True,
pool_size=(1, 2, 2),
pool_strides=(1, 2, 2),
drop_prob=0.33,
)
m = self.ConvLSTM_block(
model=m,
filters=64,
kernel_size=(3, 3),
activation=activation,
padding='valid',
return_sequences=True,
pool_size=(1, 3, 3),
pool_strides=(1, 2, 2),
drop_prob=0.33,
)
m = TimeDistributed(Flatten(), name='TD.Flatten')(m)
return img, m
def TD_Dense_block(self, model, drop_prob, units, activation, name):
model = TimeDistributed(Dense(units, activation=activation),
name='TD.Dense' + name)(model)
model = TimeDistributed(BatchNormalization(),
name='TD.BN' + name)(model)
model = TimeDistributed(Dropout(drop_prob),
name='TD.Dropout' + name)(model)
return model
def convblock1(model, filters, kernel, activation, conv_stride,
padding, pool_size, pool_stride, drop_prob):
model = Conv3D(filters,
kernel,
strides=conv_stride,
padding=padding)(model)
model = TimeDistributed(BatchNormalization())(model)
model = Activation(activation)(model)
model = MaxPooling3D(pool_size=pool_size,
strides=pool_stride)(model)
model = TimeDistributed(Dropout(drop_prob))(model)
return model
def create(cls, stations: typing.Dict[typing.AnyStr,
typing.Tuple[float, float, float]],
target_time_offsets: typing.List[datetime.timedelta],
config: typing.Dict[typing.AnyStr, typing.Any]):
self = cls()
initialiser = 'glorot_uniform'
reg_lambda = 0
self.sequence = Sequential()
self.sequence.add(
TimeDistributed(Conv2D(32, (8, 8),
padding='same',
kernel_initializer=initialiser,
kernel_regularizer=l2(reg_lambda)),
input_shape=(config['seq_len'],
config['crop_size'],
config['crop_size'],
len(config['channels']))))
self.sequence.add(TimeDistributed(BatchNormalization()))
self.sequence.add(TimeDistributed(Activation('relu')))
self.sequence.add(TimeDistributed(Dropout(0.5)))
self.sequence.add(
TimeDistributed(
Conv2D(32, (3, 3),
kernel_initializer=initialiser,
kernel_regularizer=l2(reg_lambda))))
self.sequence.add(TimeDistributed(BatchNormalization()))
self.sequence.add(TimeDistributed(Activation('relu')))
self.sequence.add(TimeDistributed(MaxPooling2D((2, 2),
strides=(2, 2))))
# 2nd-5th (default) blocks
self.sequence = self.add_default_block(self.sequence,
32,
init=initialiser,
reg_lambda=reg_lambda)
self.sequence = self.add_default_block(self.sequence,
128,
init=initialiser,
reg_lambda=reg_lambda)
# self.sequence = self.add_default_block(self.sequence, 256, init=initialiser, reg_lambda=reg_lambda)
# self.sequence = self.add_default_block(self.sequence, 512, init=initialiser, reg_lambda=reg_lambda)
# LSTM output head
self.sequence.add(Flatten())
self.sequence.add(Dense(512, activation='tanh'))
self.sequence.add(Dense(512, activation='tanh'))
self.sequence.add(Dropout(0.5))
self.sequence.add(Dense(len(target_time_offsets)))
self.last = Add()
return self
def add_default_block(self, model, kernel_filters, init, reg_lambda):
# conv
model.add(
TimeDistributed(
Conv2D(kernel_filters, (3, 3),
padding='same',
kernel_initializer=init,
kernel_regularizer=l2(reg_lambda))))
model.add(TimeDistributed(BatchNormalization()))
model.add(TimeDistributed(Activation('relu')))
# conv
model.add(
TimeDistributed(
Conv2D(kernel_filters, (3, 3),
padding='same',
kernel_initializer=init,
kernel_regularizer=l2(reg_lambda))))
model.add(TimeDistributed(BatchNormalization()))
model.add(TimeDistributed(Activation('relu')))
# max pool
model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))
return model
def squeeze_excite_residual_block(model, kernel_filters, kernel,
init, reg_lambda):
before_residual = TimeDistributed(
Conv2D(
kernel_filters,
kernel,
padding='same',
kernel_initializer=init,
kernel_regularizer=l2(reg_lambda),
))(model)
before_residual = TimeDistributed(
BatchNormalization())(before_residual)
before_residual = Activation(activation)(before_residual)
residual = TimeDistributed(
Conv2D(
kernel_filters,
kernel,
padding='same',
kernel_initializer=init,
kernel_regularizer=l2(reg_lambda),
))(before_residual)
residual = TimeDistributed(BatchNormalization())(residual)
residual = Activation(activation)(residual)
squeeze_excite = TimeDistributed(
AveragePooling2D(tuple(residual.shape[2:-1])))(residual)
squeeze_excite = TimeDistributed(
BatchNormalization())(squeeze_excite)
squeeze_excite = TimeDistributed(
Dense(kernel_filters,
activation='linear',
kernel_initializer=init,
use_bias=False))(squeeze_excite)
squeeze_excite = TimeDistributed(
BatchNormalization())(squeeze_excite)
squeeze_excite = Activation(activation)(squeeze_excite)
squeeze_excite = TimeDistributed(
Dense(kernel_filters // 8,
activation='linear',
kernel_initializer=init,
use_bias=False))(squeeze_excite)
squeeze_excite = TimeDistributed(
BatchNormalization())(squeeze_excite)
squeeze_excite = Activation(activation)(squeeze_excite)
squeeze_excite = TimeDistributed(
Dense(kernel_filters,
activation='linear',
kernel_initializer=init,
use_bias=False))(squeeze_excite)
squeeze_excite = TimeDistributed(
BatchNormalization())(squeeze_excite)
squeeze_excite = Activation('sigmoid')(squeeze_excite)
after_squeeze_excite = TimeDistributed(
Conv2D(
kernel_filters,
kernel,
padding='same',
kernel_initializer=init,
kernel_regularizer=l2(reg_lambda),
))(residual)
after_squeeze_excite = TimeDistributed(
BatchNormalization())(after_squeeze_excite)
after_squeeze_excite = Activation(activation)(
after_squeeze_excite)
scaled = Multiply()([after_squeeze_excite, squeeze_excite])
model = Add()([before_residual, scaled])
model = TimeDistributed(MaxPooling2D((3, 3),
strides=(3, 3)))(model)
model = TimeDistributed(Dropout(dropping))(model)
return model
def img_model():
def squeeze_excite_residual_block(model, kernel_filters, kernel,
init, reg_lambda):
before_residual = TimeDistributed(
Conv2D(
kernel_filters,
kernel,
padding='same',
kernel_initializer=init,
kernel_regularizer=l2(reg_lambda),
))(model)
before_residual = TimeDistributed(
BatchNormalization())(before_residual)
before_residual = Activation(activation)(before_residual)
residual = TimeDistributed(
Conv2D(
kernel_filters,
kernel,
padding='same',
kernel_initializer=init,
kernel_regularizer=l2(reg_lambda),
))(before_residual)
residual = TimeDistributed(BatchNormalization())(residual)
residual = Activation(activation)(residual)
squeeze_excite = TimeDistributed(
AveragePooling2D(tuple(residual.shape[2:-1])))(residual)
squeeze_excite = TimeDistributed(
BatchNormalization())(squeeze_excite)
squeeze_excite = TimeDistributed(
Dense(kernel_filters,
activation='linear',
kernel_initializer=init,
use_bias=False))(squeeze_excite)
squeeze_excite = TimeDistributed(
BatchNormalization())(squeeze_excite)
squeeze_excite = Activation(activation)(squeeze_excite)
squeeze_excite = TimeDistributed(
Dense(kernel_filters // 8,
activation='linear',
kernel_initializer=init,
use_bias=False))(squeeze_excite)
squeeze_excite = TimeDistributed(
BatchNormalization())(squeeze_excite)
squeeze_excite = Activation(activation)(squeeze_excite)
squeeze_excite = TimeDistributed(
Dense(kernel_filters,
activation='linear',
kernel_initializer=init,
use_bias=False))(squeeze_excite)
squeeze_excite = TimeDistributed(
BatchNormalization())(squeeze_excite)
squeeze_excite = Activation('sigmoid')(squeeze_excite)
after_squeeze_excite = TimeDistributed(
Conv2D(
kernel_filters,
kernel,
padding='same',
kernel_initializer=init,
kernel_regularizer=l2(reg_lambda),
))(residual)
after_squeeze_excite = TimeDistributed(
BatchNormalization())(after_squeeze_excite)
after_squeeze_excite = Activation(activation)(
after_squeeze_excite)
scaled = Multiply()([after_squeeze_excite, squeeze_excite])
model = Add()([before_residual, scaled])
model = TimeDistributed(MaxPooling2D((3, 3),
strides=(3, 3)))(model)
model = TimeDistributed(Dropout(dropping))(model)
return model
img_input_shape = (config['seq_len'], config['crop_size'],
config['crop_size'], len(config['channels']))
img_in = Input(shape=img_input_shape)
m = ZeroPadding3D(padding=(0, (80 - config['crop_size']) // 2,
(80 - config['crop_size']) // 2),
data_format=None)(img_in)
m = TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2)))(m)
m = squeeze_excite_residual_block(m,
64,
kernel=(7, 7),
init=initialiser,
reg_lambda=reg_lambda)
m = squeeze_excite_residual_block(m,
64,
kernel=(5, 5),
init=initialiser,
reg_lambda=reg_lambda)
m = squeeze_excite_residual_block(m,
128,
kernel=(3, 3),
init=initialiser,
reg_lambda=reg_lambda)
m = TimeDistributed(Flatten())(m)
m = TimeDistributed(BatchNormalization())(m)
m = Bidirectional(
LSTM(512, dropout=dropping, return_sequences=True))(m)
m = TimeDistributed(Dense(64, activation='linear'))(m)
m = TimeDistributed(BatchNormalization())(m)
m = Activation(activation)(m)
m = Bidirectional(
LSTM(256, dropout=dropping, return_sequences=False))(m)
m = Dense(64, activation='linear')(m)
m = BatchNormalization()(m)
m = Activation(activation)(m)
m = Dense(len(target_time_offsets), activation='linear')(m)
m = BatchNormalization()(m)
img_out = Activation(activation)(m)
return img_in, img_out
def img_model():
img_input_shape = (config['seq_len'], config['crop_size'],
config['crop_size'], len(config['channels']))
img_in = Input(shape=img_input_shape)
m = ZeroPadding3D(padding=(0, (80 - config['crop_size']) // 2,
(80 - config['crop_size']) // 2),
data_format=None)(img_in)
m = TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2)))(m)
m = convblock2(
model=img_in,
filters=64,
kernel=(5, 5, 5),
activation='relu',
conv_stride=(1, 1, 1),
padding='same',
pool_size=(1, 2, 2),
pool_stride=(1, 1, 1),
drop_prob=0.2,
)
m = convblock2(
model=m,
filters=64,
kernel=(3, 3, 3),
activation='relu',
conv_stride=(1, 1, 1),
padding='same',
pool_size=(1, 2, 2),
pool_stride=(1, 1, 1),
drop_prob=0.2,
)
m = convblock2(
model=m,
filters=128,
kernel=(2, 2, 2),
activation='relu',
conv_stride=(1, 1, 1),
padding='valid',
pool_size=(1, 3, 3),
pool_stride=(1, 2, 2),
drop_prob=0.2,
)
m = convblock2(
model=m,
filters=128,
kernel=(2, 2, 2),
activation='relu',
conv_stride=(1, 1, 1),
padding='valid',
pool_size=(1, 3, 3),
pool_stride=(1, 2, 2),
drop_prob=0.2,
)
m = convblock1(
model=m,
filters=512,
kernel=(1, 3, 3),
activation='relu',
conv_stride=(1, 1, 1),
padding='valid',
pool_size=(2, 3, 3),
pool_stride=(1, 2, 2),
drop_prob=0.2,
)
img_out = Flatten()(m)
return img_in, img_out