def get_gnet(n_ch, patch_height, patch_width):
inputs = Input((n_ch, patch_height, patch_width))
conv1 = Convolution2D(32, 3, 3, activation='relu',
border_mode='same')(inputs)
conv1 = Dropout(0.2)(conv1)
conv1 = Convolution2D(32, 3, 3, activation='relu',
border_mode='same')(conv1)
up1 = UpSampling2D(size=(2, 2))(conv1)
#
conv2 = Convolution2D(16, 3, 3, activation='relu', border_mode='same')(up1)
conv2 = Dropout(0.2)(conv2)
conv2 = Convolution2D(16, 3, 3, activation='relu',
border_mode='same')(conv2)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv2)
#
conv3 = Convolution2D(32, 3, 3, activation='relu',
border_mode='same')(pool1)
conv3 = Dropout(0.2)(conv3)
conv3 = Convolution2D(32, 3, 3, activation='relu',
border_mode='same')(conv3)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv3)
#
conv4 = Convolution2D(64, 3, 3, activation='relu',
border_mode='same')(pool2)
conv4 = Dropout(0.2)(conv4)
conv4 = Convolution2D(64, 3, 3, activation='relu',
border_mode='same')(conv4)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv4)
#
conv5 = Convolution2D(128, 3, 3, activation='relu',
border_mode='same')(pool3)
conv5 = Dropout(0.2)(conv5)
conv5 = Convolution2D(128, 3, 3, activation='relu',
border_mode='same')(conv5)
#
up2 = merge([UpSampling2D(size=(2, 2))(conv5), conv4],
mode='concat',
concat_axis=1)
conv6 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(up2)
conv6 = Dropout(0.2)(conv6)
conv6 = Convolution2D(64, 3, 3, activation='relu',
border_mode='same')(conv6)
#
up3 = merge([UpSampling2D(size=(2, 2))(conv6), conv3],
mode='concat',
concat_axis=1)
conv7 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(up3)
conv7 = Dropout(0.2)(conv7)
conv7 = Convolution2D(32, 3, 3, activation='relu',
border_mode='same')(conv7)
#
up4 = merge([UpSampling2D(size=(2, 2))(conv7), conv2],
mode='concat',
concat_axis=1)
conv8 = Convolution2D(16, 3, 3, activation='relu', border_mode='same')(up4)
conv8 = Dropout(0.2)(conv8)
conv8 = Convolution2D(16, 3, 3, activation='relu',
border_mode='same')(conv8)
#
pool4 = MaxPooling2D(pool_size=(2, 2))(conv8)
conv9 = Convolution2D(32, 3, 3, activation='relu',
border_mode='same')(pool4)
conv9 = Dropout(0.2)(conv9)
conv9 = Convolution2D(32, 3, 3, activation='relu',
border_mode='same')(conv9)
#
conv10 = Convolution2D(2, 1, 1, activation='relu',
border_mode='same')(conv9)
conv10 = Reshape((2, patch_height * patch_width))(conv10)
conv10 = Permute((2, 1))(conv10)
############
conv10 = Activation('softmax')(conv10)
model = Model(input=inputs, output=conv10)
# sgd = SGD(lr=0.01, decay=1e-6, momentum=0.3, nesterov=False)
model.compile(optimizer='sgd',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def HRNetResidual(input_shape=(128, 128, 3), num_keypoints=20):
# stem
inputs = Input(shape=input_shape, name='image')
x1 = stem(inputs, 64)
x1 = Conv2D(64 * 4, 1, padding='same', use_bias=False)(x1)
x1 = BatchNormalization()(x1)
for block in range(4):
x1 = bottleneck(x1)
# stage I
x1 = Conv2D(32, 3, padding='same', use_bias=False)(x1)
x1 = BatchNormalization()(x1)
x1 = Activation('relu')(x1)
x2 = transition_block(x1, 2)
# stage II
for block in range(4):
x1 = residual_block(x1, 32)
x2 = residual_block(x2, 64)
x1, x2 = fuse([x1, x2])
x3 = transition_block(x2, 2)
# stage III
for module in range(4):
for block in range(4):
x1 = residual_block(x1, 32)
x2 = residual_block(x2, 64)
x3 = residual_block(x3, 128)
x1, x2, x3 = fuse([x1, x2, x3])
x4 = transition_block(x3, 2)
# stage IV
for module in range(3):
for block in range(4):
x1 = residual_block(x1, 32)
x2 = residual_block(x2, 64)
x3 = residual_block(x3, 128)
x4 = residual_block(x4, 256)
x1, x2, x3, x4 = fuse([x1, x2, x3, x4])
# head
x2 = UpSampling2D(size=(2, 2))(x2)
x3 = UpSampling2D(size=(4, 4))(x3)
x4 = UpSampling2D(size=(8, 8))(x4)
x = Concatenate()([x1, x2, x3, x4])
x = Conv2D(480, 1)(x)
x = BatchNormalization(epsilon=1.001e-5)(x)
x = Activation('relu')(x)
x = Conv2D(num_keypoints, 1)(x)
# extra
x = BatchNormalization(epsilon=1.001e-5)(x)
x = Activation('relu')(x)
x = UpSampling2D(size=(4, 4), interpolation='bilinear')(x)
x = Permute([3, 1, 2])(x)
x = Reshape([num_keypoints, input_shape[0] * input_shape[1]])(x)
x = Activation('softmax')(x)
x = Reshape([num_keypoints, input_shape[0], input_shape[1]])(x)
outputs = ExpectedValue2D(name='keypoints')(x)
model = Model(inputs, outputs, name='hrnet-residual')
return model
def forward(self, x):
x = Permute((2, 1, 3))(x)
x = self.pad(x)
return Permute((2, 1, 3))(x)
def __init__(self, sr=44100):
"""
Build the CNN model and load the weights
Parameters
----------
model_capacity : 'tiny', 'small', 'medium', 'large', or 'full'
String specifying the model capacity, which determines the model's
capacity multiplier to 4 (tiny), 8 (small), 16 (medium), 24 (large),
or 32 (full). 'full' uses the model size specified in the paper,
and the others use a reduced number of filters in each convolutional
layer, resulting in a smaller model that is faster to evaluate at the
cost of slightly reduced pitch estimation accuracy.
Returns
-------
model : tensorflow.keras.models.Model
The pre-trained keras model loaded in memory
"""
from tensorflow.keras.layers import Input, Reshape, Conv2D, BatchNormalization
from tensorflow.keras.layers import MaxPool2D, Dropout, Permute, Flatten, Dense
from tensorflow.keras.models import Model
models = {
'tiny': None,
'small': None,
'medium': None,
'large': None,
'full': None
}
# the model is trained on 16kHz audio
#model_srate = 16000
self.sr = sr
#if models[model_capacity] is None:
capacity_multiplier = 32 #{
# 'tiny': 4, 'small': 8, 'medium': 16, 'large': 24, 'full': 32
# }[model_capacity]
layers = [1, 2, 3, 4, 5, 6]
filters = [n * capacity_multiplier for n in [32, 4, 4, 4, 8, 16]]
widths = [512, 64, 64, 64, 64, 64]
strides = [(4, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)]
x = Input(shape=(1024,), name='input', dtype='float32')
y = Reshape(target_shape=(1024, 1, 1), name='input-reshape')(x)
for l, f, w, s in zip(layers, filters, widths, strides):
y = Conv2D(f, (w, 1), strides=s, padding='same',
activation='relu', name="conv%d" % l)(y)
y = BatchNormalization(name="conv%d-BN" % l)(y)
y = MaxPool2D(pool_size=(2, 1), strides=None, padding='valid',
name="conv%d-maxpool" % l)(y)
y = Dropout(0.25, name="conv%d-dropout" % l)(y)
y = Permute((2, 1, 3), name="transpose")(y)
y = Flatten(name="flatten")(y)
y = Dense(360, activation='sigmoid', name="classifier")(y)
self.model = Model(inputs=x, outputs=y)
filename = "model-full.h5"
self.model.load_weights(filename)
self.model.compile('adam', 'binary_crossentropy')
def build_model(self):
"""Helper method for creating the model"""
vocab = set()
for story, q, answer in self.train_stories + self.test_stories:
vocab |= set(story + q + [answer])
vocab = sorted(vocab)
# Reserve 0 for masking via pad_sequences
vocab_size = len(vocab) + 1
story_maxlen = max(
len(x) for x, _, _ in self.train_stories + self.test_stories)
query_maxlen = max(
len(x) for _, x, _ in self.train_stories + self.test_stories)
word_idx = {c: i + 1 for i, c in enumerate(vocab)}
self.inputs_train, self.queries_train, self.answers_train = vectorize_stories(
word_idx, story_maxlen, query_maxlen, self.train_stories)
self.inputs_test, self.queries_test, self.answers_test = vectorize_stories(
word_idx, story_maxlen, query_maxlen, self.test_stories)
# placeholders
input_sequence = Input((story_maxlen, ))
question = Input((query_maxlen, ))
# encoders
# embed the input sequence into a sequence of vectors
input_encoder_m = Sequential()
input_encoder_m.add(Embedding(input_dim=vocab_size, output_dim=64))
input_encoder_m.add(Dropout(self.config.get("dropout", 0.3)))
# output: (samples, story_maxlen, embedding_dim)
# embed the input into a sequence of vectors of size query_maxlen
input_encoder_c = Sequential()
input_encoder_c.add(
Embedding(input_dim=vocab_size, output_dim=query_maxlen))
input_encoder_c.add(Dropout(self.config.get("dropout", 0.3)))
# output: (samples, story_maxlen, query_maxlen)
# embed the question into a sequence of vectors
question_encoder = Sequential()
question_encoder.add(
Embedding(input_dim=vocab_size,
output_dim=64,
input_length=query_maxlen))
question_encoder.add(Dropout(self.config.get("dropout", 0.3)))
# output: (samples, query_maxlen, embedding_dim)
# encode input sequence and questions (which are indices)
# to sequences of dense vectors
input_encoded_m = input_encoder_m(input_sequence)
input_encoded_c = input_encoder_c(input_sequence)
question_encoded = question_encoder(question)
# compute a "match" between the first input vector sequence
# and the question vector sequence
# shape: `(samples, story_maxlen, query_maxlen)`
match = dot([input_encoded_m, question_encoded], axes=(2, 2))
match = Activation("softmax")(match)
# add the match matrix with the second input vector sequence
response = add([match, input_encoded_c
]) # (samples, story_maxlen, query_maxlen)
response = Permute(
(2, 1))(response) # (samples, query_maxlen, story_maxlen)
# concatenate the match matrix with the question vector sequence
answer = concatenate([response, question_encoded])
# the original paper uses a matrix multiplication.
# we choose to use a RNN instead.
answer = LSTM(32)(answer) # (samples, 32)
# one regularization layer -- more would probably be needed.
answer = Dropout(self.config.get("dropout", 0.3))(answer)
answer = Dense(vocab_size)(answer) # (samples, vocab_size)
# we output a probability distribution over the vocabulary
answer = Activation("softmax")(answer)
# build the final model
model = Model([input_sequence, question], answer)
return model
def STSnet(n_clss, ch_rows=4, ch_cols=5, samples=128, kernLength=None, F1=4,
D=2, F2=8, norm_rate=0.25, dropoutRate=0.25, dropoutType='Dropout'):
"""Keras Implementation of STSnet.
# Arguments:
n_clss: int, number of classes to clasify (default 4)
ch_rows: number of rows of 2D spatial distribution (default 4)
ch_cols: number of columns of 2D spatial distribution (default 5)
samples: number of epoch samples form each trial (default 128)
dropoutRate: dropout value (default 0.25)
kernLength: kernel length value in Conv2D layer(default None)
F1: number of filters in Conv2D layer (default 4)
D: deep multiplier to DepthwiseConv2D (default 2)
F2: number of filters to each kernel from SeparableConv2D (default 8)
norm_rate: maximum norm for the incoming dense weights(default 0.25)
dropoutType: 'SpatialDropout2D' or 'Dropout' (defautl 'Dropout')
"""
if dropoutType == 'SpatialDropout2D':
dropoutType = SpatialDropout2D
elif dropoutType == 'Dropout':
dropoutType = Dropout
else:
raise ValueError('dropoutType must be one of SpatialDropout2D '
'or Dropout, passed as a string.')
if kernLength is None:
kernLength = np.int(samples/2)
print('kernel_length', kernLength)
# -------------------------------------------------------------------------
# DEPTWISE 2D MODEL
DW_in = Input(shape=(F1, ch_rows, ch_cols))
DW_out = DepthwiseConv2D((ch_rows, ch_cols), use_bias=False,
depth_multiplier=D,
depthwise_constraint=max_norm(1.))(DW_in)
DW = Model(inputs=DW_in, outputs=DW_out)
# -------------------------------------------------------------------------
input1 = Input(shape=(ch_rows * ch_cols, samples))
# -------------------------------------------------------------------------
reshape1 = Reshape(target_shape=(1, ch_rows * ch_cols, samples))(input1)
block1 = Conv2D(F1, (1, kernLength), padding='same',
input_shape=(1, ch_rows*ch_cols, samples),
use_bias=False)(reshape1)
block1 = BatchNormalization(axis=1)(block1)
reshape2 = Reshape(target_shape=(F1, ch_rows, ch_cols, samples))(block1)
perm_dims1 = 4, 1, 2, 3
perm1 = Permute(perm_dims1)(reshape2)
block1 = TimeDistributed(DW)(perm1)
perm_dims2 = 2, 3, 4, 1
perm2 = Permute(perm_dims2)(block1)
reshape3 = Reshape(target_shape=(F1*D, 1, samples))(perm2)
block1 = BatchNormalization(axis=1)(reshape3)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D((1, np.int(kernLength/16)))(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = SeparableConv2D(F2, (1, 16),
use_bias=False, padding='same')(block1)
block2 = BatchNormalization(axis=1)(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((1, np.int(kernLength/8)))(block2)
block2 = dropoutType(dropoutRate)(block2)
flatten = Flatten(name='flatten')(block2)
dense = Dense(n_clss, name='dense',
kernel_constraint=max_norm(norm_rate))(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input1, outputs=softmax)
def _crnn_layers(self, input_shape, output_shape):
channel_axis = 3
melgram_input = Input(shape=input_shape, dtype="float32")
# Input block
padding = self.network_input_width - input_shape[1]
left_pad = int(padding / 2)
if padding % 2:
right_pad = left_pad + 1
else:
right_pad = left_pad
input_padding = ((0, 0), (left_pad, right_pad))
hidden = ZeroPadding2D(padding=input_padding)(melgram_input)
# Conv block 1
hidden = Conv2D(64, (3, 3), padding=self.padding, name='conv1')(hidden)
hidden = BatchNormalization(axis=channel_axis, name='bn1')(hidden)
hidden = ELU()(hidden)
hidden = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
name='pool1')(hidden)
hidden = Dropout(0.1, name='dropout1')(hidden)
# Conv block 2
hidden = Conv2D(128, (3, 3), padding=self.padding,
name='conv2')(hidden)
hidden = BatchNormalization(axis=channel_axis, name='bn2')(hidden)
hidden = ELU()(hidden)
hidden = MaxPooling2D(pool_size=(3, 3), strides=(3, 3),
name='pool2')(hidden)
hidden = Dropout(0.1, name='dropout2')(hidden)
# Conv block 3
hidden = Conv2D(128, (3, 3), padding=self.padding,
name='conv3')(hidden)
hidden = BatchNormalization(axis=channel_axis, name='bn3')(hidden)
hidden = ELU()(hidden)
hidden = MaxPooling2D(pool_size=(4, 4), strides=(4, 4),
name='pool3')(hidden)
hidden = Dropout(0.1, name='dropout3')(hidden)
# Conv block 4
hidden = Conv2D(128, (3, 3), padding=self.padding,
name='conv4')(hidden)
hidden = BatchNormalization(axis=channel_axis, name='bn4')(hidden)
hidden = ELU()(hidden)
hidden = MaxPooling2D(pool_size=(4, 4), strides=(4, 4),
name='pool4')(hidden)
hidden = Dropout(0.1, name='dropout4')(hidden)
# reshaping
hidden = Reshape((15, 128))(hidden)
# GRU block 1, 2, output
embed_size = 32
hidden = GRU(embed_size, return_sequences=True, name='gru1')(hidden)
hidden = GRU(embed_size, return_sequences=self.attention,
name='gru2')(hidden)
if self.attention:
attention = Dense(1)(hidden)
attention = Flatten()(attention)
attention_act = Activation("softmax")(attention)
attention = RepeatVector(embed_size)(attention_act)
attention = Permute((2, 1))(attention)
merged = Multiply()([hidden, attention])
hidden = Lambda(lambda xin: K.sum(xin, axis=1))(merged)
if self.output_dropout:
hidden = Dropout(self.output_dropout)(hidden)
output = Dense(output_shape, activation='sigmoid',
name='crnn_output')(hidden)
return melgram_input, output
def bottleneck_encoder(self, tensor, nfilters, downsampling=False, dilated=False, asymmetric=False, normal=False,
drate=0.1,
name=''):
"""
Encoder
:param tensor: input tensor
:param nfilters: Number of filters
:param downsampling: Downsample the feature map
:param dilated: determines if ther should be dilated convultion
:param asymmetric: Determines if there should be asymmetric convolution
:param normal: enables 3x3 convolution on feature map
:param drate: rate of dilation
:param name: the name for the weight variable.
:return: encoder output
"""
y = tensor
skip = tensor
stride = 1
ksize = 1
# Filters operating on downsampled images have a bigger receptive field and hence gathers more context.
if downsampling:
stride = 2
ksize = 2
skip = MaxPooling2D(pool_size=(2, 2), name=f'max_pool_{name}')(skip)
skip = Permute((1, 3, 2), name=f'permute_1_{name}')(skip) # (B, H, W, C) -> (B, H, C, W)
ch_pad = nfilters - tf.compat.v2.keras.backend.int_shape(tensor)[-1]
skip = ZeroPadding2D(padding=((0, 0), (0, ch_pad)), name=f'zeropadding_{name}')(skip)
skip = Permute((1, 3, 2), name=f'permute_2_{name}')(skip) # (B, H, C, W) -> (B, H, W, C)
y = Conv2D(filters=nfilters // 4, kernel_size=(ksize, ksize), kernel_initializer='he_normal',
strides=(stride, stride), padding='same', use_bias=False, name=f'1x1_conv_{name}')(y)
y = BatchNormalization(momentum=0.1, name=f'bn_1x1_{name}')(y)
y = PReLU(shared_axes=[1, 2], name=f'prelu_1x1_{name}')(y)
if normal:
# deconv with 3x3 filter
y = Conv2D(filters=nfilters // 4, kernel_size=(3, 3), kernel_initializer='he_normal', padding='same',
name=f'3x3_conv_{name}')(y)
elif asymmetric:
# decompose 5x5 convolution to two asymmetric layers as 5x1 and 1x5
y = Conv2D(filters=nfilters // 4, kernel_size=(5, 1), kernel_initializer='he_normal', padding='same',
use_bias=False, name=f'5x1_conv_{name}')(y)
y = Conv2D(filters=nfilters // 4, kernel_size=(1, 5), kernel_initializer='he_normal', padding='same',
name=f'1x5_conv_{name}')(y)
elif dilated:
y = Conv2D(filters=nfilters // 4, kernel_size=(3, 3), kernel_initializer='he_normal',
dilation_rate=(dilated, dilated), padding='same', name=f'dilated_conv_{name}')(y)
y = BatchNormalization(momentum=0.1, name=f'bn_main_{name}')(y)
y = PReLU(shared_axes=[1, 2], name=f'prelu_{name}')(y)
y = Conv2D(filters=nfilters, kernel_size=(1, 1), kernel_initializer='he_normal', use_bias=False,
name=f'final_1x1_{name}')(y)
y = BatchNormalization(momentum=0.1, name=f'bn_final_{name}')(y)
y = SpatialDropout2D(rate=drate, name=f'spatial_dropout_final_{name}')(y)
y = Add(name=f'add_{name}')([y, skip])
y = PReLU(shared_axes=[1, 2], name=f'prelu_out_{name}')(y)
return y
def HRNetResidual(input_shape=(128, 128, 3), num_keypoints=20):
"""Instantiates HRNET Residual model
# Arguments
input_shape: List of three elements e.g. ''(H, W, 3)''
num_keypoints: Int.
# Returns
Tensorflow-Keras model.
# References
-[High-Resolution Representations for Labeling Pixels
and Regions](https://arxiv.org/pdf/1904.04514.pdf)
"""
# stem
inputs = Input(shape=input_shape, name='image')
x1 = stem(inputs, 64)
x1 = Conv2D(64 * 4, 1, padding='same', use_bias=False)(x1)
x1 = BatchNormalization()(x1)
for block in range(4):
x1 = bottleneck(x1)
# stage I
x1 = Conv2D(32, 3, padding='same', use_bias=False)(x1)
x1 = BatchNormalization()(x1)
x1 = Activation('relu')(x1)
x2 = transition_block(x1, 2)
# stage II
for block in range(4):
x1 = residual_block(x1, 32)
x2 = residual_block(x2, 64)
x1, x2 = fuse([x1, x2])
x3 = transition_block(x2, 2)
# stage III
for module in range(4):
for block in range(4):
x1 = residual_block(x1, 32)
x2 = residual_block(x2, 64)
x3 = residual_block(x3, 128)
x1, x2, x3 = fuse([x1, x2, x3])
x4 = transition_block(x3, 2)
# stage IV
for module in range(3):
for block in range(4):
x1 = residual_block(x1, 32)
x2 = residual_block(x2, 64)
x3 = residual_block(x3, 128)
x4 = residual_block(x4, 256)
x1, x2, x3, x4 = fuse([x1, x2, x3, x4])
# head
x2 = UpSampling2D(size=(2, 2))(x2)
x3 = UpSampling2D(size=(4, 4))(x3)
x4 = UpSampling2D(size=(8, 8))(x4)
x = Concatenate()([x1, x2, x3, x4])
x = Conv2D(480, 1)(x)
x = BatchNormalization(epsilon=1.001e-5)(x)
x = Activation('relu')(x)
x = Conv2D(num_keypoints, 1)(x)
# extra
x = BatchNormalization(epsilon=1.001e-5)(x)
x = Activation('relu')(x)
x = UpSampling2D(size=(4, 4), interpolation='bilinear')(x)
x = Permute([3, 1, 2])(x)
x = Reshape([num_keypoints, input_shape[0] * input_shape[1]])(x)
x = Activation('softmax')(x)
x = Reshape([num_keypoints, input_shape[0], input_shape[1]])(x)
outputs = ExpectedValue2D(name='keypoints')(x)
model = Model(inputs, outputs, name='hrnet-residual')
return model
def interest_evolution(
concat_behavior,
deep_input_item,
user_behavior_length,
gru_type="GRU",
use_neg=False,
neg_concat_behavior=None,
embedding_size=8,
att_hidden_size=(64, 16),
att_activation='sigmoid',
att_weight_normalization=False,
):
if gru_type not in ["GRU", "AIGRU", "AGRU", "AUGRU"]:
raise ValueError("gru_type error ")
aux_loss_1 = None
rnn_outputs = DynamicGRU(embedding_size * 2,
return_sequence=True,
name="gru1")(
[concat_behavior, user_behavior_length])
if gru_type == "AUGRU" and use_neg:
aux_loss_1 = auxiliary_loss(rnn_outputs[:, :-1, :],
concat_behavior[:, 1:, :],
neg_concat_behavior[:, 1:, :],
tf.subtract(user_behavior_length, 1),
stag="gru") # [:, 1:]
if gru_type == "GRU":
rnn_outputs2 = DynamicGRU(embedding_size * 2,
return_sequence=True,
name="gru2")(
[rnn_outputs, user_behavior_length])
# attention_score = AttentionSequencePoolingLayer(hidden_size=att_hidden_size, activation=att_activation, weight_normalization=att_weight_normalization, return_score=True)([
# deep_input_item, rnn_outputs2, user_behavior_length])
# outputs = Lambda(lambda x: tf.matmul(x[0], x[1]))(
# [attention_score, rnn_outputs2])
# hist = outputs
hist = AttentionSequencePoolingLayer(
att_hidden_units=att_hidden_size,
att_activation=att_activation,
weight_normalization=att_weight_normalization,
return_score=False)(
[deep_input_item, rnn_outputs2, user_behavior_length])
else: # AIGRU AGRU AUGRU
scores = AttentionSequencePoolingLayer(
att_hidden_units=att_hidden_size,
att_activation=att_activation,
weight_normalization=att_weight_normalization,
return_score=True)(
[deep_input_item, rnn_outputs, user_behavior_length])
if gru_type == "AIGRU":
hist = multiply([rnn_outputs, Permute([2, 1])(scores)])
final_state2 = DynamicGRU(
embedding_size * 2,
gru_type="GRU",
return_sequence=False,
name='gru2')([hist, user_behavior_length])
else: # AGRU AUGRU
final_state2 = DynamicGRU(embedding_size * 2,
gru_type=gru_type,
return_sequence=False,
name='gru2')([
rnn_outputs, user_behavior_length,
Permute([2, 1])(scores)
])
hist = final_state2
return hist, aux_loss_1
def build_net(self, dev):
with tf.variable_scope('a3c') and tf.device(dev):
screenInput = Input(
shape=(U.screen_channel(), self.ssize, self.ssize),
name='screenInput',
)
permutedScreenInput = Permute((2,3,1))(screenInput)
conv1 = Conv2D(16, kernel_size=5, strides=(1,1), padding='same',name='conv1')(permutedScreenInput)
conv2 = Conv2D(32, kernel_size=3, strides=(1,1), padding='same',name='conv2')(conv1)
infoInput = Input(
shape=(self.isize,),
name='infoInput',
)
customInput = Input(
shape=(self.custom_input_size,),
name='customInput',
)
nonSpatialInput = Concatenate(name='nonSpatialInputConcat')([infoInput, customInput])
broadcasted = Lambda(self.broadcast,name='broadcasting')(nonSpatialInput)
combinedSpatialNonSpatial = Concatenate(name='combinedConcat')([broadcasted, conv2])
conv3 = Conv2D(1, kernel_size=1, strides=(1,1), padding='same',name='conv3')(combinedSpatialNonSpatial)
flatConv3 = Flatten(name='flatConv3')(conv3)
lstmInput = Lambda(self.expand_dims, name='lstmInput')(flatConv3)
self.NUM_LSTM = 100
hStateInput = Input(
shape=(self.NUM_LSTM,),
name='hStateInput'
)
cStateInput = Input(
shape=(self.NUM_LSTM,),
name='cStateInput'
)
lstm, hStates, cStates = LSTM(self.NUM_LSTM, return_state=True)(lstmInput, initial_state=[hStateInput, cStateInput])
fc1 = Dense(256, activation='relu',name='dense1')(lstm)
fc2 = Dense(1, activation='linear',name='fc2')(fc1)
value = Lambda(self.Squeeze,name='value')(fc2)
policy = Dense(self.isize, activation='softmax',name='policy')(fc1)
broadcastLstm = Lambda(self.broadcast, name='breadcastLstm')(lstm)
spatialLstm = Concatenate(name='spatialLstm')([conv3, broadcastLstm])
conv4 = Conv2D(1,kernel_size=1, strides=(1,1), padding='same',name='conv4')(spatialLstm)
flatConv4 = Flatten(name='flattenedConv3')(conv4)
spatialPolicy = Softmax(name='spatialPolicy')(flatConv4)
conv5 = Conv2D(1, kernel_size=1, strides=(1,1), padding='same',name='conv5')(spatialLstm)
flatConv5 = Flatten(name='flattenedConv5')(conv5)
bestRoach = Softmax(name='bestRoach')(flatConv5)
self.model = Model(
inputs=[screenInput, infoInput, customInput, hStateInput, cStateInput],
outputs=[value, policy, spatialPolicy, hStates, cStates, bestRoach]
)
self.model._make_predict_function()
# encode input sequence and questions (which are indices)
# to sequences of dense vectors
input_encoded_m = input_encoder_m(input_sequence)
input_encoded_c = input_encoder_c(input_sequence)
question_encoded = question_encoder(question)
# compute a 'match' between the first input vector sequence
# and the question vector sequence
# shape: `(samples, story_maxlen, query_maxlen)`
match = dot([input_encoded_m, question_encoded], axes=(2, 2))
match = Activation('softmax')(match)
# add the match matrix with the second input vector sequence
response = add([match,
input_encoded_c]) # (samples, story_maxlen, query_maxlen)
response = Permute((2, 1))(response) # (samples, query_maxlen, story_maxlen)
# concatenate the match matrix with the question vector sequence
answer = concatenate([response, question_encoded])
# the original paper uses a matrix multiplication for this reduction step.
# we choose to use a RNN instead.
answer = LSTM(32)(answer) # (samples, 32)
# one regularization layer -- more would probably be needed.
answer = Dropout(0.3)(answer)
answer = Dense(vocab_size)(answer) # (samples, vocab_size)
# we output a probability distribution over the vocabulary
answer = Activation('softmax')(answer)
# build the final model
def cnn_line_lstm_ctc(input_shape, output_shape, **kwargs):
image_height, image_width = input_shape
output_length, num_classes = output_shape
image_input = Input(shape=input_shape, name='image')
y_true = Input(shape=(output_length, ), name='y_true')
input_length = Input(shape=(1, ), name='input_length')
label_length = Input(shape=(1, ), name='label_length')
gpu_present = len(device_lib.list_local_devices()) > 1
lstm_fn = CuDNNLSTM if gpu_present else LSTM
##### Your code below (Lab 3)
image_reshaped = Reshape((image_height, image_width, 1))(image_input)
# (image_height, image_width, 1)
convnet_outputs = image_reshaped
# convnet_outputs = Dropout(0.5)(convnet_outputs)
convnet_outputs = Conv2D(16, 3, padding='SAME')(convnet_outputs)
convnet_outputs = BatchNormalization()(convnet_outputs)
convnet_outputs = LeakyReLU()(convnet_outputs)
convnet_outputs = MaxPooling2D(2, 2)(convnet_outputs)
# convnet_outputs = Dropout(0.5)(convnet_outputs)
convnet_outputs = Conv2D(32, 3, padding='SAME')(convnet_outputs)
convnet_outputs = BatchNormalization()(convnet_outputs)
convnet_outputs = LeakyReLU()(convnet_outputs)
convnet_outputs = MaxPooling2D(2, 2)(convnet_outputs)
convnet_outputs = Dropout(0.2)(convnet_outputs)
convnet_outputs = Conv2D(48, 3, padding='SAME')(convnet_outputs)
convnet_outputs = BatchNormalization()(convnet_outputs)
convnet_outputs = LeakyReLU()(convnet_outputs)
convnet_outputs = MaxPooling2D(2, 2)(convnet_outputs)
convnet_outputs = Dropout(0.2)(convnet_outputs)
convnet_outputs = Conv2D(64, 3, padding='SAME')(convnet_outputs)
convnet_outputs = BatchNormalization()(convnet_outputs)
convnet_outputs = LeakyReLU()(convnet_outputs)
convnet_outputs = Dropout(0.2)(convnet_outputs)
convnet_outputs = Conv2D(80, 3, padding='SAME')(convnet_outputs)
convnet_outputs = BatchNormalization()(convnet_outputs)
convnet_outputs = LeakyReLU()(convnet_outputs)
num_windows = 119
convnet_outputs = Permute([2, 1, 3])(convnet_outputs)
convnet_outputs = Reshape([num_windows, 240])(convnet_outputs)
# (num_windows, 128)
for i in range(5):
convnet_outputs = Dropout(0.5)(convnet_outputs)
lstm_output = Bidirectional(lstm_fn(
256, return_sequences=True))(convnet_outputs)
lstm_output = Dropout(0.5)(lstm_output)
softmax_output = Dense(num_classes,
activation='softmax',
name='softmax_output')(lstm_output)
# (num_windows, num_classes)
##### Your code above (Lab 3)
input_length_processed = Lambda(
lambda x, num_windows=None: x * num_windows,
arguments={'num_windows': num_windows})(input_length)
ctc_loss_output = Lambda(
lambda x: K.ctc_batch_cost(x[0], x[1], x[2], x[3]), name='ctc_loss')(
[y_true, softmax_output, input_length_processed, label_length])
ctc_decoded_output = Lambda(
lambda x: ctc_decode(x[0], x[1], output_length),
name='ctc_decoded')([softmax_output, input_length_processed])
model = KerasModel(
inputs=[image_input, y_true, input_length, label_length],
outputs=[ctc_loss_output, ctc_decoded_output])
return model
def get_unet(n_ch, patch_height, patch_width):
inputs = Input(shape=(n_ch, patch_height, patch_width))
conv1 = Conv2D(32, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(inputs)
conv1 = Dropout(0.2)(conv1)
conv1 = Conv2D(32, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(conv1)
pool1 = MaxPooling2D((2, 2))(conv1)
#
conv2 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(pool1)
conv2 = Dropout(0.2)(conv2)
conv2 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(conv2)
pool2 = MaxPooling2D((2, 2))(conv2)
#
conv3 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(pool2)
conv3 = Dropout(0.2)(conv3)
conv3 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(conv3)
up1 = UpSampling2D(size=(2, 2))(conv3)
up1 = concatenate([conv2, up1], axis=1)
conv4 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(up1)
conv4 = Dropout(0.2)(conv4)
conv4 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(conv4)
#
up2 = UpSampling2D(size=(2, 2))(conv4)
up2 = concatenate([conv1, up2], axis=1)
conv5 = Conv2D(32, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(up2)
conv5 = Dropout(0.2)(conv5)
conv5 = Conv2D(32, (3, 3),
activation='relu',
padding='same',
data_format='channels_first')(conv5)
#
conv6 = Conv2D(2, (1, 1),
activation='relu',
padding='same',
data_format='channels_first')(conv5)
conv6 = Reshape((2, patch_height * patch_width))(conv6)
conv6 = Permute((2, 1))(conv6)
############
conv7 = Activation('softmax')(conv6)
model = Model(inputs=inputs, outputs=conv7)
# sgd = SGD(lr=0.01, decay=1e-6, momentum=0.3, nesterov=False)
model.compile(optimizer='sgd',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def create_network(n_commands,
n_channel,
n_value1,
n_value2,
n_durations,
embed_size=100,
rnn_units=256,
use_attention=False):
""" create the structure of the neural network """
commands_in = Input(shape=(None, ), name="commands_in")
channel_in = Input(shape=(None, ), name="channels_in")
value1_in = Input(shape=(None, ), name="values1_in")
value2_in = Input(shape=(None, ), name="values2_in")
durations_in = Input(shape=(None, ), name="durations_in")
x1 = Embedding(n_commands, embed_size)(commands_in)
x2 = Embedding(n_channel, embed_size)(channel_in)
x3 = Embedding(n_value1, embed_size)(value1_in)
x4 = Embedding(n_value2, embed_size)(value2_in)
x5 = Embedding(n_durations, embed_size)(durations_in)
x = Concatenate()([x1, x2, x3, x4, x5])
x = LSTM(rnn_units, return_sequences=True)(x)
# x = Dropout(0.2)(x)
if use_attention:
x = LSTM(rnn_units, return_sequences=True)(x)
# x = Dropout(0.2)(x)
e = Dense(1, activation='tanh')(x)
e = Reshape([-1])(e)
alpha = Activation('softmax')(e)
alpha_repeated = Permute([2, 1])(RepeatVector(rnn_units)(alpha))
c = Multiply()([x, alpha_repeated])
c = Lambda(lambda xin: K.sum(xin, axis=1),
output_shape=(rnn_units, ))(c)
else:
c = LSTM(rnn_units)(x)
# c = Dropout(0.2)(c)
commands_out = Dense(n_commands, activation='softmax',
name='commands_out')(c)
channel_out = Dense(n_channel, activation='softmax',
name='channels_out')(c)
value1_out = Dense(n_value1, activation='softmax', name='values1_out')(c)
value2_out = Dense(n_value2, activation='softmax', name='values2_out')(c)
durations_out = Dense(n_durations,
activation='softmax',
name='durations_out')(c)
model = Model(
[commands_in, channel_in, value1_in, value2_in, durations_in],
[commands_out, channel_out, value1_out, value2_out, durations_out])
if use_attention:
att_model = Model(
[commands_in, channel_in, value1_in, value2_in, durations_in],
alpha)
else:
att_model = None
opti = RMSprop(lr=0.001)
model.compile(loss=[
'categorical_crossentropy', 'categorical_crossentropy',
'categorical_crossentropy', 'categorical_crossentropy',
'categorical_crossentropy'
],
optimizer=opti)
return model, att_model
def EEGNet_old(nb_classes,
Chans=64,
Samples=128,
regRate=0.0001,
dropoutRate=0.25,
kernels=[(2, 32), (8, 4)],
strides=(2, 4)):
""" Keras Implementation of EEGNet_v1 (https://arxiv.org/abs/1611.08024v2)
This model is the original EEGNet model proposed on arxiv
https://arxiv.org/abs/1611.08024v2
with a few modifications: we use striding instead of max-pooling as this
helped slightly in classification performance while also providing a
computational speed-up.
Note that we no longer recommend the use of this architecture, as the new
version of EEGNet performs much better overall and has nicer properties.
Inputs:
nb_classes : total number of final categories
Chans, Samples : number of EEG channels and samples, respectively
regRate : regularization rate for L1 and L2 regularizations
dropoutRate : dropout fraction
kernels : the 2nd and 3rd layer kernel dimensions (default is
the [2, 32] x [8, 4] configuration)
strides : the stride size (note that this replaces the max-pool
used in the original paper)
"""
# start the model
input_main = Input((Chans, Samples))
layer1 = Conv2D(16, (Chans, 1),
input_shape=(Chans, Samples, 1),
kernel_regularizer=l1_l2(l1=regRate,
l2=regRate))(input_main)
layer1 = BatchNormalization()(layer1)
layer1 = Activation('elu')(layer1)
layer1 = Dropout(dropoutRate)(layer1)
permute_dims = 2, 1, 3
permute1 = Permute(permute_dims)(layer1)
layer2 = Conv2D(4,
kernels[0],
padding='same',
kernel_regularizer=l1_l2(l1=0.0, l2=regRate),
strides=strides)(permute1)
layer2 = BatchNormalization()(layer2)
layer2 = Activation('elu')(layer2)
layer2 = Dropout(dropoutRate)(layer2)
layer3 = Conv2D(4,
kernels[1],
padding='same',
kernel_regularizer=l1_l2(l1=0.0, l2=regRate),
strides=strides)(layer2)
layer3 = BatchNormalization()(layer3)
layer3 = Activation('elu')(layer3)
layer3 = Dropout(dropoutRate)(layer3)
flatten = Flatten(name='flatten')(layer3)
dense = Dense(nb_classes, name='dense')(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input_main, outputs=softmax)
def est_net(lr, Nt, net, mode):
# Second part, channel estimator
x0 = Input(shape=(R, 2))
if net == 'CNN':
# change the order
x1 = Permute((2, 1))(x0)
x1 = Flatten()(x1)
x1 = Reshape((R, 2))(x1)
x1 = Conv1D(filters=96, kernel_size=7, padding='same')(x1)
x1 = BatchNormalization()(x1)
x1 = Activation('relu')(x1)
if mode == 'naive':
x1 = naive_attention(x1)
if mode == 'feature':
x1 = feature_attention(x1, x0)
x2 = Conv1D(filters=96, kernel_size=5, padding='same')(x1)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
if mode == 'naive':
x2 = naive_attention(x2)
if mode == 'feature':
x2 = feature_attention(x2, x0)
x3 = Flatten()(x2)
prediction = Dense(2 * Nt, activation='linear')(x3)
if net == 'CNN2':
x1 = Conv1D(filters=96, kernel_size=7, padding='same')(x0)
x1 = BatchNormalization()(x1)
x1 = Activation('relu')(x1)
x2 = Conv1D(filters=96, kernel_size=7, padding='same')(x1)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
x2 = Conv1D(filters=96, kernel_size=5, padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
x2 = Conv1D(filters=96, kernel_size=5, padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
x2 = Conv1D(filters=96, kernel_size=3, padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
x3 = Flatten()(x2)
prediction = Dense(2 * Nt, activation='linear')(x3)
if net == 'CNN3':
x1 = Flatten()(x0)
#x1 = Dense(4*Nt,activation='relu')(x1)
#x1 = BatchNormalization()(x1)
x1 = Dense(2 * Nt, activation='linear')(x1)
x1 = Reshape((Nt, 2))(x1)
x1 = Conv1D(filters=96, kernel_size=7, padding='same')(x1)
x1 = BatchNormalization()(x1)
x1 = Activation('relu')(x1)
if mode == 'naive':
x1 = naive_attention(x1)
if mode == 'feature':
x1 = feature_attention(x1, x0)
x2 = Conv1D(filters=96, kernel_size=5, padding='same')(x1)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
if mode == 'naive':
x2 = naive_attention(x2)
if mode == 'feature':
x2 = feature_attention(x2, x0)
x3 = Flatten()(x2)
#prediction = Conv1D(filters=2,kernel_size=3,padding='same')(x2)
#prediction = Flatten()(prediction)
prediction = Dense(2 * Nt, activation='linear')(x3)
if net == 'res_CNN':
x = Conv1D(filters=96, kernel_size=7, padding='same')(x0)
x = BatchNormalization()(x)
x_init = Activation('relu')(x)
if mode == 'naive':
x_init = naive_attention(x_init)
if mode == 'feature':
x_init = feature_attention(x_init, x0)
for i in range(3): # number of residual blocks
x = Conv1D(filters=96, kernel_size=5, padding='same')(x_init)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv1D(filters=96, kernel_size=3, padding='same')(x)
x = BatchNormalization()(x)
if mode == 'naive':
x = naive_attention(x)
if mode == 'feature':
x = feature_attention(x, x0)
x_init = Add()([x, x_init])
x_init = Activation('relu')(x_init)
x2 = Conv1D(filters=96, kernel_size=3, padding='same')(x_init)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
if mode == 'naive':
x2 = naive_attention(x2)
if mode == 'feature':
x2 = feature_attention(x2, x0)
x3 = Flatten()(x2)
prediction = Dense(2 * Nt, activation='linear')(x3)
if net == 'FNN':
x1 = Flatten()(x0) # 256,512,256
x1 = Dense(256, activation='relu')(x1)
x1 = BatchNormalization()(x1)
x1 = Dense(512, activation='relu')(x1)
x1 = BatchNormalization()(x1)
x1 = Dense(256, activation='relu')(x1)
x1 = BatchNormalization()(x1)
x3 = Flatten()(x1)
prediction = Dense(2 * Nt, activation='linear')(x3)
if net == 'FNN3':
x1 = Flatten()(x0)
x1 = Dense(96 * 96 // 2, activation='relu')(x1)
x1 = BatchNormalization()(x1)
x3 = Flatten()(x1)
prediction = Dense(2 * Nt, activation='linear')(x3)
if net == 'FNN2':
x1 = Flatten()(x0)
#x1 = Dense(128,activation='relu')(x1)
#x1 = BatchNormalization()(x1)
x1 = Dense(16 * 192, activation='relu')(x1)
x1 = BatchNormalization()(x1)
# grouping
x1 = Reshape((16, 192))(x1)
# attention
x1 = naive_attention(x1)
#x1 = feature_attention(x1,x0)
#x1 = Flatten()(x1)
#x1 = Dense(512,activation='relu')(x1)
#x1 = BatchNormalization()(x1)
x3 = Flatten()(x1)
prediction = Dense(2 * Nt, activation='linear')(x3)
model = Model(inputs=x0, outputs=prediction)
model.compile(loss='mse', optimizer=Adam(lr=lr))
model.summary()
return model
def attRNN():
sr = 8000
inputs = Input((8000, 1), name='input')
x = Reshape((1, -1))(inputs)
m = Melspectrogram(n_dft=1024,
n_hop=128,
input_shape=(1, 8000),
padding='same',
sr=sr,
n_mels=80,
fmin=40.0,
fmax=sr / 2,
power_melgram=1.0,
return_decibel_melgram=True,
trainable_fb=False,
trainable_kernel=False,
name='mel_stft')
m.trainable = False
x = m(x)
x = Normalization2D(int_axis=0, name='mel_stft_norm')(x)
# note that Melspectrogram puts the sequence in shape (batch_size, melDim, timeSteps, 1)
# we would rather have it the other way around for LSTMs
x = Permute((2, 1, 3))(x)
x = Conv2D(10, (5, 1), activation='relu', padding='same')(x)
x = BatchNormalization()(x)
x = Conv2D(1, (5, 1), activation='relu', padding='same')(x)
x = BatchNormalization()(x)
# x = Reshape((125, 80)) (x)
# keras.backend.squeeze(x, axis)
x = Lambda(lambda q: K.squeeze(q, -1), name='squeeze_last_dim')(x)
x = Bidirectional(LSTM(64, return_sequences=True))(
x) # [b_s, seq_len, vec_dim]
x = Bidirectional(LSTM(64, return_sequences=True))(
x) # [b_s, seq_len, vec_dim]
xFirst = Lambda(lambda q: q[:, -1])(x) # [b_s, vec_dim]
query = Dense(128)(xFirst)
# dot product attention
attScores = Dot(axes=[1, 2])([query, x])
attScores = Softmax(name='attSoftmax')(attScores) # [b_s, seq_len]
# rescale sequence
attVector = Dot(axes=[1, 1])([attScores, x]) # [b_s, vec_dim]
x = Dense(64, activation='relu')(attVector)
x = Dense(32)(x)
output = Dense(9, activation='softmax', name='output')(x)
model = Model(inputs=[inputs], outputs=[output])
model.compile(optimizer='adam',
loss=['sparse_categorical_crossentropy'],
metrics=['sparse_categorical_accuracy'])
return model
def _buildDQN(self):
"""
Build a network consistent with each type of inputs
"""
layers = []
outs_conv = []
inputs = []
for i, dim in enumerate(self._input_dimensions):
# - observation[i] is a FRAME
if len(dim) == 3 or len(dim) == 4:
if (len(dim) == 4):
input = Input(shape=(dim[-4], dim[-3], dim[-2], dim[-1]))
inputs.append(input)
input = Reshape((dim[-4] * dim[-3], dim[-2], dim[-1]),
input_shape=(dim[-4], dim[-3], dim[-2],
dim[-1]))(input)
x = Permute(
(2, 3, 1),
input_shape=(dim[-4] * dim[-3], dim[-2], dim[-1]))(
input) #data_format='channels_last'
else:
input = Input(shape=(dim[-3], dim[-2], dim[-1]))
inputs.append(input)
x = Permute((2, 3, 1),
input_shape=(dim[-3], dim[-2], dim[-1]))(
input) #data_format='channels_last'
x = Conv2D(8, (4, 4), activation='relu',
padding='valid')(x) #Conv on the frames
x = Conv2D(16, (3, 3), activation='relu',
padding='valid')(x) #Conv on the frames
x = MaxPooling2D(pool_size=(2, 2),
strides=None,
padding='valid')(x)
x = Conv2D(16, (3, 3), activation='relu',
padding='valid')(x) #Conv on the frames
out = Flatten()(x)
# - observation[i] is a VECTOR
elif len(dim) == 2:
if dim[0] > 3:
input = Input(shape=(dim[0], dim[1]))
inputs.append(input)
reshaped = Reshape((dim[0], dim[1], 1),
input_shape=(dim[0], dim[1]))(input)
x = Conv2D(16, (2, 1), activation='relu',
padding='valid')(reshaped) #Conv on the history
x = Conv2D(16, (2, 1), activation='relu', padding='valid')(
x) #Conv on the history & features
out = Flatten()(x)
else:
input = Input(shape=(dim[0], dim[1]))
inputs.append(input)
out = Flatten()(input)
# - observation[i] is a SCALAR -
else:
if dim[0] > 3:
# this returns a tensor
input = Input(shape=(dim[0], ))
inputs.append(input)
reshaped = Reshape((1, dim[0], 1),
input_shape=(dim[0], ))(input)
x = Conv2D(8, (1, 2), activation='relu',
padding='valid')(reshaped) #Conv on the history
x = Conv2D(8, (1, 2), activation='relu',
padding='valid')(x) #Conv on the history
out = Flatten()(x)
else:
input = Input(shape=(dim[0], ))
inputs.append(input)
out = input
outs_conv.append(out)
if (self._action_as_input == True):
if (isinstance(self._n_actions, int)):
print(
"Error, env.nActions() must be a continuous set when using actions as inputs in the NN"
)
else:
input = Input(shape=(len(self._n_actions), ))
inputs.append(input)
outs_conv.append(input)
if len(outs_conv) > 1:
x = concatenate(outs_conv)
else:
x = outs_conv[0]
# we stack a deep fully-connected network on top
x = Dense(50, activation='relu')(x)
x = Dense(20, activation='relu')(x)
if (self._action_as_input == False):
if (isinstance(self._n_actions, int)):
out = Dense(self._n_actions)(x)
else:
out = Dense(len(self._n_actions))(x)
else:
out = Dense(1)(x)
model = Model(inputs=inputs, outputs=out)
layers = model.layers
# Grab all the parameters together.
params = [
param for layer in layers for param in layer.trainable_weights
]
if (self._action_as_input == True):
return model, params, inputs
else:
return model, params
def build_and_load_model_tf(model_capacity: str):
"""
Build the CNN model and load the weights, using the TensorFlow backend
Parameters
----------
model_capacity : 'tiny', 'small', 'medium', 'large', or 'full'
String specifying the model capacity, which determines the model's
capacity multiplier to 4 (tiny), 8 (small), 16 (medium), 24 (large),
or 32 (full). 'full' uses the model size specified in the paper,
and the others use a reduced number of filters in each convolutional
layer, resulting in a smaller model that is faster to evaluate at the
cost of slightly reduced pitch estimation accuracy.
Returns
-------
model : tensorflow.keras.models.Model
The pre-trained model loaded in memory
"""
from tensorflow.keras.layers import (Input, Reshape, Conv2D,
BatchNormalization)
from tensorflow.keras.layers import (MaxPool2D, Dropout, Permute, Flatten,
Dense)
from tensorflow.keras.models import Model
if models['tf'][model_capacity] is None:
capacity_multiplier = {
'tiny': 4,
'small': 8,
'medium': 16,
'large': 24,
'full': 32
}[model_capacity]
layers = [1, 2, 3, 4, 5, 6]
filters = [n * capacity_multiplier for n in [32, 4, 4, 4, 8, 16]]
widths = [512, 64, 64, 64, 64, 64]
strides = [(4, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)]
x = Input(shape=(1024, ), name='input', dtype='float32')
y = Reshape(target_shape=(1024, 1, 1), name='input-reshape')(x)
for l, f, w, s in zip(layers, filters, widths, strides):
y = Conv2D(f, (w, 1),
strides=s,
padding='same',
activation='relu',
name="conv%d" % l)(y)
y = BatchNormalization(name="conv%d-BN" % l)(y)
y = MaxPool2D(pool_size=(2, 1),
strides=None,
padding='valid',
name="conv%d-maxpool" % l)(y)
y = Dropout(0.25, name="conv%d-dropout" % l)(y)
y = Permute((2, 1, 3), name="transpose")(y)
y = Flatten(name="flatten")(y)
y = Dense(360, activation='sigmoid', name="classifier")(y)
model = Model(inputs=x, outputs=y)
package_dir = os.path.dirname(os.path.realpath(__file__))
filename = "model-{}.h5".format(model_capacity)
model.load_weights(os.path.join(package_dir, filename))
model.compile('adam', 'binary_crossentropy')
models['tf'][model_capacity] = model
def attention_3d_block(inputs):
a = Permute((2, 1))(inputs)
a = Dense(3, activation='softmax')(a)
a_probs = Permute((2, 1))(a)
output_attention_mul = Multiply()([inputs, a_probs])
return output_attention_mul
def buildModel(
num_input_channels=24,
num_timesteps=20,
num_filters=20,
num_LSTM=20,
num_Dense=200,
drop_rate=0.2,
reg=None,
):
kernel_size = (3, 3)
num_classes = 2
ac = "relu"
opt = Adam(lr=0.001, decay=0, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
inp = Input(
(num_timesteps, num_input_channels, 8, 8)
) # timesteps, channels, rows, columns
permuted = Permute((1, 3, 4, 2))(inp) # expects channels_last
x = TimeDistributed(
Conv2D(
num_filters,
kernel_size,
activation=ac,
kernel_regularizer=reg,
# data_format="channels_first",
)
)(permuted)
x = BatchNormalization(axis=-1)(x)
x = TimeDistributed(
Conv2D(
2 * num_filters,
kernel_size,
activation=ac,
kernel_regularizer=reg,
# data_format="channels_first",
)
)(x)
x = BatchNormalization(axis=-1)(x)
x = TimeDistributed(
Conv2D(
4 * num_filters,
kernel_size,
activation=ac,
kernel_regularizer=reg,
# data_format="channels_first",
)
)(x)
x = BatchNormalization(axis=-1)(x)
x = TimeDistributed(
Conv2D(
8 * num_filters,
kernel_size,
activation=ac,
kernel_regularizer=reg,
padding="same",
# data_format="channels_first",
)
)(x)
x = BatchNormalization(axis=-1)(x)
x = TimeDistributed(Flatten())(x)
x = LSTM(num_LSTM)(x)
x = Dropout(drop_rate)(x)
x = Dense(num_Dense)(x)
x = Dropout(drop_rate)(x)
out = Dense(num_classes, activation="softmax")(x)
myModel = Model(inputs=[inp], outputs=[out])
myModel.compile(loss="binary_crossentropy", metrics=["accuracy"], optimizer=opt)
LOGGER.info("built model!")
LOGGER.info(myModel.summary())
return myModel
def create_model(input_shape, config):
weight_decay = 0.001
model = Sequential()
model.add(
Convolution2D(
16,
7,
7,
kernel_regularizer=l2(weight_decay),
activation="relu",
input_shape=input_shape,
))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Convolution2D(32,
5,
5,
kernel_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Convolution2D(64,
3,
3,
kernel_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Convolution2D(128,
3,
3,
kernel_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 1)))
model.add(
Convolution2D(256,
3,
3,
kernel_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 1)))
# model.load_weights("logs/2017-04-08-13-03-44/weights.08.model", by_name=True)
# for ref_layer in ref_model.layers:
# layer = model.get_layer(ref_layer.name)
# if layer:
# layer.set_weights(ref_layer.get_weights())
for layer in model.layers:
layer.trainable = False
# (bs, y, x, c) --> (bs, x, y, c)
model.add(Permute((2, 1, 3)))
# (bs, x, y, c) --> (bs, x, y * c)
bs, x, y, c = model.layers[-1].output_shape
model.add(Reshape((x, y * c)))
model.add(
Bidirectional(LSTM(512, return_sequences=False), merge_mode="concat"))
model.add(Dense(config["num_classes"], activation="softmax"))
return model
def __init__(self, input_shape, model_folder="models/serialized/"):
super().__init__()
conv_model = Sequential()
conv_model.add(Permute((1, 2, 3), input_shape=input_shape))
for l in Convblock(64, 1, 2):
conv_model.add(l)
for l in Convblock(128, 2, 2):
conv_model.add(l)
for l in Convblock(256, 3, 3):
conv_model.add(l)
for l in Convblock(512, 4, 3):
conv_model.add(l)
for l in Convblock(512, 5, 3):
conv_model.add(l)
conv_model.add(
Convolution2D(2048,
kernel_size=(7, 7),
padding="same",
activation="relu",
name="fc_6"))
# Replacing fully connected layers of VGG Net using convolutions -- reduced filters
conv_model.add(
Convolution2D(2048,
kernel_size=(1, 1),
padding="same",
activation="relu",
name="fc7"))
conv_model.add(
Convolution2D(1,
kernel_size=(1, 1),
padding="same",
activation="relu",
name="score_fr"))
Conv_size = conv_model.layers[-1].output_shape[2]
print("Convolution size: " + str(Conv_size))
#
conv_model.add(
Conv2DTranspose(1,
strides=(2, 2),
kernel_size=(4, 4),
padding="valid",
activation=None,
name="score2"))
# I = (O-1)*Stride + K
Deconv_size = conv_model.layers[-1].output_shape[2]
print("Deconvolution size: " + str(Deconv_size))
# 2 if image size is 512*512
Extra = (Deconv_size - 2 * Conv_size)
print("Extra size: " + str(Extra))
# Cropping to get correct size
conv_model.add(Cropping2D(cropping=((0, Extra), (0, Extra))))
Conv_size = conv_model.layers[-1].output_shape[2]
skip_con1 = Convolution2D(1,
kernel_size=(1, 1),
padding="same",
activation=None,
name="score_pool4")
# Adding skip connection which takes adds the output of Max pooling layer 4 to current layer
Summed = add(inputs=[
skip_con1(conv_model.layers[14].output),
conv_model.layers[-1].output
])
# Upsampling output of first skip connection
x = Conv2DTranspose(21,
kernel_size=(4, 4),
strides=(2, 2),
padding="valid",
activation=None,
name="score4")(Summed)
x = Cropping2D(cropping=((0, 2), (0, 2)))(x)
# Conv to be applied to pool3
skip_con2 = Convolution2D(21,
kernel_size=(1, 1),
padding="same",
activation=None,
name="score_pool3")
# Adding skip connection which takes output og Max pooling layer 3 to current layer
Summed = add(inputs=[skip_con2(conv_model.layers[10].output), x])
Up = Conv2DTranspose(1,
kernel_size=(16, 16),
strides=(8, 8),
padding="valid",
activation=None,
name="upsample")(Summed)
# Cropping the extra part obtained due to transpose convolution
final = Cropping2D(cropping=((0, 8), (0, 8)))(Up)
self.seg_model = Model(conv_model.input, final)
self.seg_model.summary()
self.model_folder = model_folder
self.weights_path = (model_folder +
"fcn_weights.best.hdf5").format('seg_model')
self.set_callbacks()
self.name = "FCN8"
def HRNetDense(input_shape=(128, 128, 3), num_keypoints=20, growth_rate=4):
# stem
inputs = Input(shape=input_shape)
x1 = stem(inputs, 64)
x1 = Conv2D(64 * 4, 1, padding='same', use_bias=False)(x1)
x1 = BatchNormalization()(x1)
for block in range(4):
x1 = bottleneck(x1)
# stage I
x1 = Conv2D(32, 3, padding='same', use_bias=False)(x1)
x1 = BatchNormalization()(x1)
x1 = Activation('relu')(x1)
x2 = transition_block(x1, 2)
print('stage 1', x1.shape, x2.shape)
# stage II
x1 = dense_block(x1, 4, growth_rate)
x2 = dense_block(x2, 4, growth_rate)
x1, x2 = fuse([x1, x2])
x3 = transition_block(x2, 0.5)
print('stage 2', x1.shape, x2.shape, x3.shape)
# stage III
x1 = dense_block(x1, 4, growth_rate)
x2 = dense_block(x2, 4, growth_rate)
x3 = dense_block(x3, 4, growth_rate)
x1, x2, x3 = fuse([x1, x2, x3])
x4 = transition_block(x3, 0.5)
print('stage 3', x1.shape, x2.shape, x3.shape, x4.shape)
# stage IV
x1 = dense_block(x1, 3, growth_rate)
x2 = dense_block(x2, 3, growth_rate)
x3 = dense_block(x3, 3, growth_rate)
x4 = dense_block(x4, 3, growth_rate)
x1, x2, x3, x4 = fuse([x1, x2, x3, x4])
print('stage 4', x1.shape, x2.shape, x3.shape, x4.shape)
x2 = UpSampling2D(size=(2, 2))(x2)
x3 = UpSampling2D(size=(4, 4))(x3)
x4 = UpSampling2D(size=(8, 8))(x4)
x = Concatenate()([x1, x2, x3, x4])
# head
x = Conv2D(480, 1)(x)
x = BatchNormalization(epsilon=1.001e-5)(x)
x = Activation('relu')(x)
x = Conv2D(num_keypoints, 1)(x)
# extra
x = BatchNormalization(epsilon=1.001e-5)(x)
x = Activation('relu')(x)
x = UpSampling2D(size=(4, 4), interpolation='bilinear')(x)
x = Permute([3, 1, 2])(x)
x = Reshape([num_keypoints, input_shape[0] * input_shape[1]])(x)
x = Activation('softmax')(x)
x = Reshape([num_keypoints, input_shape[0], input_shape[1]])(x)
outputs = ExpectedValue2D(name='expected_uv')(x)
model = Model(inputs, outputs, name='hrnet-dense')
return model
def build_model(X, nb_classes):
pool_size = (3, 3) # size of pooling area for max pooling
kernel_size = (5, 5) # convolution kernel size
nb_layers = 6
input_shape = (X.shape[1], X.shape[2], X.shape[3])
model = Sequential()
# 4 2D Convolution Layers
model.add(
Conv2D(64,
input_shape=input_shape,
kernel_size=kernel_size,
kernel_regularizer=l2(0.01),
bias_regularizer=l2(0.01),
padding="same",
activation="relu"))
model.add(MaxPooling2D(pool_size=pool_size))
model.add(BatchNormalization())
model.add(Dropout(0.25))
# model.add(Conv2D(64,kernel_size=kernel_size,kernel_regularizer=l2(0.01),bias_regularizer=l2(0.01),padding="same",activation="relu"))
# model.add(MaxPooling2D(pool_size=pool_size))
# model.add(BatchNormalization())
# model.add(Dropout(0.25))
# model.add(Conv2D(128,kernel_size=kernel_size,kernel_regularizer=l2(0.01),bias_regularizer=l2(0.01),padding="same",activation="relu"))
# model.add(MaxPooling2D(pool_size=pool_size))
# model.add(BatchNormalization())
# model.add(Dropout(0.2))
# model.add(Conv2D(128,kernel_size=kernel_size,kernel_regularizer=l2(0.01),bias_regularizer=l2(0.01),padding="same",activation="relu"))
# model.add(MaxPooling2D(pool_size=pool_size))
# model.add(BatchNormalization())
# model.add(Dropout(0.25))
model.add(Permute((2, 1, 3)))
last_shape = model.layers[-1].output_shape
model.add(Reshape((last_shape[1], last_shape[2] * last_shape[3])))
# 2 LSTM layers
model.add(
LSTM(64,
kernel_regularizer=l2(0.01),
recurrent_regularizer=l2(0.01),
recurrent_dropout=0.5,
return_sequences=True,
go_backwards=False,
activation='tanh'))
model.add(
LSTM(64,
kernel_regularizer=l2(0.01),
recurrent_regularizer=l2(0.01),
recurrent_dropout=0.5,
return_sequences=True,
go_backwards=True,
activation='tanh'))
model.add(Dropout(0.25))
# model.add(Flatten())
model.add(TimeDistributed(Dense(128, activation='relu')))
model.add(Dropout(0.25))
model.add(TimeDistributed(Dense(64, activation='relu')))
model.add(Dropout(0.2))
model.add(Flatten())
#relu throughout the network and softmax at output layer (multiple classes)
model.add(Dense(nb_classes, activation='softmax'))
return model
def Create_Model(length, vocab_size):
# INPUT, EMBEDDING
inputs0 = Input(batch_shape=(BATCH_SIZE, len(net_train[0])))
embedding1 = Embedding(vocab_size, 150)(inputs0)
drop_out1 = Dropout(0.1, name='dropout1')(embedding1)
# 2 PARTs FOR BIDIRECTIONAL LSTM
lstm_fwd = LSTM(150, return_sequences=True, name='lstm_fwd')(drop_out1)
lstm_bwd = LSTM(150,
return_sequences=True,
go_backwards=True,
name='lstm_bwd')(drop_out1)
# CONCAT 2 PARTS FOR BILSTM
bilstm = concatenate([lstm_fwd, lstm_bwd],
name='bilstm') # , mode='concat'
drop_out2 = Dropout(0.1)(bilstm)
# "LAST ELEMENT FROM TIME DIM"
h_n = Lambda(get_H_n, output_shape=(300, ), name="h_n")(drop_out2)
Whn = Dense(300, kernel_regularizer=l2(0.01), name="Wh_n")(h_n)
Whn_x_e = RepeatVector(L, name="Wh_n_x_e")(Whn)
# "FIRST first-sentence-MAXLEN ELEMENTS FROM TIME DIM"
Y = Lambda(get_Y,
arguments={"xmaxlen": L},
name="Y",
output_shape=(L, 300))(drop_out2)
WY = TimeDistributed(Dense(300, kernel_regularizer=l2(0.01)), name="WY")(Y)
# CONCAT 2 LAMBA "ATTENTIONS"?
merged1 = Add()(
[Whn_x_e,
WY]) # concatenate([Whn_x_e, WY], name="merged1", mode='sum')
M = Activation('tanh', name="M")(merged1)
alpha_ = TimeDistributed(Dense(1, activation='linear'), name="alpha_")(M)
# FLATTEN AND GATE
flat_alpha = Flatten(name="flat_alpha")(alpha_)
alpha = Dense(L, activation='softmax', name="alpha")(flat_alpha)
# CHANGE SHAPE OF Y
Y_trans = Permute((2, 1), name="y_trans")(Y) # of shape (None,300,20)
# CONCAT CHANGED Y WITH MAIN PATHWAY OF NN
r_ = Lambda(get_R,
output_shape=(300, 1))([Y_trans,
alpha]) #, output_shape=(300, 1)
#r_ = concatenate([Y_trans, alpha], output_shape=(300, 1), name="r_", mode=get_R)
r = Reshape((300, ), name="r")(r_)
Wr = Dense(300, kernel_regularizer=l2(0.01))(r)
# CALL ALL THE WAY BACK TO h_n... god knows why
Wh = Dense(300, kernel_regularizer=l2(0.01))(h_n)
# MERGE MAIN PATHWAY OF NN WITH CALLBACK LAYER TO h_n
merged2 = Add()([Wr, Wh]) # concatenate([Wr, Wh], mode='sum')
h_star = Activation('tanh')(merged2)
outputs = Dense(3, activation='softmax')(h_star)
model = Model(inputs=[inputs0], outputs=outputs)
model.compile(Adam(lr=0.001),
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
plot_model(model, show_shapes=True, to_file='multichannel2.png')
return model
def create_model(input_shape, config, is_training=True):
weight_decay = 0.001
model = Sequential()
model.add(
Convolution2D(64,
3,
3,
W_regularizer=l2(weight_decay),
activation="relu",
input_shape=input_shape))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Convolution2D(128,
3,
3,
W_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Convolution2D(256,
3,
3,
W_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
# model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Convolution2D(256,
3,
3,
W_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Convolution2D(512,
3,
3,
W_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
# model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Convolution2D(512,
3,
3,
W_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(
Convolution2D(512,
3,
3,
W_regularizer=l2(weight_decay),
activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# (bs, y, x, c) --> (bs, x, y, c)
model.add(Permute((2, 1, 3)))
# (bs, x, y, c) --> (bs, x, y * c)
bs, x, y, c = model.layers[-1].output_shape
model.add(Reshape((x, y * c)))
model.add(
Bidirectional(LSTM(256, return_sequences=False), merge_mode="concat"))
model.add(Dense(config["num_classes"], activation="softmax"))
return model
def _build_model(self):
max_seq_length = self.sentence_len * self.num_sentences
#Embedding Layer
embedding_layer = Embedding(self.vocab_size,
self.embedding_dim,
input_length=max_seq_length,
trainable=self.train_embedding,
embeddings_regularizer=regularizers.l2(
self.embedding_regularizer_l2),
name='imdb_embedding')
if self.embedding_weights is not None:
embedding_layer = Embedding(self.vocab_size,
self.embedding_dim,
weights=[self.embedding_weights],
input_length=max_seq_length,
trainable=self.train_embedding,
embeddings_regularizer=regularizers.l2(
self.embedding_regularizer_l2),
name='imdb_embedding')
#input layer : sequence of word indices for each sentence
sequence_input = Input(shape=(max_seq_length, ), dtype='int32')
z = embedding_layer(sequence_input)
if self.input_dropout > 0:
z = Dropout(self.input_dropout)(z)
conv_blocks = []
i = 0
#same convolution filters to be used for all sentences.
word_conv_model = Conv1D(filters=self.word_filters,
kernel_size=self.word_kernel_size,
padding="valid",
activation=self.conv_activation,
trainable=self.learn_word_conv,
name="word_conv",
strides=1)
for sent in range(self.num_sentences):
#get once sentence from the input
sentence = Lambda(lambda x: x[:, sent * self.sentence_len:(
sent + 1) * self.sentence_len, :])(z)
conv = word_conv_model(sentence)
conv = KMaxPooling(k=self.sent_k_maxpool)(conv)
#transpose pooled values per sentence
conv = Reshape([self.word_filters * self.sent_k_maxpool, 1])(conv)
conv_blocks.append(conv)
#append all sentence convolution feature maps and make sentence embeddings
z = Concatenate()(
conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
#transform to (steps, input_dim)
z = Permute([2, 1], name='sentence_embeddings')(z)
if self.sent_dropout > 0:
z = Dropout(self.sent_dropout)(z)
sent_conv = Conv1D(filters=self.sent_filters,
kernel_size=self.sent_kernel_size,
padding="valid",
activation=self.conv_activation,
trainable=self.learn_sent_conv,
name='sentence_conv',
strides=1)(z)
z = KMaxPooling(k=self.doc_k_maxpool)(sent_conv)
z = Flatten(name='document_embedding')(z)
if self.hidden_gaussian_noise_sd:
z = GaussianNoise(self.hidden_gaussian_noise_sd)(z)
elif self.hidden_dropout:
z = Dropout(self.hidden_dropout)(z)
for i in range(self.num_hidden_layers):
layer_name = 'hidden_{}'.format(i)
z = Dense(self.hidden_dims,
activation=self.hidden_activation,
name=layer_name,
kernel_regularizer=regularizers.l2(
self.hidden_layer_kernel_regularizer))(z)
output_activation = 'sigmoid'
if self.num_units_final_layer > 1:
output_activation = 'softmax'
model_output = Dense(self.num_units_final_layer,
activation=output_activation,
kernel_regularizer=regularizers.l2(
self.final_layer_kernel_regularizer),
name='final')(z)
self._model = Model(sequence_input, model_output)
def get_model(self):
inputs_cnn = []
inputs_lstm = []
outputs_cnn = []
lstm = []
for i in range(self.window_size):
input_cnn = Input(shape=(self.mesh_rows, self.mesh_columns,
self.depth),
name="input" + str(i + 1))
input_lstm = Input(shape=(self.number_channels, self.depth),
name="input" + str(i + 1 + self.window_size))
inputs_cnn.append(input_cnn)
inputs_lstm.append(input_lstm)
conv1 = self.tfaugmented_conv2d(input_cnn,
self.conv1_filters,
self.conv1_kernel_shape,
dk=self.depth_k,
dv=self.depth_v,
Nh=self.num_heads,
relative=self.relative)
norm1 = BatchNormalization(axis=-1)(conv1)
conv2 = Conv2D(self.conv2_filters,
self.conv2_kernel_shape,
padding=self.padding2,
activation=self.conv2_activation)(norm1)
conv3 = Conv2D(self.conv3_filters,
self.conv3_kernel_shape,
padding=self.padding3,
activation=self.conv3_activation)(conv2)
flat = Flatten()(conv3)
dense = Dense(self.dense_nodes,
activation=self.dense_activation)(flat)
outputs_cnn.append(dense)
permut = Permute((2, 1),
input_shape=(self.number_channels, 1))(input_lstm)
dense = Dense(self.dense_nodes,
activation=self.dense_activation,
input_shape=(1, self.number_channels))(permut)
lstm.append(dense)
merge = concatenate(lstm, axis=1)
lstm1 = LSTM(self.lstm1_cells, return_sequences=True)(merge)
attention_output = self.attention_block(lstm1)
attention_output = tf.expand_dims(attention_output, -1)
lstm2 = LSTM(self.lstm2_cells,
return_sequences=False)(attention_output)
dense3 = Dense(self.dense3_nodes,
activation=self.dense3_activation)(lstm2)
added = Add()([i for i in outputs_cnn])
final = concatenate([dense3, added], axis=-1)
output = Dense(4, activation="softmax")(final)
model = Model(inputs=inputs_cnn + inputs_lstm, outputs=output)
return model
