def _upscale(self, inputs=None, filters=64, kernel_size=3, strides=1, padding='same'):
outputs = Conv2D(filters=filters*4, kernel_size=kernel_size, padding=padding)(inputs)
outputs = LeakyReLU(0.1)(outputs)
outputs = self.pixel_shuffler(outputs)
return outputs
def encdec_VQVAE():
#parameters:
rate = 0.4
dense_size = 1000
embedding_dim = 300
num_embeddings = 180
commitment_cost = 0.25
glove_size = 300
fMRI_size = 65730
reduced_size = 3221
gordon_areas = 333
sizes = np.load(
str(os.path.dirname(os.path.abspath(__file__))) +
'/data/look_ups/sizes.npy')
reduced = np.load(
str(os.path.dirname(os.path.abspath(__file__))) +
'/data/look_ups/reduced_sizes.npy')
index1 = 0
index = 0
# small ROI dense layers: Each ROI region has its own dense layer. The outputs are then concatenated and used in further layers
input_voxel = Input(shape=(fMRI_size, ))
branch_outputs = []
for i in range(gordon_areas):
new_index = index + sizes[i]
small_input = Lambda(lambda x: x[:, index:new_index],
output_shape=(sizes[i], ))(input_voxel)
small_out = Dense(reduced[i])(small_input)
small_out = BatchNormalization()(small_out)
branch_outputs.append(small_out)
index = new_index
dense1 = Concatenate()(branch_outputs)
dense1 = LeakyReLU(alpha=0.3)(dense1)
dense1 = BatchNormalization()(dense1)
dense1 = Dropout(rate=rate)(dense1)
#intermediate Layer: Reduce the output from the ROI small dense layer further.
# The output from this layer is also used for the autoencoder to reconstruct the fMRIs
dense5 = Dense(dense_size)
out_further = dense5(dense1)
out_further = LeakyReLU(alpha=0.3)(out_further)
out_further = BatchNormalization()(out_further)
out_further = Dropout(rate=rate)(out_further)
#VQ-VAE layer and Classification layer: It returns a proability vector for a given fMRI belonging to a certainword out of the possible 180 words.
# These layers hve each one loss: One for the classification and one for the VQ-VAE layer output
out_mid = Dense(glove_size)(out_further)
enc_inputs = out_mid
enc = VQVAELayer(embedding_dim,
num_embeddings,
commitment_cost,
name="vqvae")(enc_inputs)
out_mid = Lambda(
lambda enc: enc_inputs + K.stop_gradient(enc - enc_inputs),
name="encoded")(enc)
out_class = Dense(180, activation='softmax')(out_mid)
dense4 = DenseTranspose(dense5)(out_further)
dense4 = LeakyReLU(alpha=0.3)(dense4)
dense4 = BatchNormalization()(dense4)
dense4 = Dropout(rate=rate)(dense4)
branch_outputs1 = []
for j in range(gordon_areas):
new_index1 = index1 + reduced[j]
small_input = Lambda(lambda x: x[:, index1:new_index1],
output_shape=(reduced[j], ))(dense4)
small_out = Dense(sizes[j])(small_input)
small_out = LeakyReLU(alpha=0.3)(small_out)
small_out = BatchNormalization()(small_out)
branch_outputs1.append(small_out)
index1 = new_index1
out = Concatenate()(branch_outputs1)
pred_glove = Lambda(lambda t: t, name='pred_glove')(out_mid)
fMRI_rec = Lambda(lambda t: t, name='fMRI_rec')(out)
pred_class = Lambda(lambda t: t, name='pred_class')(out_class)
model = Model(inputs=[input_voxel],
outputs=[fMRI_rec, pred_glove, pred_class])
return model, enc, enc_inputs
def autoencoder(trainable, mean):
#parameters:
rate = 0.4
dense_size = 200
glove_size = 300
fMRI_size = 65730
reduced_size = 3221
gordon_areas = 333
sizes = np.load(
str(os.path.dirname(os.path.abspath(__file__))) +
'/data/look_ups/sizes.npy')
reduced = np.load(
str(os.path.dirname(os.path.abspath(__file__))) +
'/data/look_ups/reduced_sizes.npy')
index1 = 0
index = 0
input_voxel = Input(shape=(fMRI_size, ))
# small ROI dense layers: Each ROI region has its own dense layer. The outputs are then concatenated and used in further layers
branch_outputs = []
dense_layers = []
for i in range(gordon_areas):
new_index = index + sizes[i]
small_input = Lambda(lambda x: x[:, index:new_index],
output_shape=(sizes[i], ))(input_voxel)
dense_layers.append(Dense(reduced[i]))
small_out = dense_layers[i](small_input)
small_out = LeakyReLU(alpha=0.3)(small_out)
small_out = BatchNormalization()(small_out)
branch_outputs.append(small_out)
index = new_index
Concat = Concatenate()(branch_outputs)
dense1 = BatchNormalization()(Concat)
dense1 = Dropout(rate=rate)(dense1)
#intermediate Layer: Reduce the output from the ROI small dense layer further.
# The output from this layer is also used for the autoencoder to reconstruct the fMRIs
dense5 = Dense(dense_size)
out_furtherr = dense5(dense1)
out_further = LeakyReLU(alpha=0.3)(out_furtherr)
out_further = BatchNormalization()(out_further)
out_further = Dropout(rate=rate)(out_further)
#Glove layer: The output of this layer should represent the matching glove embeddings for their respective fMRI.
# A loss is only applied if the glove prediction model is run.
dense_glove = Dense(300, trainable=trainable)
out_glove = dense_glove(out_further)
out_gloverr = LeakyReLU(alpha=0.3)(out_glove)
out_gloverr = BatchNormalization()(out_gloverr)
out_gloverr = Dropout(rate=rate)(out_gloverr)
#Classification layer: It returns a proability vector for a given fMRI belonging to a certainword out of the possible 180 words.
# The loss is only calculated if the classification model is run.
out_mid = Dense(180,
activation='softmax',
trainable=trainable,
kernel_regularizer=l2(0.005),
bias_regularizer=l2(0.005))(out_gloverr)
dense4 = DenseTranspose(dense5)(out_further)
dense4 = LeakyReLU(alpha=0.3)(dense4)
dense4 = BatchNormalization()(dense4)
dense4 = Dropout(rate=rate)(dense4)
branch_outputs1 = []
for j in range(gordon_areas):
new_index1 = index1 + reduced[j]
small_input = Lambda(lambda x: x[:, index1:new_index1],
output_shape=(reduced[j], ))(dense4)
small_out = DenseTranspose(dense_layers[j])(small_input)
small_out = LeakyReLU(alpha=0.3)(small_out)
small_out = BatchNormalization()(small_out)
branch_outputs1.append(small_out)
index1 = new_index1
out = Concatenate()(branch_outputs1)
Concat_layer = Lambda(lambda t: t, name='concat')(Concat)
Dense_layer = Lambda(lambda t: t, name='dense_mid')(out_furtherr)
pred_class = Lambda(lambda t: t, name='pred_class')(out_mid)
pred_glove = Lambda(lambda t: t, name='pred_glove')(out_glove)
fMRI_rec = Lambda(lambda t: t, name='fMRI_rec')(out)
if not mean:
model = Model(inputs=[input_voxel],
outputs=[fMRI_rec, pred_glove, pred_class])
else:
model = Model(inputs=[input_voxel],
outputs=[
fMRI_rec, pred_glove, pred_class, Concat_layer,
Dense_layer
])
return model
def __init__(self,
n_attention,
n_attention_hidden,
n_attention_out,
n_feat,
n_concat_hidden,
n_hidden1,
n_hidden2,
activation="sigmoid",
dropout=False,
concat_activity_regularizer=None,
kernel_initializer=VarianceScaling(distribution="uniform"),
kernel_regularizer='l1',
bias_initializer=Zeros(),
bias_regularizer='l1',
attention_initializer=VarianceScaling(distribution="uniform"),
attention_trainable=True,
attention_feat_weight_trainable=True,
**kwargs):
super(AttentionModelwFeatWeights, self).__init__(**kwargs)
self.n_attention = n_attention
self.n_attention_hidden = n_attention_hidden
self.n_attention_out = n_attention_out
self.n_feat = n_feat
self.n_concat_hidden = n_concat_hidden
self.n_hidden1 = n_hidden1
self.n_hidden2 = n_hidden2
self.activation = activations.get(activation)
self.dropout = dropout
self.concat_activity_regularizer = concat_activity_regularizer
self.kernel_initializer = kernel_initializer
self.kernel_regularizer = kernel_regularizer
self.bias_initializer = bias_initializer
self.bias_regularizer = bias_regularizer
self.attention_initializer = attention_initializer
self.attention_trainable = attention_trainable
self.attention_feat_weight_trainable = attention_feat_weight_trainable
self.attentions = ConcatAttentionswFeatWeights(
n_attention=self.n_attention,
n_attention_hidden=self.n_attention_hidden,
n_attention_out=self.n_attention_out,
n_feat=self.n_feat,
n_hidden=self.n_concat_hidden,
concat_activity_regularizer=self.concat_activity_regularizer,
kernel_initializer=self.kernel_initializer,
kernel_regularizer=self.kernel_regularizer,
bias_initializer=self.bias_initializer,
bias_regularizer=self.bias_regularizer,
attention_initializer=self.attention_initializer,
attention_trainable=self.attention_trainable,
attention_feat_weight_trainable=self.
attention_feat_weight_trainable)
self.dense1 = Dense(
n_hidden1,
kernel_initializer=self.kernel_initializer,
kernel_regularizer=self.kernel_regularizer,
bias_initializer=self.bias_initializer,
bias_regularizer=self.bias_regularizer
) #input_shape=(self.n_attention*self.n_attention_out,))
# self.dense2=Dense(n_hidden2,
# activation=self.hidden_activation
# )
self.hidden_activation1 = LeakyReLU()
self.dropout1 = Dropout(0.1)
self.dense2 = Dense(n_hidden2,
kernel_initializer=self.kernel_initializer,
kernel_regularizer=self.kernel_regularizer,
bias_initializer=self.bias_initializer,
bias_regularizer=self.bias_regularizer)
self.hidden_activation2 = LeakyReLU()
self.dropout2 = Dropout(0.2)
self.output_layer = Dense(
1,
activation=self.activation,
# kernel_initializer=self.kernel_initializer,
# kernel_regularizer=self.kernel_regularizer,
# bias_regularizer=self.bias_regularizer
)
def __init__(self, input_size, weights=None):
input_image = Input(shape=(input_size[0], input_size[1], 3))
# the function to implement the orgnization layer (thanks to github.com/allanzelener/YAD2K)
def space_to_depth_x2(x):
return tensorflow.nn.space_to_depth(x, block_size=2)
# Layer 1
x = Conv2D(32, (3, 3),
strides=(1, 1),
padding='same',
name='conv_1',
use_bias=False)(input_image)
x = BatchNormalization(name='norm_1')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 2
x = Conv2D(64, (3, 3),
strides=(1, 1),
padding='same',
name='conv_2',
use_bias=False)(x)
x = BatchNormalization(name='norm_2')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 3
x = Conv2D(128, (3, 3),
strides=(1, 1),
padding='same',
name='conv_3',
use_bias=False)(x)
x = BatchNormalization(name='norm_3')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 4
x = Conv2D(64, (1, 1),
strides=(1, 1),
padding='same',
name='conv_4',
use_bias=False)(x)
x = BatchNormalization(name='norm_4')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 5
x = Conv2D(128, (3, 3),
strides=(1, 1),
padding='same',
name='conv_5',
use_bias=False)(x)
x = BatchNormalization(name='norm_5')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 6
x = Conv2D(256, (3, 3),
strides=(1, 1),
padding='same',
name='conv_6',
use_bias=False)(x)
x = BatchNormalization(name='norm_6')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 7
x = Conv2D(128, (1, 1),
strides=(1, 1),
padding='same',
name='conv_7',
use_bias=False)(x)
x = BatchNormalization(name='norm_7')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 8
x = Conv2D(256, (3, 3),
strides=(1, 1),
padding='same',
name='conv_8',
use_bias=False)(x)
x = BatchNormalization(name='norm_8')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 9
x = Conv2D(512, (3, 3),
strides=(1, 1),
padding='same',
name='conv_9',
use_bias=False)(x)
x = BatchNormalization(name='norm_9')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 10
x = Conv2D(256, (1, 1),
strides=(1, 1),
padding='same',
name='conv_10',
use_bias=False)(x)
x = BatchNormalization(name='norm_10')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 11
x = Conv2D(512, (3, 3),
strides=(1, 1),
padding='same',
name='conv_11',
use_bias=False)(x)
x = BatchNormalization(name='norm_11')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 12
x = Conv2D(256, (1, 1),
strides=(1, 1),
padding='same',
name='conv_12',
use_bias=False)(x)
x = BatchNormalization(name='norm_12')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 13
x = Conv2D(512, (3, 3),
strides=(1, 1),
padding='same',
name='conv_13',
use_bias=False)(x)
x = BatchNormalization(name='norm_13')(x)
x = LeakyReLU(alpha=0.1)(x)
skip_connection = x
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 14
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_14',
use_bias=False)(x)
x = BatchNormalization(name='norm_14')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 15
x = Conv2D(512, (1, 1),
strides=(1, 1),
padding='same',
name='conv_15',
use_bias=False)(x)
x = BatchNormalization(name='norm_15')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 16
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_16',
use_bias=False)(x)
x = BatchNormalization(name='norm_16')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 17
x = Conv2D(512, (1, 1),
strides=(1, 1),
padding='same',
name='conv_17',
use_bias=False)(x)
x = BatchNormalization(name='norm_17')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 18
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_18',
use_bias=False)(x)
x = BatchNormalization(name='norm_18')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 19
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_19',
use_bias=False)(x)
x = BatchNormalization(name='norm_19')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 20
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_20',
use_bias=False)(x)
x = BatchNormalization(name='norm_20')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 21
skip_connection = Conv2D(64, (1, 1),
strides=(1, 1),
padding='same',
name='conv_21',
use_bias=False)(skip_connection)
skip_connection = BatchNormalization(name='norm_21')(skip_connection)
skip_connection = LeakyReLU(alpha=0.1)(skip_connection)
skip_connection = Lambda(space_to_depth_x2)(skip_connection)
x = Concatenate()([skip_connection, x])
# Layer 22
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_22',
use_bias=False)(x)
x = BatchNormalization(name='norm_22')(x)
x = LeakyReLU(alpha=0.1)(x)
self.feature_extractor = Model(input_image, x)
if weights == 'imagenet':
print(
'Imagenet for YOLO backend are not available yet, defaulting to random weights'
)
elif weights == None:
pass
else:
print('Loaded backend weigths: ' + weights)
self.feature_extractor.load_weights(weights)
adj = adj.flatten()
matrices.append(adj)
# Convert lists to np arrays
matrices = np.array(matrices)
days = np.array(days)
print(matrices.shape)
# Define an encoder
i = Input(shape=(160000,))
e = Dense(512)(i)
e = BatchNormalization()(e)
#e = LeakyReLU()(e)
# Encoder level 2
e = Dense(256)(e)
e = BatchNormalization()(e)
e = LeakyReLU()(e)
# Size of latent space
sizeLatentDims = 50
bottleneck = Dense(sizeLatentDims)(e)
# Define decoder
d = Dense(sizeLatentDims)(bottleneck)
d = BatchNormalization()(d)
d = LeakyReLU()(d)
# Decoder level 2
d = Dense(256)(d)
d = BatchNormalization()(d)
#d = LeakyReLU()(d)
def _build(self):
input_dim = self.input_dim
z_dim = self.z_dim
encoder_conv_filters = self.encoder_conv_filters
encoder_conv_kernel_size = self.encoder_conv_kernel_size
encoder_conv_strides = self.encoder_conv_strides
decoder_conv_t_filters = self.decoder_conv_t_filters
decoder_conv_t_kernel_size = self.decoder_conv_t_kernel_size
decoder_conv_t_strides = self.decoder_conv_t_strides
use_batch_norm = self.use_batch_norm
use_dropout = self.use_dropout
### encoder
num_encoder_layers = len(encoder_conv_filters)
encoder_input_layer = Input(shape=input_dim, name='encoder_input')
x = encoder_input_layer
for i in range(num_encoder_layers):
x = Conv2D(filters=encoder_conv_filters[i],
kernel_size=encoder_conv_kernel_size[i],
strides=encoder_conv_strides[i],
padding='same',
name='encoder_conv' + str(i))(x)
x = LeakyReLU()(x)
if use_batch_norm:
x = BatchNormalization()(x)
if use_dropout:
x = Dropout(rate=0.3)(x)
shape_before_flattening = x.shape[
1:] # 1: from (None, 7, 7, 64) to (7, 7, 64)
x = Flatten()(x)
# the most different part compared to vanilla Autoencoder
self.mu = Dense(self.z_dim, name='mu')(x)
self.log_var = Dense(self.z_dim, name='log_var')(x)
def sampling(args):
mu, log_var = args
epsilon = K.random_normal(shape=K.shape(mu), mean=0.0,
stddev=1.0) # normal distribution
return mu + K.exp(
log_var / 2.0) * epsilon # re-parameterization trick
# the latent space
# This layer notebooks a point z in the latent space from the normal distribution defined by mu and log_var.
encoder_output_layer = Lambda(
sampling, name='encoder_output')([self.mu, self.log_var])
self.encoder = Model(encoder_input_layer,
encoder_output_layer,
name='Encoder')
### decoder
num_decoder_layers = len(decoder_conv_t_filters)
decoder_input_layer = Input(shape=z_dim, name='decoder_input')
x = Dense(np.prod(shape_before_flattening))(decoder_input_layer)
x = Reshape(shape_before_flattening)(x)
for i in range(num_decoder_layers):
x = Conv2DTranspose(filters=decoder_conv_t_filters[i],
kernel_size=decoder_conv_t_kernel_size[i],
strides=decoder_conv_t_strides[i],
padding='same',
name='decoder_conv_t' + str(i))(x)
if i < (num_decoder_layers - 1):
x = LeakyReLU()(x)
if use_batch_norm:
x = BatchNormalization()(x)
if use_dropout:
x = Dropout(rate=0.3)(x)
else:
x = Activation('sigmoid')(x)
decoder_output_layer = x
self.decoder = Model(decoder_input_layer,
decoder_output_layer,
name='Decoder')
model_input_layer = encoder_input_layer
model_output_layer = self.decoder(encoder_output_layer)
self.model = Model(model_input_layer,
model_output_layer,
name='Autoencoder')
self._save()
def block(x, filters, dilation_rate, alpha):
x = Conv2D(filters, (3, 3), dilation_rate=dilation_rate, padding='same')(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha)(x)
return x
def __init__(self):
super(UNet, self).__init__()
self.conv1 = Conv2D(64,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv1") #,bias=False) ### in_channel:3
self.lrelu2 = LeakyReLU(alpha=0.2, name="lrelu2")
self.conv2 = Conv2D(128,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv2") #,bias=False) ### in_channel:64
self.bn2 = BatchNormalization(epsilon=1e-05, momentum=0.1,
name="bn2") ### b_in_channel:128
self.lrelu3 = LeakyReLU(alpha=0.2, name="lrelu3")
self.conv3 = Conv2D(256,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv3") #,bias=False) ### in_channel:128
self.bn3 = BatchNormalization(epsilon=1e-05, momentum=0.1,
name="bn3") ### b_in_channel:256
self.lrelu4 = LeakyReLU(alpha=0.2, name="lrelu4")
self.conv4 = Conv2D(512,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv4") #,bias=False) ### in_channel:256
self.bn4 = BatchNormalization(epsilon=1e-05, momentum=0.1,
name="bn4") ### b_in_channel:512
self.lrelu5 = LeakyReLU(alpha=0.2, name="lrelu5")
self.conv5 = Conv2D(512,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv5") #,bias=False) ### in_channel:512
self.bn5 = BatchNormalization(epsilon=1e-05, momentum=0.1,
name="bn5") ### b_in_channel:512
self.lrelu6 = LeakyReLU(alpha=0.2, name="lrelu6")
self.conv6 = Conv2D(512,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv6") #,bias=False) ### in_channel:512
self.bn6 = BatchNormalization(epsilon=1e-05, momentum=0.1,
name="bn6") ### b_in_channel:512
###################
# 最底層
self.lrelu7 = LeakyReLU(alpha=0.2, name="lrelu7")
self.conv7 = Conv2D(512,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv7") #,bias=False) ### in_channel:512
self.relu7t = ReLU(name="relu7t")
self.conv7t = Conv2DTranspose(
512,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv7t") #,bias=False) ### in_channel:512
self.bn7t = BatchNormalization(epsilon=1e-05,
momentum=0.1,
name="bn7t") ### b_in_channel:512
self.concat7 = Concatenate(name="concat7")
###################
self.relu6t = ReLU(name="relu6t")
self.conv6t = Conv2DTranspose(
512,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv6t") #,bias=False) ### in_channel:1024
self.bn6t = BatchNormalization(epsilon=1e-05,
momentum=0.1,
name="bn6t") ### b_in_channel:512
self.concat6 = Concatenate(name="concat6")
self.relu5t = ReLU(name="relu5t")
self.conv5t = Conv2DTranspose(
512,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv5t") #,bias=False) ### in_channel:1024
self.bn5t = BatchNormalization(epsilon=1e-05,
momentum=0.1,
name="bn5t") ### b_in_channel:512
self.concat5 = Concatenate(name="concat5")
self.relu4t = ReLU(name="relu4t")
self.conv4t = Conv2DTranspose(
256,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv4t") #,bias=False) ### in_channel:1024
self.bn4t = BatchNormalization(epsilon=1e-05,
momentum=0.1,
name="bn4t") ### b_in_channel:256
self.concat4 = Concatenate(name="concat4")
self.relu3t = ReLU(name="relu3t")
self.conv3t = Conv2DTranspose(
128,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv3t") #,bias=False) ### in_channel:512
self.bn3t = BatchNormalization(epsilon=1e-05,
momentum=0.1,
name="bn3t") ### b_in_channel:128
self.concat3 = Concatenate(name="concat3")
self.relu2t = ReLU(name="relu2t")
self.conv2t = Conv2DTranspose(
64,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv2t") #,bias=False) ### in_channel:256
self.bn2t = BatchNormalization(epsilon=1e-05,
momentum=0.1,
name="bn2t") ### b_in_channel:64
self.concat2 = Concatenate(name="concat2")
self.relu1t = ReLU(name="relu1t")
self.conv1t = Conv2DTranspose(1,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
name="conv1t") ### in_channel:128
testSetAccAtBestTune = 0
# Read in the images. Only the TEST examples really need to be kept (at the end a web page of testset errors produced).
X_train, y_train, y_onehot_train, img_files_train = load_course_images(directory="{}/trainset".format(IMG_DIR), image_size=imageDimension)
X_tune, y_tune, y_onehot_tune, img_files_tune = load_course_images(directory="{}/tuneset".format( IMG_DIR), image_size=imageDimension)
X_test, y_test, y_onehot_test, img_files_test = load_course_images(directory="{}/testset".format( IMG_DIR), image_size=imageDimension)
print("There are {:,} training examples.".format(len(X_train)))
print("There are {:,} tuning examples.".format( len(X_tune)))
print("There are {:,} testing examples.".format( len(X_test)))
# Define model architecture See https://keras.io/getting-started/sequential-model-guide/
model = Sequential()
#model.add(Dropout(input_dropoutProb)) # Can't specify dropOut for input units? See https://github.com/fchollet/keras/issues/96
leakyReLUtoUse = LeakyReLU(alpha = 0.1)
model.add(Conv2D(platesConv1,
kernel_size = kernelSizeConv1,
input_shape = [imageDimension, imageDimension, numberOfColors],
data_format = "channels_last", # Says that the color channels are LAST.
strides = strideConv1,
padding = "valid", # I'm not sure what this does? Says zero padding is ok????
use_bias = True))
model.add(leakyReLUtoUse); # Have to add as a layer, not as an argument to Conv2D. See https://github.com/fchollet/keras/issues/3380
model.add(ZeroPadding2D(padding = zeroPaddingConv1, data_format = "channels_last"))
model.add(Dropout(conv1_dropoutProb))
model.add(MaxPooling2D(pool_size = kernelSizePool1, strides = stridePool1, padding = 'valid'))
model.add(Dropout(pool1_dropoutProb))
model.add(ZeroPadding2D(padding = zeroPaddingPool1))
import pickle import copy import os import gym state_dim = 2 action_dim = 3 # Input state curr_state = keras.Input(shape=(state_dim,),name="curr_state") curr_action = keras.Input(shape=(action_dim,),name="curr_action") # FDM model curr_state_action = concatenate([curr_state, curr_action]) fdm_h1 = Dense(16,name="dense1_FDM")(curr_state_action) fdm_h1 = LeakyReLU(alpha=0.2,name="LeakyRelu1_FDM")(fdm_h1) fdm_h2 = Dense(16,name="dense2_FDM")(fdm_h1) fdm_h2 = LeakyReLU(alpha=0.2,name="LeakyRelu2_FDM")(fdm_h2) fdm_pred_state = layers.Dense(state_dim,name="dense3_FDM")(fdm_h2) # This model maps an input & action to its next state FDM = keras.Model(inputs=[curr_state,curr_action], outputs=fdm_pred_state,name="FDM") opt_FDM = tf.keras.optimizers.RMSprop(learning_rate=0.0005) FDM.compile(loss='mean_squared_error', optimizer=opt_FDM, metrics=['mse']) # Import Data filename = 'Data/NState.npy'
def __init__(self, opts, *args, **kwargs):
super().__init__(name='netG', *args, **kwargs)
self.opts = opts
opts.num_upsampling_layers = opts.num_upsampling_layers.casefold()
self.sw, self.sh = self._compute_latent_vector_size(
opts.num_upsampling_layers, opts.crop_size, opts.aspect_ratio)
# Build SPADEGenerator
netG = []
netG.append([
Lambda(lambda x: tf.image.resize(
x, (self.sh, self.sw), method=tf.image.ResizeMethod.BILINEAR),
trainable=False),
Conv2D(16 * opts.ngf, kernel_size=3, padding='same', name='fc')
])
opts.injection_layer = opts.injection_layer.casefold()
use_spade = opts.injection_layer in ['all', '1']
netG.append([
SPADEResnetBlock(opts,
16 * opts.ngf,
16 * opts.ngf,
use_spade=use_spade,
use_spectral_norm=True,
name='head_0'),
UpSampling2D(size=2)
])
name = ['G_middle_{:d}'.format(i) for i in range(2)]
use_spade = opts.injection_layer in ['all', '2']
netG.append([
SPADEResnetBlock(opts,
16 * opts.ngf,
16 * opts.ngf,
use_spade=use_spade,
use_spectral_norm=True,
name=name[0]),
UpSampling2D(size=2) if opts.num_upsampling_layers
in ['more', 'most'] else Lambda(lambda x: x, trainable=False)
])
netG.append([
SPADEResnetBlock(opts,
16 * opts.ngf,
16 * opts.ngf,
use_spade=use_spade,
use_spectral_norm=True,
name=name[1]),
UpSampling2D(size=2)
])
use_spade = opts.injection_layer in ['all', '3']
netG.append([
SPADEResnetBlock(opts,
16 * opts.ngf,
8 * opts.ngf,
use_spade=use_spade,
use_spectral_norm=True,
name='up_0'),
UpSampling2D(size=2)
])
use_spade = opts.injection_layer in ['all', '4']
netG.append([
SPADEResnetBlock(opts,
8 * opts.ngf,
4 * opts.ngf,
use_spade=use_spade,
use_spectral_norm=True,
name='up_1'),
UpSampling2D(size=2)
])
use_spade = opts.injection_layer in ['all', '5']
netG.append([
SPADEResnetBlock(opts,
4 * opts.ngf,
2 * opts.ngf,
use_spade=use_spade,
use_spectral_norm=True,
name='up_2'),
UpSampling2D(size=2)
])
use_spade = opts.injection_layer in ['all', '6']
netG.append([
SPADEResnetBlock(opts,
2 * opts.ngf,
1 * opts.ngf,
use_spade=use_spade,
use_spectral_norm=True,
name='up_3')
])
if opts.num_upsampling_layers == 'most':
netG.append([
UpSampling2D(size=2),
SPADEResnetBlock(opts,
1 * opts.ngf,
opts.ngf // 2,
use_spade=True,
use_spectral_norm=True,
name='up_4')
])
final_nc = opts.ngf // 2
else:
final_nc = opts.ngf
netG.append([
LeakyReLU(alpha=2e-1),
Conv2D(3, kernel_size=3, padding='same', name='conv_img'),
Lambda(lambda x: tanh(x), trainable=False)
])
self.inner_layers = netG
action_dim = env.action_space.n
reward_writer = open('log_files/'+trial_no+"/Reward.txt", "w") # Log episode result
########################################################
pause = 0
while pause == 1:
pause = 1
# This model maps an input to its next state
# Input state
AE_state = keras.Input(shape=(state_dim,),name="AE_state")
# 2layer neural network to predict the next state
encoded = Dense(32,name="dense1_NS")(AE_state)
encoded = LeakyReLU(alpha=0.2,name="LeakyRelu1_NS")(encoded)
encoded = Dense(32,name="dense2_NS")(encoded)
encoded = LeakyReLU(alpha=0.2,name="LeakyRelu2_NS")(encoded)
n_state = layers.Dense(state_dim,name="dense3_NS")(encoded)
AE = keras.Model(inputs=AE_state, outputs=n_state,name="AE")
#print(AE.summary())
#tf.keras.utils.plot_model(AE, to_file='AE_model_plot.png', show_shapes=True, show_layer_names=True)
opt_AE = tf.keras.optimizers.RMSprop(learning_rate=0.00015)
AE.compile(loss='mean_squared_error', optimizer=opt_AE, metrics=['mse'])
def unet3D(
x_in,
img_shape,
out_im_chans,
nf_enc=[64, 64, 128, 128, 256, 256, 512],
nf_dec=None,
layer_prefix='unet',
n_convs_per_stage=1,
):
ks = 3
x = x_in
encodings = []
encoding_vol_sizes = []
for i in range(len(nf_enc)):
for j in range(n_convs_per_stage):
x = Conv3D(nf_enc[i],
kernel_size=ks,
strides=(1, 1, 1),
padding='same',
name='{}_enc_conv3D_{}_{}'.format(
layer_prefix, i, j + 1))(x)
x = LeakyReLU(0.2)(x)
encodings.append(x)
encoding_vol_sizes.append(np.asarray(x.get_shape().as_list()[1:-1]))
if i < len(nf_enc) - 1:
x = MaxPooling3D(pool_size=(2, 2, 2),
padding='same',
name='{}_enc_maxpool_{}'.format(layer_prefix,
i))(x)
if nf_dec is None:
nf_dec = list(reversed(nf_enc[1:]))
for i in range(len(nf_dec)):
curr_shape = x.get_shape().as_list()[1:-1]
# only do upsample if we are not yet at max resolution
if np.any(curr_shape < list(img_shape[:len(curr_shape)])):
us = (2, 2, 2)
x = UpSampling3D(size=us,
name='{}_dec_upsamp_{}'.format(layer_prefix,
i))(x)
# just concatenate the final layer here
if i <= len(encodings) - 2:
x = _pad_or_crop_to_shape_3D(
x, np.asarray(x.get_shape().as_list()[1:-1]),
encoding_vol_sizes[-i - 2])
x = Concatenate(axis=-1)([x, encodings[-i - 2]])
for j in range(n_convs_per_stage):
x = Conv3D(nf_dec[i],
kernel_size=ks,
strides=(1, 1, 1),
padding='same',
name='{}_dec_conv3D_{}_{}'.format(layer_prefix, i,
j))(x)
x = LeakyReLU(0.2)(x)
y = Conv3D(out_im_chans,
kernel_size=1,
padding='same',
name='{}_dec_conv3D_final'.format(layer_prefix))(
x) # add your own activation after this model
# add your own activation after this model
return y
def __init__(
self,
num_filters,
reduce_filters,
#clip_constraint,
upsample=True,
is_end=True,
**kwargs):
super(gen_block, self).__init__(**kwargs)
# bool to remove upsamples where neccesary
self.upsample = upsample
# on creation it will be end.
self.is_end = is_end
# after 32x32 must start reducing filter size
if reduce_filters:
self.num_filters = int(num_filters / 2)
else:
self.num_filters = num_filters
self.upspl1 = UpSampling2D()
self.conv1 = Conv2DEQ(
filters=self.num_filters,
kernel_size=(3, 3),
padding="same",
#kernel_constraint=clip_constraint
)
self.act1 = LeakyReLU(alpha=0.2)
self.pixel_w_norm1 = PixelNormalization()
self.conv2 = Conv2DEQ(
filters=self.num_filters,
kernel_size=(3, 3),
padding="same",
#kernel_constraint=clip_constraint
)
self.act2 = LeakyReLU(alpha=0.2)
self.pixel_w_norm2 = PixelNormalization()
# for if last
self.conv_last1 = Conv2DEQ(
filters=16,
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal',
#kernel_constraint=clip_constraint
)
self.act_last1 = LeakyReLU(alpha=0.2)
self.conv_last2 = Conv2DEQ(
filters=16,
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal',
#kernel_constraint=clip_constraint
)
self.act_last2 = LeakyReLU(alpha=0.2)
self.RGB_out = Conv2DEQ(
filters=3,
kernel_size=(1, 1),
padding='same',
kernel_initializer='he_normal',
#kernel_constraint=clip_constraint
)
def __init__(self, model_architecture, num_inputs,
output_activation = 'sigmoid', cost_function = 'binary_crossentropy',
num_epochs = 10, batch_size = None,
default_adam = True, optimizer = None, opt_params = None,
regularization = 'l2', regul_param = 0, input_dropout = 0,
weights_init = 'glorot_uniform', bias_init = 'zeros'):
self.model_architecture = model_architecture
self.num_inputs = num_inputs
self.output_activation = output_activation
self.cost_function = cost_function
self.num_epochs = num_epochs
self.batch_size = batch_size
self.default_adam = default_adam
self.optimizer = optimizer
self.opt_params = opt_params
self.regularization = regularization
self.regul_param = regul_param
self.input_dropout = input_dropout
self.weights_init = weights_init
self.bias_init = bias_init
# Declaring the model object:
self.model = Sequential()
# Dropout for the input layer:
self.model.add(Dropout(input_dropout, input_shape=(self.num_inputs,)))
# Hidden layers with dropout:
for i in self.model_architecture.keys():
if self.model_architecture[i]['activation'] == 'leaky_relu':
self.model.add(Dense(units = self.model_architecture[i]['neurons'],
kernel_regularizer = eval('{0}(l = self.regul_param)'.format(self.regularization)),
kernel_initializer = self.weights_init,
bias_initializer = self.bias_init))
self.model.add(LeakyReLU(alpha = 0.01))
elif self.model_architecture[i]['activation'] == 'prelu':
self.model.add(Dense(units = self.model_architecture[i]['neurons'],
kernel_regularizer = eval('{0}(l = self.regul_param)'.format(self.regularization)),
kernel_initializer = self.weights_init,
bias_initializer = self.bias_init))
self.model.add(PReLU(alpha_initializer = "zeros",
alpha_regularizer = None, alpha_constraint = None, shared_axes = None))
elif self.model_architecture[i]['activation'] in ['swish']:
self.model.add(Dense(units = self.model_architecture[i]['neurons'],
activation = eval(self.model_architecture[i]['activation']),
kernel_regularizer = eval('{0}(l = self.regul_param)'.format(self.regularization)),
kernel_initializer = self.weights_init,
bias_initializer = self.bias_init))
else:
self.model.add(Dense(units = self.model_architecture[i]['neurons'],
activation = self.model_architecture[i]['activation'],
kernel_regularizer = eval('{0}(l = self.regul_param)'.format(self.regularization)),
kernel_initializer = self.weights_init,
bias_initializer = self.bias_init))
self.model.add(Dropout(rate = self.model_architecture[i]['dropout_param']))
# Final layer with one neuron:
self.model.add(Dense(units = 1, activation = self.output_activation))
# Compiling the model to prepare it to be fitted:
if self.default_adam:
self.model.compile(loss = self.cost_function, optimizer = 'adam')
else:
if self.optimizer == 'sgd':
opt = SGD(learning_rate = self.opt_params['learning_rate'], momentum = self.opt_params['momentum'],
decay = self.opt_params['decay'])
elif self.optimizer == 'adam':
opt = Adam(learning_rate = self.opt_params['learning_rate'], beta_1 = self.opt_params['beta_1'],
beta_2 = self.opt_params['beta_2'], epsilon = self.opt_params['epsilon'])
self.model.compile(loss = self.cost_function, optimizer = opt)
def activation():
return LeakyReLU(alpha=lrelu_factor)
def get_test_model_exhaustive():
"""Returns a exhaustive test model."""
input_shapes = [(2, 3, 4, 5, 6), (2, 3, 4, 5, 6), (7, 8, 9, 10),
(7, 8, 9, 10), (11, 12, 13), (11, 12, 13), (14, 15),
(14, 15), (16, ),
(16, ), (2, ), (1, ), (2, ), (1, ), (1, 3), (1, 4),
(1, 1, 3), (1, 1, 4), (1, 1, 1, 3), (1, 1, 1, 4),
(1, 1, 1, 1, 3), (1, 1, 1, 1, 4), (26, 28, 3), (4, 4, 3),
(4, 4, 3), (4, ), (2, 3), (1, ), (1, ), (1, ), (2, 3),
(9, 16, 1), (1, 9, 16)]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Conv1D(1, 3, padding='valid')(inputs[6]))
outputs.append(Conv1D(2, 1, padding='same')(inputs[6]))
outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6]))
outputs.append(ZeroPadding1D(2)(inputs[6]))
outputs.append(Cropping1D((2, 3))(inputs[6]))
outputs.append(MaxPooling1D(2)(inputs[6]))
outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(AveragePooling1D(2)(inputs[6]))
outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(GlobalMaxPooling1D()(inputs[6]))
outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6]))
outputs.append(GlobalAveragePooling1D()(inputs[6]))
outputs.append(
GlobalAveragePooling1D(data_format="channels_first")(inputs[6]))
outputs.append(Conv2D(4, (3, 3))(inputs[4]))
outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4]))
outputs.append(SeparableConv2D(3, (3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((1, 2))(inputs[4]))
outputs.append(MaxPooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
#outputs.append(MaxPooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default MaxPoolingOp only supports NHWC on device type CPU
outputs.append(
MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(AveragePooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
#outputs.append(AveragePooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default AvgPoolingOp only supports NHWC on device type CPU
outputs.append(
AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(GlobalAveragePooling2D()(inputs[4]))
outputs.append(
GlobalAveragePooling2D(data_format="channels_first")(inputs[4]))
outputs.append(GlobalMaxPooling2D()(inputs[4]))
outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4]))
outputs.append(Permute((3, 4, 1, 5, 2))(inputs[0]))
outputs.append(Permute((1, 5, 3, 2, 4))(inputs[0]))
outputs.append(Permute((3, 4, 1, 2))(inputs[2]))
outputs.append(Permute((2, 1, 3))(inputs[4]))
outputs.append(Permute((2, 1))(inputs[6]))
outputs.append(Permute((1, ))(inputs[8]))
outputs.append(Permute((3, 1, 2))(inputs[31]))
outputs.append(Permute((3, 1, 2))(inputs[32]))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[31])))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[32])))
outputs.append(BatchNormalization()(inputs[0]))
outputs.append(BatchNormalization(axis=1)(inputs[0]))
outputs.append(BatchNormalization(axis=2)(inputs[0]))
outputs.append(BatchNormalization(axis=3)(inputs[0]))
outputs.append(BatchNormalization(axis=4)(inputs[0]))
outputs.append(BatchNormalization(axis=5)(inputs[0]))
outputs.append(BatchNormalization()(inputs[2]))
outputs.append(BatchNormalization(axis=1)(inputs[2]))
outputs.append(BatchNormalization(axis=2)(inputs[2]))
outputs.append(BatchNormalization(axis=3)(inputs[2]))
outputs.append(BatchNormalization(axis=4)(inputs[2]))
outputs.append(BatchNormalization()(inputs[4]))
# todo: check if TensorFlow >= 2.1 supports this
#outputs.append(BatchNormalization(axis=1)(inputs[4])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[4]))
outputs.append(BatchNormalization(axis=3)(inputs[4]))
outputs.append(BatchNormalization()(inputs[6]))
outputs.append(BatchNormalization(axis=1)(inputs[6]))
outputs.append(BatchNormalization(axis=2)(inputs[6]))
outputs.append(BatchNormalization()(inputs[8]))
outputs.append(BatchNormalization(axis=1)(inputs[8]))
outputs.append(BatchNormalization()(inputs[27]))
outputs.append(BatchNormalization(axis=1)(inputs[27]))
outputs.append(BatchNormalization()(inputs[14]))
outputs.append(BatchNormalization(axis=1)(inputs[14]))
outputs.append(BatchNormalization(axis=2)(inputs[14]))
outputs.append(BatchNormalization()(inputs[16]))
# todo: check if TensorFlow >= 2.1 supports this
#outputs.append(BatchNormalization(axis=1)(inputs[16])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[16]))
outputs.append(BatchNormalization(axis=3)(inputs[16]))
outputs.append(BatchNormalization()(inputs[18]))
outputs.append(BatchNormalization(axis=1)(inputs[18]))
outputs.append(BatchNormalization(axis=2)(inputs[18]))
outputs.append(BatchNormalization(axis=3)(inputs[18]))
outputs.append(BatchNormalization(axis=4)(inputs[18]))
outputs.append(BatchNormalization()(inputs[20]))
outputs.append(BatchNormalization(axis=1)(inputs[20]))
outputs.append(BatchNormalization(axis=2)(inputs[20]))
outputs.append(BatchNormalization(axis=3)(inputs[20]))
outputs.append(BatchNormalization(axis=4)(inputs[20]))
outputs.append(BatchNormalization(axis=5)(inputs[20]))
outputs.append(Dropout(0.5)(inputs[4]))
outputs.append(ZeroPadding2D(2)(inputs[4]))
outputs.append(ZeroPadding2D((2, 3))(inputs[4]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Cropping2D(2)(inputs[4]))
outputs.append(Cropping2D((2, 3))(inputs[4]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Dense(3, use_bias=True)(inputs[13]))
outputs.append(Dense(3, use_bias=True)(inputs[14]))
outputs.append(Dense(4, use_bias=False)(inputs[16]))
outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18]))
outputs.append(Dense(4, use_bias=False)(inputs[20]))
outputs.append(Reshape(((2 * 3 * 4 * 5 * 6), ))(inputs[0]))
outputs.append(Reshape((2, 3 * 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5, 6))(inputs[0]))
outputs.append(Reshape((16, ))(inputs[8]))
outputs.append(Reshape((2, 8))(inputs[8]))
outputs.append(Reshape((2, 2, 4))(inputs[8]))
outputs.append(Reshape((2, 2, 2, 2))(inputs[8]))
outputs.append(Reshape((2, 2, 1, 2, 2))(inputs[8]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4]))
for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]:
outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]]))
for axis in [-4, -3, -2, -1, 1, 2, 3, 4]:
outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]]))
for axis in [-3, -2, -1, 1, 2, 3]:
outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]]))
for axis in [-2, -1, 1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]]))
for axis in [-1, 1]:
outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]]))
for axis in [-1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]]))
for axis in [-1, 3]:
outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]]))
for axis in [-1, 4]:
outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]]))
for axis in [-1, 5]:
outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]]))
outputs.append(UpSampling1D(size=2)(inputs[6]))
# outputs.append(UpSampling1D(size=2)(inputs[8])) # ValueError: Input 0 of layer up_sampling1d_1 is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 16]
outputs.append(Multiply()([inputs[10], inputs[11]]))
outputs.append(Multiply()([inputs[11], inputs[10]]))
outputs.append(Multiply()([inputs[11], inputs[13]]))
outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]]))
outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1),
padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(Add()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Subtract()([inputs[26], inputs[30]]))
outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Average()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5)(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[25]),
Activation('hard_sigmoid')(inputs[25]),
Activation('selu')(inputs[25]),
Activation('sigmoid')(inputs[25]),
Activation('softplus')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('relu')(inputs[25]),
LeakyReLU()(inputs[25]),
ELU()(inputs[25]),
PReLU()(inputs[24]),
PReLU()(inputs[25]),
PReLU()(inputs[26]),
shared_activation(inputs[25]),
Activation('linear')(inputs[26]),
Activation('linear')(inputs[23]),
x,
shared_activation(x),
]
model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 2
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
width = video.shape[2] height = video.shape[1] video_size = len(video) train_gen = data_generator(video[:int(video_size*split)], speeds[:int(video_size*split)], batch_size, sequence_length) val_gen = data_generator(video[int(video_size*split):], speeds[int(video_size*split):], batch_size, sequence_length) pred_gen = prediction_generator(video, sequence_length) # Will return a feature and label set. # Features are a list of image sequences in the form: (sequence_length, img_height, img_width, dimensions) inputs = Input((sequence_length,height,width,3)) # A convolution being applied to each image seperately x = Conv3D(32,(1,3,3),strides=(1,2,2),activation=None)(inputs) x = LeakyReLU(alpha=0.1)(x) x = BatchNormalization()(x) x = Conv3D(32,(3,3,3),strides=(2,2,2),activation=None)(x) x = LeakyReLU(alpha=0.1)(x) x = BatchNormalization()(x) x = Conv3D(32,(3,3,3),strides=(2,2,2),activation=None)(x) x = LeakyReLU(alpha=0.1)(x) x = BatchNormalization()(x) x = Flatten()(x) x = Dropout(.5)(x) x = Dense(32,activation=None)(x) x = LeakyReLU(alpha=0.1)(x) x = Dense(16,activation=None)(x) x = LeakyReLU(alpha=0.1)(x) outputs = Dense(1,activation=None)(x)
(x_train, y_train), (x_test,
y_test) = tfutils.datasets.mnist.load_data(one_hot=False)
x_train = tfutils.datasets.mnist.load_subset([0], x_train, y_train)
x_test = tfutils.datasets.mnist.load_subset([0], x_test, y_test)
x = np.concatenate([x_train, x_test], axis=0)
# %%
tfutils.datasets.mnist.plot_ten_random_examples(plt, x,
np.zeros(
(x.shape[0], 1))).show()
# %%
size = 28
noise_dim = 1
discriminator = Sequential([
Conv2D(64, 3, strides=2, input_shape=(28, 28, 1)),
LeakyReLU(),
BatchNormalization(),
Conv2D(128, 5, strides=2),
LeakyReLU(),
BatchNormalization(),
Conv2D(256, 5, strides=2),
LeakyReLU(),
BatchNormalization(),
Flatten(),
Dense(1, activation='sigmoid')
])
opt = tf.keras.optimizers.Adam(lr=2e-4, beta_1=0.5)
discriminator.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
discriminator.summary()
def cifar10():
I = Input(shape=[H, W, 3])
C1 = Conv2D(64, (7, 7),
padding='same',
kernel_initializer=init,
use_bias=True)(I)
L1 = LeakyReLU()(C1)
C2 = Conv2D(64, (3, 3),
padding='same',
kernel_initializer=init,
use_bias=True)(L1)
B2 = BatchNormalization()(C2)
L2 = LeakyReLU()(B2)
C3 = Conv2D(64, (3, 3),
padding='same',
kernel_initializer=init,
use_bias=True)(L2)
B3 = BatchNormalization()(C3)
A3 = Add()([L1, B3])
L3 = LeakyReLU()(A3)
D3 = Dropout(0.5)(L3)
C4 = Conv2D(64, (3, 3),
strides=2,
padding='same',
kernel_initializer=init,
use_bias=True)(D3)
B4 = BatchNormalization()(C4)
L4 = LeakyReLU()(B4)
C5 = Conv2D(64, (3, 3),
padding='same',
kernel_initializer=init,
use_bias=True)(L4)
B5 = BatchNormalization()(C5)
L5 = LeakyReLU()(B5)
U6 = Conv2D(64, (3, 3),
padding='same',
kernel_initializer=init,
use_bias=True)(L5)
B6 = BatchNormalization()(U6)
A6 = Add()([L4, B6])
L6 = LeakyReLU()(A6)
D6 = Dropout(0.5)(L6)
U7 = Conv2D(128, (3, 3),
strides=2,
padding='same',
kernel_initializer=init,
use_bias=True)(D6)
B7 = BatchNormalization()(U7)
L7 = LeakyReLU()(B7)
U8 = Conv2D(128, (3, 3),
padding='same',
kernel_initializer=init,
use_bias=True)(L7)
B8 = BatchNormalization()(U8)
L8 = LeakyReLU()(B8)
U9 = Conv2D(128, (3, 3),
padding='same',
kernel_initializer=init,
use_bias=True)(L8)
B9 = BatchNormalization()(U9)
A9 = Add()([L7, B9])
L9 = LeakyReLU()(A9)
D9 = Dropout(0.5)(L9)
U10 = Conv2D(128, (3, 3),
strides=2,
padding='same',
kernel_initializer=init,
use_bias=True)(D9)
B10 = BatchNormalization()(U10)
L10 = LeakyReLU()(B10)
U11 = Conv2D(128, (3, 3),
padding='same',
kernel_initializer=init,
use_bias=True)(L10)
B11 = BatchNormalization()(U11)
L11 = LeakyReLU()(B11)
U12 = Conv2D(128, (3, 3),
padding='same',
kernel_initializer=init,
use_bias=True)(L11)
B12 = BatchNormalization()(U12)
A12 = Add()([L10, B12])
L12 = LeakyReLU()(A12)
D12 = Dropout(0.5)(A12)
U13 = Conv2D(256, (3, 3),
padding='same',
kernel_initializer=init,
use_bias=True)(A12)
B13 = BatchNormalization()(U13)
L13 = LeakyReLU()(B13)
F14 = Flatten()(L13)
DE14 = Dense(256)(F14)
D14 = Dropout(0.5)(DE14)
DE15 = Dense(128)(D14)
D15 = Dropout(0.5)(DE15)
DE16 = Dense(64)(D15)
D16 = Dropout(0.5)(DE16)
out = Dense(10, activation='softmax')(D16)
model = Model(inputs=I, outputs=out)
return model
def segsrgan_discriminator_block(name: str, shape: tuple, kernel: int):
# In:
inputs = Input(shape=(2, shape[0], shape[1], shape[2]), name='dis_input')
# Input 64
disnet = Conv3D(kernel * 1,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_1')(inputs)
disnet = LeakyReLU(0.01)(disnet)
# Hidden 1 : 32
disnet = Conv3D(kernel * 2,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_2')(disnet)
disnet = LeakyReLU(0.01)(disnet)
# Hidden 2 : 16
disnet = Conv3D(kernel * 4,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_3')(disnet)
disnet = LeakyReLU(0.01)(disnet)
# Hidden 3 : 8
disnet = Conv3D(kernel * 8,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_4')(disnet)
disnet = LeakyReLU(0.01)(disnet)
# Hidden 4 : 4
disnet = Conv3D(kernel * 16,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_5')(disnet)
disnet = LeakyReLU(0.01)(disnet)
# Decision : 2
decision = Conv3D(1,
2,
strides=1,
use_bias=False,
kernel_initializer='he_normal',
data_format='channels_first',
name='dis_decision')(disnet)
decision = Reshape((1, ))(decision)
model = Model(inputs=[inputs], outputs=[decision], name=name)
return model
def call(self, feature_dict, training=False, backward=False):
"""Run the model."""
context = None
flow = None
flow_up = None
context_up = None
flows = []
if backward:
feature_pyramid1 = feature_dict['features2']
feature_pyramid2 = feature_dict['features1']
else:
feature_pyramid1 = feature_dict['features1']
feature_pyramid2 = feature_dict['features2']
# Go top down through the levels to the second to last one to estimate flow.
for level, (features1, features2) in reversed(
list(enumerate(
zip(feature_pyramid1,
feature_pyramid2)))[self._output_flow_at_level:]):
# init flows with zeros for coarsest level if needed
if self._shared_flow_decoder and flow_up is None:
batch_size, height, width, _ = features1.shape.as_list()
flow_up = tf.zeros([batch_size, height, width, 2])
if self._num_context_up_channels:
num_channels = int(self._num_context_up_channels *
self._channel_multiplier)
context_up = tf.zeros(
[batch_size, height, width, num_channels])
# Warp features2 with upsampled flow from higher level.
if flow_up is None or not self._use_feature_warp:
warped2 = features2
else:
warp_up = smurf_utils.flow_to_warp(flow_up)
warped2 = smurf_utils.resample(features2, warp_up)
# Compute cost volume by comparing features1 and warped features2.
features1_normalized, warped2_normalized = normalize_features(
[features1, warped2],
normalize=self._normalize_before_cost_volume,
center=self._normalize_before_cost_volume,
moments_across_channels=True,
moments_across_images=True)
if self._use_cost_volume:
cost_volume = compute_cost_volume(features1_normalized,
warped2_normalized,
max_displacement=4)
else:
concat_features = Concatenate(axis=-1)(
[features1_normalized, warped2_normalized])
cost_volume = self._cost_volume_surrogate_convs[level](
concat_features)
cost_volume = LeakyReLU(alpha=self._leaky_relu_alpha)(cost_volume)
if self._shared_flow_decoder:
# this will ensure to work for arbitrary feature sizes per level
conv_1x1 = self._1x1_shared_decoder[level]
features1 = conv_1x1(features1)
# Compute context and flow from previous flow, cost volume, and features1.
if flow_up is None:
x_in = Concatenate(axis=-1)([cost_volume, features1])
else:
if context_up is None:
x_in = Concatenate(axis=-1)(
[flow_up, cost_volume, features1])
else:
x_in = Concatenate(axis=-1)(
[context_up, flow_up, cost_volume, features1])
# Use dense-net connections.
x_out = None
if self._shared_flow_decoder:
# reuse the same flow decoder on all levels
flow_layers = self._flow_layers
else:
flow_layers = self._flow_layers[level]
for layer in flow_layers[:-1]:
x_out = layer(x_in)
x_in = Concatenate(axis=-1)([x_in, x_out])
context = x_out
flow = flow_layers[-1](context)
if (training and self._drop_out_rate):
maybe_dropout = tf.cast(
tf.math.greater(tf.random.uniform([]),
self._drop_out_rate), tf.float32)
context *= maybe_dropout
flow *= maybe_dropout
if flow_up is not None and self._accumulate_flow:
flow += flow_up
# Upsample flow for the next lower level.
flow_up = upsample(flow, is_flow=True)
if self._num_context_up_channels:
context_up = self._context_up_layers[level](context)
# Append results to list.
flows.insert(0, flow)
# Refine flow at level '_output_flow_at_level'.
refinement = self._refine_model(context, flow)
if (training and self._drop_out_rate):
refinement *= tf.cast(
tf.math.greater(tf.random.uniform([]), self._drop_out_rate),
tf.float32)
refined_flow = flow + refinement
flows[0] = refined_flow
# Upsample flow to the highest available feature resolution.
for _ in range(self._output_flow_at_level):
upsampled_flow = upsample(flows[0], is_flow=True)
flows.insert(0, upsampled_flow)
# Upsample flow to the original input resolution.
upsampled_flow = upsample(flows[0], is_flow=True)
flows.insert(0, upsampled_flow)
return [tf.cast(flow, tf.float32) for flow in flows]
def upsampling_transpose_block(model, kernel_size, filters, strides):
model = Conv2DTranspose(filters = filters, kernel_size = kernel_size, strides = strides, padding = "same")(model)
model = LeakyReLU(alpha = 0.2)(model)
return model
def __init__(self, weight_url=None):
if weight_url:
self.model = load_model(filepath=weight_url,
custom_objects={
'dice':
dice,
'iou':
iou,
'loss_function':
weighted_loss(cce_iou_dice,
weights_list)
})
else:
model_input = Input((512, 512, 512))
x00 = conv2d(filters=int(16 * number_of_filters))(model_input)
x00 = BatchNormalization()(x00)
x00 = LeakyReLU(0.01)(x00)
x00 = Dropout(0.2)(x00)
x00 = conv2d(filters=int(16 * number_of_filters))(x00)
x00 = BatchNormalization()(x00)
x00 = LeakyReLU(0.01)(x00)
x00 = Dropout(0.2)(x00)
p0 = MaxPooling2D(pool_size=(2, 2))(x00)
x10 = conv2d(filters=int(32 * number_of_filters))(p0)
x10 = BatchNormalization()(x10)
x10 = LeakyReLU(0.01)(x10)
x10 = Dropout(0.2)(x10)
x10 = conv2d(filters=int(32 * number_of_filters))(x10)
x10 = BatchNormalization()(x10)
x10 = LeakyReLU(0.01)(x10)
x10 = Dropout(0.2)(x10)
p1 = MaxPooling2D(pool_size=(2, 2))(x10)
x01 = conv2dtranspose(int(16 * number_of_filters))(x10)
x01 = concatenate([x00, x01])
x01 = conv2d(filters=int(16 * number_of_filters))(x01)
x01 = BatchNormalization()(x01)
x01 = LeakyReLU(0.01)(x01)
x01 = conv2d(filters=int(16 * number_of_filters))(x01)
x01 = BatchNormalization()(x01)
x01 = LeakyReLU(0.01)(x01)
x01 = Dropout(0.2)(x01)
x20 = conv2d(filters=int(64 * number_of_filters))(p1)
x20 = BatchNormalization()(x20)
x20 = LeakyReLU(0.01)(x20)
x20 = Dropout(0.2)(x20)
x20 = conv2d(filters=int(64 * number_of_filters))(x20)
x20 = BatchNormalization()(x20)
x20 = LeakyReLU(0.01)(x20)
x20 = Dropout(0.2)(x20)
p2 = MaxPooling2D(pool_size=(2, 2))(x20)
x11 = conv2dtranspose(int(16 * number_of_filters))(x20)
x11 = concatenate([x10, x11])
x11 = conv2d(filters=int(16 * number_of_filters))(x11)
x11 = BatchNormalization()(x11)
x11 = LeakyReLU(0.01)(x11)
x11 = conv2d(filters=int(16 * number_of_filters))(x11)
x11 = BatchNormalization()(x11)
x11 = LeakyReLU(0.01)(x11)
x11 = Dropout(0.2)(x11)
x02 = conv2dtranspose(int(16 * number_of_filters))(x11)
x02 = concatenate([x00, x01, x02])
x02 = conv2d(filters=int(16 * number_of_filters))(x02)
x02 = BatchNormalization()(x02)
x02 = LeakyReLU(0.01)(x02)
x02 = conv2d(filters=int(16 * number_of_filters))(x02)
x02 = BatchNormalization()(x02)
x02 = LeakyReLU(0.01)(x02)
x02 = Dropout(0.2)(x02)
x30 = conv2d(filters=int(128 * number_of_filters))(p2)
x30 = BatchNormalization()(x30)
x30 = LeakyReLU(0.01)(x30)
x30 = Dropout(0.2)(x30)
x30 = conv2d(filters=int(128 * number_of_filters))(x30)
x30 = BatchNormalization()(x30)
x30 = LeakyReLU(0.01)(x30)
x30 = Dropout(0.2)(x30)
p3 = MaxPooling2D(pool_size=(2, 2))(x30)
x21 = conv2dtranspose(int(16 * number_of_filters))(x30)
x21 = concatenate([x20, x21])
x21 = conv2d(filters=int(16 * number_of_filters))(x21)
x21 = BatchNormalization()(x21)
x21 = LeakyReLU(0.01)(x21)
x21 = conv2d(filters=int(16 * number_of_filters))(x21)
x21 = BatchNormalization()(x21)
x21 = LeakyReLU(0.01)(x21)
x21 = Dropout(0.2)(x21)
x12 = conv2dtranspose(int(16 * number_of_filters))(x21)
x12 = concatenate([x10, x11, x12])
x12 = conv2d(filters=int(16 * number_of_filters))(x12)
x12 = BatchNormalization()(x12)
x12 = LeakyReLU(0.01)(x12)
x12 = conv2d(filters=int(16 * number_of_filters))(x12)
x12 = BatchNormalization()(x12)
x12 = LeakyReLU(0.01)(x12)
x12 = Dropout(0.2)(x12)
x03 = conv2dtranspose(int(16 * number_of_filters))(x12)
x03 = concatenate([x00, x01, x02, x03])
x03 = conv2d(filters=int(16 * number_of_filters))(x03)
x03 = BatchNormalization()(x03)
x03 = LeakyReLU(0.01)(x03)
x03 = conv2d(filters=int(16 * number_of_filters))(x03)
x03 = BatchNormalization()(x03)
x03 = LeakyReLU(0.01)(x03)
x03 = Dropout(0.2)(x03)
m = conv2d(filters=int(256 * number_of_filters))(p3)
m = BatchNormalization()(m)
m = LeakyReLU(0.01)(m)
m = conv2d(filters=int(256 * number_of_filters))(m)
m = BatchNormalization()(m)
m = LeakyReLU(0.01)(m)
m = Dropout(0.2)(m)
x31 = conv2dtranspose(int(128 * number_of_filters))(m)
x31 = concatenate([x31, x30])
x31 = conv2d(filters=int(128 * number_of_filters))(x31)
x31 = BatchNormalization()(x31)
x31 = LeakyReLU(0.01)(x31)
x31 = conv2d(filters=int(128 * number_of_filters))(x31)
x31 = BatchNormalization()(x31)
x31 = LeakyReLU(0.01)(x31)
x31 = Dropout(0.2)(x31)
x22 = conv2dtranspose(int(64 * number_of_filters))(x31)
x22 = concatenate([x22, x20, x21])
x22 = conv2d(filters=int(64 * number_of_filters))(x22)
x22 = BatchNormalization()(x22)
x22 = LeakyReLU(0.01)(x22)
x22 = conv2d(filters=int(64 * number_of_filters))(x22)
x22 = BatchNormalization()(x22)
x22 = LeakyReLU(0.01)(x22)
x22 = Dropout(0.2)(x22)
x13 = conv2dtranspose(int(32 * number_of_filters))(x22)
x13 = concatenate([x13, x10, x11, x12])
x13 = conv2d(filters=int(32 * number_of_filters))(x13)
x13 = BatchNormalization()(x13)
x13 = LeakyReLU(0.01)(x13)
x13 = conv2d(filters=int(32 * number_of_filters))(x13)
x13 = BatchNormalization()(x13)
x13 = LeakyReLU(0.01)(x13)
x13 = Dropout(0.2)(x13)
x04 = conv2dtranspose(int(16 * number_of_filters))(x13)
x04 = concatenate([x04, x00, x01, x02, x03], axis=3)
x04 = conv2d(filters=int(16 * number_of_filters))(x04)
x04 = BatchNormalization()(x04)
x04 = LeakyReLU(0.01)(x04)
x04 = conv2d(filters=int(16 * number_of_filters))(x04)
x04 = BatchNormalization()(x04)
x04 = LeakyReLU(0.01)(x04)
x04 = Dropout(0.2)(x04)
output = Conv2D(num_classes,
kernel_size=(1, 1),
activation='softmax')(x04)
self.model = tf.keras.Model(inputs=[model_input], outputs=[output])
self.optimizer = Adam(lr=0.0005)
def upsampling_then_conv_block(model, kernal_size, filters, strides, scale):
model = resize_like(scale)(model)
model = Conv2D(filters = filters, kernel_size = kernal_size, strides = strides, padding = "same")(model)
model = LeakyReLU(alpha = 0.2)(model)
return model
def encdec_big_model(concating):
#parameters:
rate = 0.4
glove_size = 300
fMRI_size = 65730
gordon_areas = 333
class_size = 180
sizes = np.load(
str(os.path.dirname(os.path.abspath(__file__))) +
'/data/look_ups/sizes.npy')
reduced = np.load(
str(os.path.dirname(os.path.abspath(__file__))) +
'/data/look_ups/reduced_sizes.npy')
index1 = 0
index = 0
# small ROI dense layers: Each ROI region has its own dense layer. The outputs are then concatenated and used in further layers
input_voxel = Input(shape=(fMRI_size, ))
branch_outputs = []
dense_layers = []
for i in range(gordon_areas):
new_index = index + sizes[i]
small_input = Lambda(lambda x: x[:, index:new_index],
output_shape=(sizes[i], ))(input_voxel)
dense_layers.append(Dense(reduced[i]))
small_out = dense_layers[i](small_input)
small_out = LeakyReLU(alpha=0.3)(small_out)
small_out = BatchNormalization()(small_out)
branch_outputs.append(small_out)
index = new_index
Con = Concatenate()(branch_outputs)
dense1 = Dropout(rate=rate)(Con)
#Glove layer: The output of this layer should represent the matching glove embeddings for their respective fMRI.
# A loss is only applied if the glove prediction model is run.
dense5 = Dense(glove_size)
out_glove = dense5(dense1)
# out_glove = LeakyReLU(alpha=0.3)(out_glove)
# out_glove = BatchNormalization()(out_glove)
#Classification layer: It returns a proability vector for a given fMRI belonging to a certainword out of the possible 180 words.
# The loss is only calculated if the classification model is run.
dense6 = Dense(class_size)
out_further = dense6(dense1)
out_class = Softmax()(out_further)
Concater = Lambda(lambda t: t, name='Concat')(Con)
pred_glove = Lambda(lambda t: t, name='pred_glove')(out_glove)
pred_class = Lambda(lambda t: t, name='pred_class')(out_class)
if concating:
model = Model(inputs=[input_voxel],
outputs=[pred_glove, pred_class, Concater])
else:
model = Model(inputs=[input_voxel], outputs=[pred_glove, pred_class])
return model
def discriminator_block(model, filters, kernel_size, strides):
model = Conv2D(filters = filters, kernel_size = kernel_size, strides = strides, padding = "same")(model)
model = LeakyReLU(alpha = 0.2)(model)
model = BatchNormalization(momentum = 0.5)(model)
return model
def add_d(self, old_model, n_input_layers=3):
init = RandomNormal(stddev=0.02)
const = max_norm(1.0)
in_shape = list(old_model.input.shape)
input_shape = (in_shape[-2].value * 2, in_shape[-2].value * 2,
in_shape[-1].value)
image = Input(shape=input_shape)
d = Conv2D(128, (1, 1),
padding='same',
kernel_initializer=init,
kernel_constraint=const)(image)
d = LeakyReLU(alpha=0.2)(d)
d = Dropout(0.25)(d)
d = Conv2D(128, (3, 3),
padding='same',
kernel_initializer=init,
kernel_constraint=const)(d)
d = LeakyReLU(alpha=0.2)(d)
d = Dropout(0.25)(d)
d = Conv2D(128, (3, 3),
padding='same',
kernel_initializer=init,
kernel_constraint=const)(d)
d = LeakyReLU(alpha=0.2)(d)
d = AveragePooling2D()(d)
block_new = d
# skip the input, 1x1 and activation for the old model
#print(len(old_model.layers))
for i in range(n_input_layers, len(old_model.layers) - 6):
#print(i)
d = old_model.layers[i](d)
# define straight-through model
features = d
validity = Dense(1, activation="sigmoid")(features)
a = Dense(1, activation="sigmoid")(features)
a = Lambda(lambda x: x * 9. + 1)(a)
v = Dense(1, activation="sigmoid")(features)
v = Lambda(lambda x: x * 9. + 1)(v)
da = Dense(1, activation="sigmoid")(features)
da = Lambda(lambda x: x * 9. + 1)(da)
'''
a = Dense(10,name="a", activation="softmax")(features)
v = Dense(10, name="v",activation="softmax")(features)
da = Dense(10,name="da", activation="softmax")(features)
'''
out_class = [validity, a, v, da]
model1 = Model(image, out_class)
'''
model1.compile(loss=["binary_crossentropy", 'sparse_categorical_crossentropy','sparse_categorical_crossentropy','sparse_categorical_crossentropy'],
optimizer=Adam(lr=0.001, beta_1=0, beta_2=0.99, epsilon=10e-8))
'''
model1.compile(loss=[
wasserstein_loss,
tf.keras.losses.LogCosh(),
tf.keras.losses.LogCosh(),
tf.keras.losses.LogCosh()
],
optimizer=Adam(lr=0.0001,
beta_1=0,
beta_2=0.99,
epsilon=10e-8))
downsample = AveragePooling2D()(image)
block_old = old_model.layers[1](downsample)
block_old = old_model.layers[2](block_old)
d = WeightedSum()([block_old, block_new])
# skip the input, 1x1 and activation for the old model
for i in range(n_input_layers, len(old_model.layers) - 6):
d = old_model.layers[i](d)
feature = d
validity = Dense(1, activation="sigmoid")(feature)
a = Dense(1, activation="sigmoid")(feature)
a = Lambda(lambda x: x * 9. + 1)(a)
v = Dense(1, activation="sigmoid")(feature)
v = Lambda(lambda x: x * 9. + 1)(v)
da = Dense(1, activation="sigmoid")(feature)
da = Lambda(lambda x: x * 9. + 1)(da)
'''
a = Dense(10, name="a",activation="softmax")(features)
v = Dense(10, name="v",activation="softmax")(features)
da = Dense(10,name="da", activation="softmax")(features)
'''
out_class = [validity, a, v, da]
model2 = Model(image, out_class)
'''
model2.compile(loss=["binary_crossentropy", 'sparse_categorical_crossentropy','sparse_categorical_crossentropy','sparse_categorical_crossentropy'],
optimizer=Adam(lr=0.001, beta_1=0, beta_2=0.99, epsilon=10e-8))
'''
model2.compile(loss=[
wasserstein_loss,
tf.keras.losses.LogCosh(),
tf.keras.losses.LogCosh(),
tf.keras.losses.LogCosh()
],
optimizer=Adam(lr=0.001,
beta_1=0,
beta_2=0.99,
epsilon=10e-8))
return [model1, model2]
def _conv(self, inputs=None, filters=64, kernel_size=5, strides=2, padding='same'):
outputs = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides,
padding=padding)(inputs)
outputs = LeakyReLU(0.1)(outputs)
return outputs
