def CNN_submodel_2(INPUTS, l2):
conv1 = Conv2D(32, (1, 128), padding='same',
input_shape=(12, 1024, 1))(INPUTS)
conv1 = BatchNormalization(axis=1)(conv1)
conv1 = DepthwiseConv2D((12, 1),
depth_multiplier=3,
depthwise_constraint=max_norm(1.))(conv1)
conv1 = BatchNormalization(axis=1)(conv1)
conv1 = Activation('elu')(conv1)
conv1 = AveragePooling2D((1, 8))(conv1)
conv1 = Dropout(0.5)(conv1)
conv2 = SeparableConv2D(64, (1, 64), padding='same')(conv1)
conv2 = BatchNormalization(axis=1)(conv2)
conv2 = Activation('elu')(conv2)
conv2 = AveragePooling2D((1, 8))(conv2)
conv2 = Dropout(0.5)(conv2)
conv3 = SeparableConv2D(64, (1, 64), padding='same')(conv2)
conv3 = BatchNormalization(axis=1)(conv3)
conv3 = Activation('elu')(conv3)
conv3 = AveragePooling2D((1, 8))(conv3)
conv3 = Dropout(0.5)(conv3)
flatten = Flatten()(conv3)
dense = Dense(16,
activation="elu",
kernel_regularizer=keras.regularizers.l2(l2))(flatten)
dense = Dense(2, kernel_constraint=max_norm(0.01))(dense)
result = Activation('softmax')(dense)
return Model(inputs=INPUTS, outputs=result)
def inception_resnet(nb_classes=3,
Chans=64,
Samples=321,
dropoutRate=0.5,
kernLength=64,
F1=8,
D=2,
F2=16,
norm_rate=0.25,
dropoutType='Dropout',
gpu=True):
if dropoutType == 'SpatialDropout2D':
dropoutType = SpatialDropout2D
elif dropoutType == 'Dropout':
dropoutType = Dropout
else:
raise ValueError('dropoutType must be one of SpatialDropout2D '
'or Dropout, passed as a string.')
input1 = Input(shape=(1, Chans, Samples))
block1 = Conv2D(F1, (1, kernLength),
padding='same',
input_shape=(1, Chans, Samples),
use_bias=False,
data_format='channels_first')(input1)
block1 = BatchNormalization(axis=1)(block1)
block1 = DepthwiseConv2D((Chans, 1),
use_bias=False,
depth_multiplier=D,
depthwise_constraint=max_norm(1.),
data_format='channels_first')(block1)
block1 = BatchNormalization(axis=1)(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D((1, 2), data_format='channels_first')(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = SeparableConv2D(F2, (1, 16),
use_bias=False,
padding='same',
data_format='channels_first')(block1)
block2 = BatchNormalization(axis=1)(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((1, 4), data_format='channels_first')(block2)
block2 = dropoutType(dropoutRate)(block2)
block3 = Conv2D(F2, (1, 8), padding='same',
data_format='channels_first')(block2)
block3 = BatchNormalization(axis=1)(block3)
block3 = Activation('elu')(block3)
block3 = AveragePooling2D((1, 8), data_format='channels_first')(block3)
flatten = Flatten(name='flatten')(block3)
dense = Dense(1, name='out',
kernel_constraint=max_norm(norm_rate))(flatten)
softmax = Activation('sigmoid', name='sigmoid')(dense)
return Model(inputs=input1, outputs=softmax)
def EEGNet3D_Branch(nb_classes, XDim, YDim, Samples, dropoutRate, kernLength,
F1, D, F2, norm_rate, dropoutType, block):
block1 = Conv3D(F1, (1, 1, kernLength),
padding='same',
input_shape=(XDim, YDim, Samples, 1),
use_bias=False)(block)
block1 = BatchNormalization()(block1)
block1 = Conv3D(D * F1, (XDim, YDim, 1),
groups=F1,
kernel_constraint=max_norm(1.),
use_bias=False)(block1)
block1 = BatchNormalization()(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling3D((1, 1, 4))(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = Conv3D(F2, (1, 1, 16), groups=F2, use_bias=False,
padding='same')(block1)
block2 = Conv3D(F2, (1, 1, 1), use_bias=False, padding='same')(block2)
block2 = BatchNormalization()(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling3D((1, 1, 8))(block2)
block2 = dropoutType(dropoutRate)(block2)
flatten = Flatten()(block2)
return Dense(nb_classes, kernel_constraint=max_norm(norm_rate))(flatten)
def __output_layer(self, connected_layer):
if self.network_type == 'val-adv':
# Reference: https://www.reddit.com/r/reinforcementlearning/comments/bu02ej/help_with_dueling_dqn/
# Value & Advantage Layer
val = Dense(1,
kernel_initializer='he_uniform',
kernel_constraint=max_norm(5),
name='Value')(connected_layer)
val = Activation('linear')(val)
adv = Dense(self.action_size,
kernel_initializer='he_uniform',
kernel_constraint=max_norm(5),
name='Advantage')(connected_layer)
adv = Activation('linear')(adv)
# Output layer
mean = Lambda(lambda x: K.mean(x, axis=1, keepdims=True),
name='Mean')(adv)
adv = Subtract(name='Advantage_Mean')([adv, mean])
outputs = Add(name='Value_Advantage')([val, adv])
else:
outputs = Dense(self.action_size,
kernel_initializer='he_uniform',
kernel_constraint=max_norm(5),
name='Output')(connected_layer)
outputs = Activation('linear')(outputs)
return outputs
def create_model(ModelInfo):
### Defining reguralization technique
# reg = l1(0.001)
model = keras.Sequential()
model.add(layers.Dense(ModelInfo['Nerouns'][0], input_shape=[7],
activation=ModelInfo['Activation_Method'][0],
# kernel_initializer =ModelInfo['W_Initialization_Method'][0],
# bias_initializer = ModelInfo['W_Initialization_Method'][0],
# activity_regularizer=ModelInfo['Reguralization'][0],
activity_regularizer=l1(ModelInfo['Reguralization'][0]),
# kernel_constraint=ModelInfo['kernel_constraint'][0])),
kernel_constraint=max_norm(ModelInfo['kernel_constraint'][0]))),
model.add(layers.Dropout(ModelInfo['Dropout_Value'][0])),
for c in range(1,ModelInfo['Layers'][0]):
print('Index=',c)
model.add(layers.Dense(ModelInfo['Nerouns'][c],
activation=tf.nn.relu,
# kernel_initializer =ModelInfo['W_Initialization_Method'][0],
# bias_initializer = ModelInfo['W_Initialization_Method'][0],
# activity_regularizer=ModelInfo['Reguralization'][c],
activity_regularizer=l1(ModelInfo['Reguralization'][c]),
# kernel_constraint=ModelInfo['kernel_constraint'][c])),
kernel_constraint=max_norm(ModelInfo['kernel_constraint'][c]))),
model.add(layers.Dropout(ModelInfo['Dropout_Value'][c])),
# model.add(layers.BatchNormalization()),
model.add(layers.Dense(1, activation=ModelInfo['Activation_Method'][0])),
return model
def add_decoder_layers(self):
self.model.add(
lay.Conv2DTranspose(128, (3, 3),
kernel_constraint=max_norm(2.0),
kernel_initializer='he_uniform'))
self.model.add(lay.BatchNormalization())
self.model.add(lay.ReLU())
self.model.add(
lay.Conv2DTranspose(64, (3, 3),
kernel_initializer='he_uniform',
kernel_constraint=max_norm(2.0)))
self.model.add(lay.BatchNormalization())
self.model.add(lay.ReLU())
self.model.add(
lay.Conv2DTranspose(32, (3, 3),
kernel_initializer='he_uniform',
kernel_constraint=max_norm(2.0)))
self.model.add(lay.BatchNormalization())
self.model.add(lay.ReLU())
self.model.add(
lay.Conv2DTranspose(3, (3, 3),
activation='sigmoid',
kernel_constraint=max_norm(2.0),
padding='same',
kernel_initializer='he_uniform'))
def build(self):
'encoder --> DCNN'
encoder_input = Input(self.input_shape)
en_conv = Conv2D(self.filter_1, (1, 64),
activation='elu',
padding="same",
kernel_constraint=max_norm(2.,
axis=(0, 1,
2)))(encoder_input)
en_conv = BatchNormalization(axis=3, epsilon=1e-05,
momentum=0.1)(en_conv)
en_conv = AveragePooling2D(pool_size=self.pool_size_1)(en_conv)
en_conv = Conv2D(self.filter_2, (1, 32),
activation='elu',
padding="same",
kernel_constraint=max_norm(2.,
axis=(0, 1, 2)))(en_conv)
en_conv = BatchNormalization(axis=3, epsilon=1e-05,
momentum=0.1)(en_conv)
en_conv = AveragePooling2D(pool_size=self.pool_size_2)(en_conv)
en_conv = Flatten()(en_conv)
encoder_output = Dense(self.latent_dim,
kernel_constraint=max_norm(0.5))(en_conv)
z = Dense(self.num_class,
activation='softmax',
kernel_constraint=max_norm(0.5),
name='classifier')(encoder_output)
return models.Model(inputs=encoder_input,
outputs=[encoder_output, z],
name='MIN2Net_without_decoder')
def add_encoder_layers(self):
self.model.add(
lay.Conv2D(32, (3, 3),
activation='relu',
kernel_constraint=max_norm(2.0),
kernel_initializer='he_uniform'))
self.model.add(lay.BatchNormalization())
self.model.add(lay.ReLU())
self.model.add(
lay.Conv2D(64, (3, 3),
kernel_constraint=max_norm(2.0),
kernel_initializer='he_uniform'))
self.model.add(lay.BatchNormalization())
self.model.add(lay.ReLU())
self.model.add(lay.Conv2D(128, (3, 3),
kernel_initializer='he_uniform'))
self.model.add(lay.BatchNormalization())
self.model.add(lay.ReLU())
self.model.add(
lay.Conv2D(128, (3, 3),
kernel_initializer='he_uniform',
kernel_constraint=max_norm(2.0),
padding='same'))
self.model.add(lay.BatchNormalization())
self.model.add(lay.ReLU())
def cifar_vgg_model(input_shape):
vgg_base = VGG16(
include_top=False,
# weights=None,
weights='imagenet',
input_tensor=Input(shape=input_shape),
input_shape=input_shape,
pooling=None
)
x = vgg_base.output
dropout1 = tf.keras.layers.Dropout(0.5)
dropout2 = tf.keras.layers.Dropout(0.5)
x = dropout1(x)
x = Flatten(name='flatten')(x)
x = Dense(4096, activation='relu', name='fc1', kernel_constraint=max_norm(2), trainable=True
)(x)
x = dropout2(x)
x = Dense(1024, activation='relu', name='fc2', kernel_constraint=max_norm(2), trainable=True
)(x)
predictions = Dense(10, activation='softmax', name='predictions', trainable=True)(x)
model = Model(inputs=vgg_base.input, outputs=predictions)
learning_rate = 0.001
lr_decay = 1e-6
sgd = SGD(lr=learning_rate, decay=lr_decay, momentum=0.9, nesterov=True)
model.compile(loss="categorical_crossentropy", optimizer=sgd, metrics=["accuracy"])
return model
def create_model():
model = Sequential()
model.add(
Dense(16,
input_dim=input_dim,
kernel_constraint=max_norm(3),
kernel_regularizer=regularizers.l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(
Dense(8,
kernel_constraint=max_norm(3),
kernel_regularizer=regularizers.l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(
Dense(output_dim,
kernel_constraint=max_norm(3),
kernel_regularizer=regularizers.l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('sigmoid'))
model.compile(optimizer=Adam(0.01),
loss='binary_crossentropy',
metrics=['accuracy'],
run_eagerly=True) #loss=odds_loss
return model
def build(self):
input1 = Input(shape=self.input_shape)
##################################################################
block1 = Conv2D(self.F1, (1, self.kernLength),
padding='same',
input_shape=self.input_shape,
use_bias=False)(input1)
block1 = BatchNormalization()(block1)
block1 = DepthwiseConv2D((self.Chans, 1),
use_bias=False,
depth_multiplier=self.D,
depthwise_constraint=max_norm(1.))(block1)
block1 = BatchNormalization()(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D((1, 4))(block1)
block1 = Dropout(self.dropout_rate)(block1)
block2 = SeparableConv2D(self.F2, (1, self.kernLength // 4),
use_bias=False,
padding='same')(block1)
block2 = BatchNormalization()(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((1, 8))(block2)
block2 = Dropout(self.dropout_rate)(block2)
flatten = Flatten(name='flatten')(block2)
dense = Dense(self.num_class,
name='dense',
kernel_constraint=max_norm(self.norm_rate))(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input1, outputs=softmax)
def ShallowConvNet(nb_classes, Chans=64, Samples=128, dropoutRate=0.5, cpu=False):
if cpu:
input_shape = (Samples, Chans, 1)
conv_filters = (25, 1)
conv_filters2 = (1, Chans)
pool_size = (45, 1)
strides = (15, 1)
axis = -1
else:
input_shape = (1, Chans, Samples)
conv_filters = (1, 20)
conv_filters2 = (Chans, 1)
pool_size = (1, 45)
strides = (1, 15)
axis = 1
input_main = Input(input_shape)
block1 = Conv2D(20, conv_filters,
input_shape=input_shape,
kernel_constraint=max_norm(2., axis=(0, 1, 2)))(input_main)
block1 = Conv2D(20, conv_filters2, use_bias=False,
kernel_constraint=max_norm(2., axis=(0, 1, 2)))(block1)
block1 = BatchNormalization(axis=axis, epsilon=1e-05, momentum=0.1)(block1)
block1 = Activation(square)(block1)
block1 = AveragePooling2D(pool_size=pool_size, strides=strides)(block1)
block1 = Activation(log)(block1)
block1 = Dropout(dropoutRate)(block1)
flatten = Flatten()(block1)
dense = Dense(nb_classes, kernel_constraint=max_norm(0.5))(flatten)
softmax = Activation('softmax')(dense)
return Model(inputs=input_main, outputs=softmax)
def EEGNet(nb_classes, Chans=64, Samples=128,
dropoutRate=0.5, kernLength=64, F1=8,
D=2, F2=16, norm_rate=0.25, dropoutType='Dropout', cpu=False):
if dropoutType == 'SpatialDropout2D':
dropoutType = SpatialDropout2D
elif dropoutType == 'Dropout':
dropoutType = Dropout
else:
raise ValueError('dropoutType must be one of SpatialDropout2D '
'or Dropout, passed as a string.')
if cpu:
input_shape = (Samples, Chans, 1)
conv_filters = (kernLength, 1)
depth_filters = (1, Chans)
pool_size = (6, 1)
pool_size2 = (12, 1)
separable_filters = (20, 1)
axis = -1
else:
input_shape = (1, Chans, Samples)
conv_filters = (1, kernLength)
depth_filters = (Chans, 1)
pool_size = (1, 6)
pool_size2 = (1, 12)
separable_filters = (1, 20)
axis = 1
input1 = Input(shape=input_shape)
block1 = Conv2D(F1, conv_filters, padding='same',
input_shape=input_shape,
use_bias=False)(input1)
block1 = BatchNormalization(axis=axis)(block1)
block1 = DepthwiseConv2D(depth_filters, use_bias=False,
depth_multiplier=D,
depthwise_constraint=max_norm(1.))(block1)
block1 = BatchNormalization(axis=axis)(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D(pool_size)(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = SeparableConv2D(F2, separable_filters,
use_bias=False, padding='same')(block1)
block2 = BatchNormalization(axis=axis)(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D(pool_size2)(block2)
block2 = dropoutType(dropoutRate)(block2)
flatten = Flatten(name='flatten')(block2)
dense = Dense(nb_classes, name='dense',
kernel_constraint=max_norm(norm_rate))(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input1, outputs=softmax)
def classification_model(input_shape, class_num):
model = Sequential()
# Convolutional layer
model.add(Conv2D(32, (3, 3), input_shape=input_shape, padding='same'))
model.add(Activation('relu'))
# Dropout 2% of the data to avoid overfitting
model.add(Dropout(0.2))
# Normalize inputs for the next layer
model.add(BatchNormalization())
# Second convolutional layer (complex representations)
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
# Learn relevant patterns
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(BatchNormalization())
# Repeat
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Conv2D(128, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
# Flatten result
model.add(Flatten())
model.add(Dropout(0.2))
# Dense layers
model.add(Dense(256, kernel_constraint=max_norm(3)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Dense(128, kernel_constraint=max_norm(3)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
# Prediction layers
model.add(Dense(class_num))
model.add(Activation('softmax'))
return model
def get_model():
"""
Returns a compiled convolutional neural network model. Assume that the
`input_shape` of the first layer is `(IMG_WIDTH, IMG_HEIGHT, 3)`.
The output layer should have `NUM_CATEGORIES` units, one for each category.
"""
# Create a convolutional neural network
model = tf.keras.models.Sequential([
# Convolutional layer #1. Learn 32 filters using a 3x3 kernel using input size
tf.keras.layers.Conv2D(FILTERS, (KERNEL, KERNEL),
input_shape=(IMG_WIDTH, IMG_HEIGHT, IMG_THI),
padding="same"),
tf.keras.layers.Activation("relu"),
tf.keras.layers.BatchNormalization(),
# Convolutional layer #2. Learn 64 filters using a 3x3 kernel, with pooling
tf.keras.layers.Conv2D(FILTERS * 2, (KERNEL, KERNEL), padding="same"),
tf.keras.layers.Activation("relu"),
tf.keras.layers.MaxPooling2D(pool_size=(POOLSIZE, POOLSIZE)),
tf.keras.layers.BatchNormalization(),
# Convolutional layer #3. Learn 128 filters using a 3x3 kernel, with pooling
tf.keras.layers.Conv2D(FILTERS * 4, (KERNEL, KERNEL), padding="same"),
tf.keras.layers.Activation("relu"),
tf.keras.layers.MaxPooling2D(pool_size=(POOLSIZE, POOLSIZE)),
tf.keras.layers.BatchNormalization(),
# Flatten units
tf.keras.layers.Flatten(),
# Add a Dense hidden layer #1, 256 filters with dropout
tf.keras.layers.Dense(FILTERS * 8, kernel_constraint=max_norm(3)),
tf.keras.layers.Activation("relu"),
tf.keras.layers.Dropout(DROPOUT),
tf.keras.layers.BatchNormalization(),
# Add a Dense hidden layer #2, 128 filters with dropout
tf.keras.layers.Dense(FILTERS * 4, kernel_constraint=max_norm(3)),
tf.keras.layers.Activation("relu"),
tf.keras.layers.Dropout(DROPOUT),
tf.keras.layers.BatchNormalization(),
# Add an output layer with output units for all categories
tf.keras.layers.Dense(NUM_CATEGORIES),
tf.keras.layers.Activation("softmax")
])
# Train neural network
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["accuracy"])
return model
def EEGNet(nb_classes,
Chans=64,
Samples=128,
dropoutRate=0.5,
kernLength=64,
F1=4,
D=2,
F2=8,
norm_rate=0.25,
dropoutType='Dropout'):
if dropoutType == 'SpatialDropout2D':
dropoutType = SpatialDropout2D
elif dropoutType == 'Dropout':
dropoutType = Dropout
else:
raise ValueError('dropoutType must be one of SpatialDropout2D '
'or Dropout, passed as a string.')
input1 = Input(shape=(Chans, Samples, 1))
##################################################################
# padding = 'valid' => no padding
# padding = 'same' => output has the same height/width dimension as the input
block1 = Conv2D(F1, (1, kernLength),
padding='same',
input_shape=(Chans, Samples, 1),
use_bias=False)(input1)
block1 = BatchNormalization()(block1)
block1 = DepthwiseConv2D((Chans, 1),
use_bias=False,
depth_multiplier=D,
depthwise_constraint=max_norm(1.))(block1)
block1 = BatchNormalization()(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D((1, 16))(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = SeparableConv2D(F2, (1, 64), use_bias=False,
padding='same')(block1)
block2 = BatchNormalization()(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((1, 16))(block2)
block2 = dropoutType(dropoutRate)(block2)
flatten = Flatten(name='flatten')(block2)
dense = Dense(nb_classes,
name='dense',
kernel_constraint=max_norm(norm_rate))(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input1, outputs=softmax)
def build_model_conv(self, actions):
"""
define the neural network model architecture for the deep q agent
"""
model = tf.keras.models.Sequential()
model.add(
Conv2D(32, (2, 2),
padding='same',
kernel_initializer='he_uniform',
kernel_constraint=max_norm(3),
input_shape=(1, self.env.height, self.env.width)))
model.add(BatchNormalization())
model.add(Activation('tanh'))
model.add(
Conv2D(64, (2, 2),
padding='same',
kernel_initializer='he_uniform',
kernel_constraint=max_norm(3)))
model.add(BatchNormalization())
model.add(Activation('tanh'))
model.add(
Conv2D(64, (2, 2),
padding='same',
kernel_initializer='he_uniform',
kernel_constraint=max_norm(3)))
model.add(BatchNormalization())
model.add(Activation('tanh'))
# model.add(MaxPooling2D(pool_size=(2,2)))
# end of convolutional layers, start of 'hidden' dense layers
model.add(Flatten())
model.add(
Dense(128,
kernel_initializer='he_uniform',
kernel_constraint=max_norm(3)))
model.add(BatchNormalization())
model.add(Activation('tanh'))
model.add(Dropout(0.5))
# Final dense layer
model.add(Dense(actions))
model.add(BatchNormalization())
model.add(Activation('linear'))
return model
def ShallowConvNet(nb_classes,
Chans=64,
Samples=128,
is_denoising=False,
dropoutRate=0.5):
""" Keras implementation of the Shallow Convolutional Network as described
in Schirrmeister et. al. (2017), Human Brain Mapping.
Assumes the input is a 2-second EEG signal sampled at 128Hz. Note that in
the original paper, they do temporal convolutions of length 25 for EEG
data sampled at 250Hz. We instead use length 13 since the sampling rate is
roughly half of the 250Hz which the paper used. The pool_size and stride
in later layers is also approximately half of what is used in the paper.
Note that we use the max_norm constraint on all convolutional layers, as
well as the classification layer. We also change the defaults for the
BatchNormalization layer. We used this based on a personal communication
with the original authors.
ours original paper
pool_size 1, 35 1, 75
strides 1, 7 1, 15
conv filters 1, 13 1, 25
Note that this implementation has not been verified by the original
authors. We do note that this implementation reproduces the results in the
original paper with minor deviations.
"""
# start the model
input_main = Input((1, Chans, Samples))
block1 = Conv2D(40, (1, 13),
input_shape=(1, Chans, Samples),
kernel_constraint=max_norm(2., axis=(0, 1, 2)))(input_main)
# if is_denoising:
# block1 = Lambda(lambda t: denoising(t, name='denosing_1', embed=True, softmax=True))(block1)
block1 = Conv2D(40, (Chans, 1),
use_bias=False,
kernel_constraint=max_norm(2., axis=(0, 1, 2)))(block1)
block1 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block1)
block1 = Activation(square)(block1)
block1 = AveragePooling2D(pool_size=(1, 35), strides=(1, 7))(block1)
block1 = Activation(log)(block1)
block1 = Dropout(dropoutRate)(block1)
flatten = Flatten()(block1)
dense = Dense(nb_classes, kernel_constraint=max_norm(0.5))(flatten)
softmax = Activation('softmax')(dense)
return Model(inputs=input_main, outputs=softmax)
def build_model(num_heroes):
model = Sequential()
model.add(Dropout(0.2, input_shape=(num_heroes, )))
# model.add(Dense(128, activation='relu', input_dim=num_heroes))
model.add(Dense(128, activation='relu', kernel_constraint=max_norm(3)))
model.add(Dropout(0.2))
model.add(Dense(64, activation='relu', kernel_constraint=max_norm(3)))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
adam = Adam(lr=0.01)
model.compile(optimizer=adam,
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def EEGNet(input_layer, F1=4, kernLength=64, D=2, Chans=22, dropout=0.1):
F2 = F1 * D
block1 = Conv2D(F1, (kernLength, 1),
padding='same',
data_format='channels_last',
use_bias=False)(input_layer)
block1 = BatchNormalization(axis=-1)(block1)
block2 = DepthwiseConv2D((1, Chans),
use_bias=False,
depth_multiplier=D,
data_format='channels_last',
depthwise_constraint=max_norm(1.))(block1)
block2 = BatchNormalization(axis=-1)(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((8, 1), data_format='channels_last')(block2)
block2 = Dropout(dropout)(block2)
block3 = SeparableConv2D(F2, (16, 1),
data_format='channels_last',
use_bias=False,
padding='same')(block2)
block3 = BatchNormalization(axis=-1)(block3)
block3 = Activation('elu')(block3)
block3 = AveragePooling2D((8, 1), data_format='channels_last')(block3)
block3 = Dropout(dropout)(block3)
return block3
def get_untrained_l6_all_digit_model(inputs):
model = Sequential()
model.add(
Conv2D(32, (3, 3),
padding='same',
activation='relu',
input_shape=inputs.shape[1:]))
model.add(Dropout(0.2))
model.add(Conv2D(32, (3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(Dropout(0.2))
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, (3, 3), padding='same', activation='relu'))
model.add(Dropout(0.2))
model.add(Conv2D(128, (3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dropout(0.2))
model.add(Dense(1024, activation='relu', kernel_constraint=max_norm(3)))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def define_discriminator(n_blocks, input_shape=(4, 4, 3)):
weight_init = RandomNormal(stddev=0.02)
weight_constr = max_norm(1.0)
model_list = []
img_input = Input(shape=input_shape)
d = Conv2D(128, (1, 1), padding='same', kernel_initializer=weight_init, kernel_constraint=weight_constr)(img_input)
d = LeakyReLU(alpha=0.2)(d)
d = MinibatchStdDev()(d)
d = Conv2D(128, (3, 3), padding='same', kernel_initializer=weight_init, kernel_constraint=weight_constr)(d)
d = LeakyReLU(alpha=0.2)(d)
d = Conv2D(128, (4, 4), padding='same', kernel_initializer=weight_init, kernel_constraint=weight_constr)(d)
d = LeakyReLU(alpha=0.2)(d)
d = Flatten()(d)
out_class = Dense(1)(d)
model = Model(img_input, out_class)
model.compile(loss=ProGAN.wasserstein_loss, optimizer=Adam(lr=0.001, beta_1=0, beta_2=0.99, epsilon=10e-8))
model_list.append([model, model])
# Create sub_models
for i in range(1, n_blocks):
old_model = model_list[i - 1][0]
new_models = Discriminator._add_discriminator_block(old_model)
model_list.append(new_models)
return model_list
def add_g(self, old_model):
# weight initialization
init = RandomNormal(stddev=0.02)
# weight constraint
const = max_norm(1.0)
block_end = old_model.layers[-2].output
upsampling = UpSampling2D()(block_end)
g = Conv2D(128, (3, 3),
padding='same',
kernel_initializer=init,
kernel_constraint=const)(upsampling)
g = PixelNormalization()(g)
g = LeakyReLU(alpha=0.2)(g)
g = Conv2D(128, (3, 3),
padding='same',
kernel_initializer=init,
kernel_constraint=const)(g)
g = PixelNormalization()(g)
g = LeakyReLU(alpha=0.2)(g)
out_image = Conv2D(3, (1, 1),
padding='same',
kernel_initializer=init,
kernel_constraint=const)(g)
model1 = Model(old_model.input, out_image)
out_old = old_model.layers[-1]
out_image2 = out_old(upsampling)
merged = WeightedSum()([out_image2, out_image])
model2 = Model(old_model.input, merged)
return [model1, model2]
def __conv_block(self, x, r):
if self.use_constraint:
constraint = max_norm(self.constraint_rate)
else:
constraint = None
for i in range(self.conv_blocks):
x = tf.keras.layers.Conv2D(filters=self.filter_start * (r + 1),
kernel_size=self.add_tuples(
self.filter_size, (r + 1, r + 1)),
kernel_constraint=constraint)(x)
if self.use_bn:
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Activation(self.activation)(x)
x = tf.keras.layers.Conv2D(filters=self.filter_start * (r + 1),
kernel_size=self.add_tuples(
self.filter_size, (r + 1, r + 1)),
kernel_constraint=constraint)(x)
if self.use_bn:
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Activation(self.activation)(x)
if self.use_dropout:
x = tf.keras.layers.Dropout(self.dropout_rate)(x)
x = tf.keras.layers.MaxPooling2D(pool_size=self.pool_size)(x)
return x
def TA_CSPNN(nb_classes,
Channels=64,
Timesamples=90,
dropOut=0.25,
timeKernelLen=50,
Ft=11,
Fs=6):
# Input shape is (trials, 1, number of channels, number of time samples)
input_e = Input(shape=(1, Channels, Timesamples))
convL1 = Conv2D(Ft, (1, timeKernelLen),
padding='same',
input_shape=(1, Channels, Timesamples),
use_bias=False)(input_e)
bNorm1 = BatchNormalization(axis=1)(convL1)
convL2 = DepthwiseConv2D((Channels, 1),
use_bias=False,
depth_multiplier=Fs,
depthwise_constraint=max_norm(1.))(bNorm1)
bNorm2 = BatchNormalization(axis=1)(convL2)
lambdaL = Lambda(lambda x: x**2)(bNorm2)
aPool = AveragePooling2D((1, Timesamples))(lambdaL)
dOutL = Dropout(dropOut)(aPool)
flatten = Flatten(name='flatten')(dOutL)
dense = Dense(nb_classes, name='dense')(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input_e, outputs=softmax)
def define_generator(latent_dim, n_blocks, output_shape=(4, 4, 3)):
weight_init = RandomNormal(stddev=0.02)
weight_constr = max_norm(1.0)
model_list = []
in_latent = Input(shape=(latent_dim,))
g = Dense(128 * output_shape[0] * output_shape[1], kernel_initializer=weight_init, kernel_constraint=weight_constr)(in_latent)
g = Reshape((output_shape[0], output_shape[1], 128))(g)
g = Conv2D(128, (4, 4), padding='same', kernel_initializer=weight_init, kernel_constraint=weight_constr)(g)
g = PixelNormalisation()(g)
g = LeakyReLU(alpha=0.2)(g)
g = Conv2D(128, (3, 3), padding='same', kernel_initializer=weight_init, kernel_constraint=weight_constr)(g)
g = PixelNormalisation()(g)
g = LeakyReLU(alpha=0.2)(g)
img_output = Conv2D(output_shape[-1], (1, 1), padding='same', kernel_initializer=weight_init, kernel_constraint=weight_constr)(g)
model = Model(in_latent, img_output)
model_list.append([model, model])
for i in range(1, n_blocks):
old_model = model_list[i - 1][0]
new_models = Generator._add_generator_block(old_model, output_shape)
model_list.append(new_models)
return model_list
def _add_generator_block(old_model, output_shape):
weight_init = RandomNormal(stddev=0.02)
weight_constr = max_norm(1.0)
block_end = old_model.layers[-2].output
upsampling = UpSampling2D()(block_end)
g = Conv2D(128, (3, 3), padding='same', kernel_initializer=weight_init, kernel_constraint=weight_constr)(upsampling)
g = PixelNormalisation()(g)
g = LeakyReLU(alpha=0.2)(g)
g = Conv2D(128, (3, 3), padding='same', kernel_initializer=weight_init, kernel_constraint=weight_constr)(g)
g = PixelNormalisation()(g)
g = LeakyReLU(alpha=0.2)(g)
img_output_new = Conv2D(output_shape[-1], (1, 1), padding='same', kernel_initializer=weight_init, kernel_constraint=weight_constr)(g)
straight_through_model = Model(old_model.input, img_output_new)
old_output = old_model.layers[-1]
img_output_old = old_output(upsampling)
merged = WeightedSum()([img_output_old, img_output_new])
fade_in_model = Model(old_model.input, merged)
return straight_through_model, fade_in_model
def EEGNetEncPartB(_input,
D=2,
D_pooling=4,
dropoutRate=0.25,
activation='elu',
l1_reg=0.000, l2_reg=0.000,
return_model=None):
# Spatial-filter-like.
# Applies D different filters along channels domain for each F1 (temporal filter above).
# As our channels-domain filter is the number of channels, and we use padding='valid',
# the channels domain is reduced from n_channels to 1.
# (num_chans, num_samples, F1) --> (1, num_samples, F1*D)
depth_regu = tf.keras.regularizers.l1_l2(l1=l1_reg, l2=l2_reg)
_y = layers.DepthwiseConv2D((_input.shape.as_list()[1], 1),
padding='valid',
# use_bias=False,
depth_multiplier=D,
depthwise_regularizer=depth_regu,
depthwise_constraint=constraints.max_norm(1.))(_input)
_y = layers.BatchNormalization()(_y)
_y = layers.Activation(activation)(_y)
# Smooth in the time-dimension
# (1, num_samples, F1 * D) --> (1, num_samples // D_pooling, F1 * D)
_y = layers.AveragePooling2D((1, D_pooling))(_y)
_y = layers.Dropout(dropoutRate)(_y)
if return_model is False:
return _y
else:
return models.Model(inputs=_input, outputs=_y)
def DepthwiseConv2DBlock(_input,
depth_multiplier=4,
depth_pooling=1,
dropout_rate=0.25, dropout_type='Dropout',
activation='elu',
return_model=None):
if dropout_type == 'SpatialDropout2D':
dropout_type = layers.SpatialDropout2D
elif dropout_type == 'Dropout':
dropout_type = layers.Dropout
else:
raise ValueError('dropoutType must be one of SpatialDropout2D '
'or Dropout, passed as a string.')
_y = layers.DepthwiseConv2D((_input.shape.as_list()[0], 1), use_bias=False,
depth_multiplier=depth_multiplier,
depthwise_constraint=constraints.max_norm(1.))(_input)
_y = layers.BatchNormalization()(_y)
_y = layers.Activation(activation)(_y)
if depth_pooling > 1:
_y = layers.AveragePooling2D((1, depth_pooling))(_y)
_y = dropout_type(dropout_rate)(_y)
if return_model is False:
return _y
else:
return models.Model(inputs=_input, outputs=_y)
def genb(self):
model = Sequential()
const = max_norm(1.0)
init = RandomNormal(stddev=0.02)
######### b to a domain
imgb = Input(shape=self.img_shape)
imga = Input(shape=self.img_shape)
noise1 = self.encoder3(imgb)
noise2 = self.encoder4(imga)
noise = Concatenate(axis=1)([noise1, noise2])
dep = 16
noise = Dense(256 * dep * dep,
kernel_initializer=init,
kernel_constraint=const)(noise)
noise = Reshape((dep, dep, 256))(noise)
model.add(
Conv2DTranspose(256,
kernel_size=3,
strides=2,
padding="same",
kernel_initializer=init,
kernel_constraint=const))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(
Conv2DTranspose(128,
kernel_size=3,
padding="same",
strides=2,
kernel_initializer=init,
kernel_constraint=const))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(
Conv2DTranspose(64,
kernel_size=7,
padding="same",
strides=1,
kernel_initializer=init,
kernel_constraint=const))
model.add(Activation("relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(
Conv2D(self.channels,
kernel_size=7,
strides=1,
padding="same",
kernel_initializer=init,
kernel_constraint=const))
# model.add(Conv2D(self.channels, kernel_size=3, stride=1,padding="same", kernel_initializer=init, kernel_constraint=const))
model.add(Activation("tanh"))
image = model(noise)
return Model([imgb, imga], image)
