def EEGNet3D_Branch(X, Y, T, F1, F2, D, N, size1, size2, block, name):
block1 = DepthwiseConv3D((X, Y, 1),
depth_multiplier=D,
kernel_regularizer=REG,
padding='valid',
activation='linear',
use_bias=False,
name="DepthwiseConv3D" + name)(block)
block1 = BatchNormalization()(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling3D(AVERAGE_POOLING_SIZE1)(block1)
block1 = Dropout(DROPOUT_VAL)(block1)
block2 = DepthwiseConv3D(size2,
depth_multiplier=D,
kernel_regularizer=REG,
padding='same',
activation='linear',
use_bias=False,
name="Separable-Depthwise" + name)(block1)
block2 = Conv3D(F2, (1, 1, 1),
padding='valid',
kernel_regularizer=REG,
activation='linear',
use_bias=False,
name="PointwiseConv3D-Separable" + name)(block2)
block2 = BatchNormalization()(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling3D(AVERAGE_POOLING_SIZE2)(block2)
block2 = Dropout(DROPOUT_VAL)(block2)
flatten = Flatten()(block2)
dense = Dense(N, activation='softmax')(flatten)
return dense
def DepthwiseConvBlock(model_in):
if settings.options.fanout:
_dm = 2
else:
_dm = 1
if settings.options.batchnorm:
model_in = BatchNormalization()(model_in)
if settings.options.regularizer:
if settings.options.D3:
model = DepthwiseConv3D(kernel_size=(3, 3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation,
depthwise_regularizer=l1(
settings.options.regurizer))(model_in)
else:
model = DepthwiseConv2D(kernel_size=(3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation,
depthwise_regularizer=l1(
settings.options.regurizer))(model_in)
elif settings.options.ista > 0:
if settings.options.D3:
model = DepthwiseConv3D(
kernel_size=(3, 3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation,
depthwise_regularizer=ISTA(mu=settings.options.ista *
settings.options.lr))(model_in)
else:
model = DepthwiseConv2D(
kernel_size=(3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation,
depthwise_regularizer=ISTA(mu=settings.options.ista *
settings.options.lr))(model_in)
else:
if settings.options.D3:
model = DepthwiseConv3D(
kernel_size=(3, 3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation)(model_in)
else:
model = DepthwiseConv2D(
kernel_size=(3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation)(model_in)
if settings.options.rescon:
model = Add()([model_in, model])
return model
def __init__(self,
co_shape,
w_shape,
co_initializer=None,
w_initializer=None,
strides=1,
cname="col_"):
super().__init__(name='conn_layer')
if co_initializer is None:
co_initializer = tf.initializers.he_normal()
# co_initializer = tf.truncated_normal_initializer(stddev=0.5, dtype=tf.float32)
if w_initializer is None:
w_initializer = tf.initializers.he_normal()
# w_initializer = tf.truncated_normal_initializer(stddev=0.05, dtype=tf.float32)
self.cname = cname
self.co_matrix = CoL_layer.co_variable_initializer(
co_initializer, co_shape, cname + 'co_matrix')
self.w_spatial = CoL_layer.w_variable_initializer(
w_initializer, w_shape, cname + 'w_filter')
self.co_shape = co_shape
self.w_shape = w_shape
self.strides = strides
self.num_bins = co_shape[0]
self.dc3d = DepthwiseConv3D(kernel_size=(3, 3, 3),
groups=self.num_bins,
padding='SAME',
depthwise_initializer='he_normal')
def EEGNet3D(X, Y, T, F1, F2, D, N):
input1 = Input(shape=(X, Y, T, 1))
block1 = Conv3D(F1, (1, 1, 64),
padding='same',
activation='linear',
use_bias=False,
name="Conv3D")(input1)
block1 = BatchNormalization()(block1)
# Implementation of DepthwiseConv3D Using Conv3D class --- Investigate use of other params
#block1 = Conv3D(D*F1, (X, Y, 1), groups=8, padding='valid', activation='linear', use_bias=False, name="DepthwiseConv3D")(block1)
block1 = DepthwiseConv3D((X, Y, 1),
depth_multiplier=D,
padding='valid',
activation='linear',
use_bias=False,
name="DepthwiseConv3D")(block1)
block1 = BatchNormalization()(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling3D(AVERAGE_POOLING_SIZE1)(block1)
block1 = Dropout(0.25)(block1)
# Implementation of SeperableConv3D using Conv3D class
#block2 = Conv3D(F2, (1, 1, 16), groups=F2, padding='same', activation='linear', use_bias=False, name="DepthwiseConv3D-Separable")(block1)
block2 = DepthwiseConv3D((1, 1, 16),
depth_multiplier=D,
padding='same',
activation='linear',
use_bias=False,
name="Separable-Depthwise")(block1)
block2 = Conv3D(F2, (1, 1, 1),
padding='valid',
activation='linear',
use_bias=False,
name="PointwiseConv3D-Separable")(block2)
block2 = BatchNormalization()(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling3D(AVERAGE_POOLING_SIZE2)(block2)
block2 = Dropout(0.25)(block2)
flatten = Flatten(name='flatten')(block2)
dense = Dense(N, activation='softmax', name='dense')(flatten)
return Model(inputs=input1, outputs=dense)
def DepthwiseConvBlock(model_in):
if settings.options.fanout:
_dm = 2
else:
_dm = 1
if settings.options.regularizer:
if settings.options.D3:
model = DepthwiseConv3D(kernel_size=(3, 3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation,
depthwise_regularizer=l1(
settings.options.regurizer))(model_in)
else:
model = DepthwiseConv2D(kernel_size=(3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation,
depthwise_regularizer=l1(
settings.options.regurizer))(model_in)
else:
if settings.options.D3:
model = DepthwiseConv3D(
kernel_size=(3, 3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation)(model_in)
else:
model = DepthwiseConv2D(
kernel_size=(3, 3),
padding='same',
depth_multiplier=_dm,
activation=settings.options.activation)(model_in)
if settings.options.fanout:
model = concatenate([model_in, model])
else:
model = Add()([model_in, model])
return model
def __init__(self, inputDim=256, hiddenDim=512, nClasses=500, frameLen=29, alpha=2):
super(LipNext, self).__init__()
# initializer = tf.initializers.VarianceScaling(scale=2.0) # added for initialization
initializer = 'glorot_uniform'
self.inputDim = inputDim
self.hiddenDim = hiddenDim
self.nClasses = nClasses
self.frameLen = frameLen
self.nLayers = 2
self.alpha = alpha
# frontend3D
#self.frontend3D = Sequential ( [
# ZeroPadding3D(padding=(1,1,1),), #input_shape=(1,29,96,96)), # double check channel placement
# Conv3D(64, kernel_size=(3,3,3), strides=(1,2,2), use_bias=False, kernel_initializer=initializer, padding='valid'),
# BatchNormalization(momentum=.1, epsilon=1e-5), # should this be .9 instead?
# ReLU(), # check in place?
# # group convolution - TODO: THIS IS NOT RIGHT
# ZeroPadding3D(padding=(1,1,1)),
# Conv3D(64, kernel_size=(3,3,3), strides=(1,2,2), use_bias=False, kernel_initializer=initializer, padding='valid'),
# ZeroPadding3D(padding=(1,0,0)), # double check channel placement
# Conv3D(64, kernel_size=(3,1,1), strides=(1,1,1), use_bias=False, kernel_initializer=initializer, padding='valid')
# ] )
self.frontend3D = Sequential ( [
ZeroPadding3D(padding=(1,1,1),), #input_shape=(1,29,96,96)), # double check channel placement
Conv3D(64, kernel_size=(3,3,3), strides=(1,2,2), use_bias=False, kernel_initializer=initializer, padding='valid'),
BatchNormalization(momentum=.1, epsilon=1e-5), # should this be .9 instead?
ReLU(), # check in place?
# group convolution - TODO: THIS IS NOT RIGHT
ZeroPadding3D(padding=(1,1,1)),])
# Conv3D(64, kernel_size=(3,3,3), strides=(1,2,2), use_bias=False, kernel_initializer=initializer, padding='valid'),
# ZeroPadding3D(padding=(1,0,0)), # double check channel placement
# Conv3D(64, kernel_size=(3,1,1), strides=(1,1,1), use_bias=False, kernel_initializer=initializer, padding='valid')
# ] )
self.perm1 = Permute((4,1,2,3))
self.DConv3D = DepthwiseConv3D(kernel_size=(3,3,3), depth_multiplier=1, strides=(1,2,2), use_bias=False, data_format='channels_last')
self.perm2 = Permute((2,3,4,1))
self.pad1 = ZeroPadding3D(padding=(1,0,0)) # double check channel placement
self.front_conv3d = Conv3D(64, kernel_size=(3,1,1), strides=(1,1,1), use_bias=False, kernel_initializer=initializer, padding='valid')
# resnet
self.permute1 = Permute((1,4,2,3))
self.resnet34 = LipRes(self.alpha)
# backend
self.backend_conv1 = Sequential ( [
Conv1D(2*self.inputDim, kernel_size=5, strides=2, use_bias=False, kernel_initializer=initializer),
BatchNormalization(momentum=0.1, epsilon=1e-5),
ReLU(),
MaxPool1D(2,2),
Conv1D(4*self.inputDim, kernel_size=5, strides=2, use_bias=False, kernel_initializer=initializer),
BatchNormalization(momentum=0.1, epsilon=1e-5),
ReLU()
] )
self.permute2 = Permute((2,1))
self.backend_conv2 = Sequential ( [
Dense(self.inputDim, input_shape=(4 * self.inputDim, )),
BatchNormalization(momentum=0.1, epsilon=1e-5),
ReLU(),
Dense(self.nClasses)
] )