def define_model_selu():
inputs = Input((128,))
x1f = Dense(128, activation='selu')(inputs)
x1b = BatchNormalization()(x1f)
x1d = AlphaDropout(0.5)(x1b)
x1 = keras.layers.add([x1d, inputs])
x2f = Dense(128, activation='selu')(x1)
x2b = BatchNormalization()(x2f)
x2d = AlphaDropout(0.5)(x2b)
x2 = keras.layers.add([x1, x2d, inputs])
x3f = Dense(64, activation='selu')(x2)
x3b = BatchNormalization()(x3f)
x3d = AlphaDropout(0.5)(x3b)
x4f = Dense(64, activation='selu')(x3d)
x4b = BatchNormalization()(x4f)
x4d = AlphaDropout(0.5)(x4b)
x4 = keras.layers.add([x3d, x4d])
x5f = Dense(64, activation='selu')(x4)
x5b = BatchNormalization()(x5f)
x6f = Dense(41, activation='softmax')(x5b)
model = Model(inputs=inputs, outputs=x6f)
model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.01),
metrics=['accuracy', ])
return model
def create_model(self):
embeddingSize = 128
maxSeqLength = self.Config["max_chars_seq_len"]
convLayersData = [[256, 10], [256, 7], [256, 5], [256, 3]]
dropout_p = 0.1
optimizer = 'adam'
inputs = Input(shape=(maxSeqLength, ), dtype='int64')
x = Embedding(len(arabic_charset()) + 1,
embeddingSize,
input_length=maxSeqLength)(inputs)
convolution_output = []
for num_filters, filter_width in convLayersData:
conv = Convolution1D(filters=num_filters,
kernel_size=filter_width,
activation='tanh')(x)
pool = GlobalMaxPooling1D()(conv)
convolution_output.append(pool)
x = Concatenate()(convolution_output)
x = Dense(1024, activation='selu',
kernel_initializer='lecun_normal')(x)
x = AlphaDropout(dropout_p)(x)
x = Dense(1024, activation='selu',
kernel_initializer='lecun_normal')(x)
x = AlphaDropout(dropout_p)(x)
predictions = Dense(len(self.Config["predefined_categories"]),
activation='sigmoid')(x)
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def define_model_snn_cifar10():
model = Sequential()
model.add(Conv1D(16, 3, strides=2, padding='same', input_shape=[128, 1],
kernel_initializer='lecun_normal', bias_initializer='zeros'))
model.add(Activation('selu'))
model.add(Conv1D(16, 3, kernel_initializer='lecun_normal', bias_initializer='zeros'))
model.add(Activation('selu'))
model.add(MaxPooling1D(pool_size=2))
model.add(AlphaDropout(0.1))
model.add(Conv1D(32, 3, padding='same', kernel_initializer='lecun_normal', bias_initializer='zeros'))
model.add(Activation('selu'))
model.add(Conv1D(32, 3, kernel_initializer='lecun_normal', bias_initializer='zeros'))
model.add(Activation('selu'))
model.add(MaxPooling1D(pool_size=2))
model.add(AlphaDropout(0.1))
model.add(Flatten())
model.add(Dense(41, kernel_initializer='lecun_normal', bias_initializer='zeros'))
model.add(Activation('selu'))
model.add(AlphaDropout(0.2))
model.add(Dense(41, kernel_initializer='lecun_normal', bias_initializer='zeros'))
model.add(Activation('softmax'))
# initiate RMSprop optimizer
opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6)
# Let's train the model using RMSprop
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy', 'top_k_categorical_accuracy'])
print(model.summary())
return model
def loadModel(vocabularySize, sequenceLength, vecSpaceSize):
kernelSizes = [1, 2, 3, 4, 5]
#kernelSizes2 = [3, 4, 5, 6, 7]
#init = RandomNormal(stddev=0.02)
inputs = Input(shape=(sequenceLength, ))
embedding = Embedding(vocabularySize, vecSpaceSize)(inputs)
#reshape = Reshape((sequenceLength, vecSpaceSize, 1))(embedding)
conved = [
Conv1D(filters=32,
kernel_size=kernelSize,
activation="selu",
kernel_initializer="lecun_normal")(embedding)
for kernelSize in kernelSizes
]
#convedActivated = [LeakyReLU(alpha=0.20)(conv) for conv in conved]
pooled = [MaxPooling1D(2, strides=2)(conv) for conv in conved]
#batchNorms1 = [BatchNormalization()(pool) for pool in pooled]
#dropouts = [Dropout(0.30)(pool) for pool in pooled]
conved2 = [
Conv1D(filters=64,
kernel_size=5,
strides=3,
activation="selu",
kernel_initializer="lecun_normal")(pool) for pool in pooled
]
#convedActivated2 = [LeakyReLU(alpha=0.20)(conv) for conv in conved2]
pooled2 = [MaxPooling1D(2, strides=2)(conv) for conv in conved2]
#batchNorms2 = [BatchNormalization()(pool) for pool in pooled2]
#dropouts = [Dropout(0.30)(pool) for pool in pooled]
#conved3 = [Conv1D(filters=32, kernel_size=3, strides=2, activation="relu")(batchNorm) for batchNorm in batchNorms2]
#convedActivated2 = [LeakyReLU(alpha=0.20)(conv) for conv in conved2]
#pooled3 = [MaxPooling1D(2, strides=2)(conv) for conv in conved3]
#batchNorms2 = [BatchNormalization()(pool) for pool in pooled2]
#dropouts2 = [Dropout(0.50)(pool) for pool in pooled2]
flattened = [Flatten()(pool) for pool in pooled]
merged = Concatenate()(flattened)
dropoutDense1 = AlphaDropout(0.30)(merged)
dense = Dense(10, activation="selu",
kernel_initializer="lecun_normal")(dropoutDense1)
#leakyReLU = LeakyReLU(alpha=0.20)(dense)
#batchNorm3 = BatchNormalization()(leakyReLU)
dropoutDense2 = AlphaDropout(0.20)(dense)
outputs = Dense(1, activation="sigmoid",
kernel_initializer="lecun_normal")(dropoutDense2)
model = Model(inputs=inputs, outputs=outputs, name="sentiment_analysis")
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def build_snn_model(
x_shape=(100, 1, 10),
n_layers=1,
n_units=64,
kernel_reg=1e-9,
activity_reg=1e-9,
bias_reg=1e-9,
dropout_rate=0.5,
optimizer='nadam',
lr_rate=1e-5,
gauss_noise_std=1e-3,
n_gpus=0,
):
"""build snn model"""
opts_map = {
'adam': opts.Adam,
'nadam': opts.Nadam,
'adamax': opts.Adamax,
'sgd': opts.SGD,
'rmsprop': opts.RMSprop
}
snn_cfg = {
'units':
int(n_units),
#'batch_input_shape': (batch_size, x_shape[1], x_shape[2]),
#'batch_size': batch_size,
'input_shape':
x_shape,
'kernel_regularizer':
regularizers.l2(kernel_reg),
'activity_regularizer':
regularizers.l2(activity_reg),
'bias_regularizer':
regularizers.l2(bias_reg),
'kernel_initializer':
initializers.lecun_normal(seed=cfg.data_cfg['random_seed']),
'activation':
'selu',
}
model = Sequential()
model.add(Dense(**snn_cfg))
model.add(GaussianNoise(gauss_noise_std))
model.add(AlphaDropout(dropout_rate))
if n_layers > 1:
for i in range(n_layers - 1):
snn_cfg.pop('batch_input_shape', None)
model.add(Dense(**snn_cfg))
model.add(GaussianNoise(gauss_noise_std))
model.add(AlphaDropout(dropout_rate))
model.add(Flatten()) # todo: WHy lol
model.add(Dense(len(cfg.data_cfg['Target_param_names'])))
opt = opts_map[optimizer](lr=lr_rate)
model.compile(optimizer=opt, loss='mse')
return model
def build_model(self, img_height=256, img_width=256, activation = 'selu', filter_initializer = 'lecun_normal', blocks = [64, 64, 64, 64] ,use_tfboard = False):
bands = 4
if K.image_data_format() == 'channels_first':
ch_axis = 1
input_shape = (bands, img_height, img_width)
if K.image_data_format() == 'channels_last':
ch_axis = 3
input_shape = (img_height, img_width, bands)
inp = Input(input_shape)
encoder = inp
list_encoders = []
print('building 2D Convolution Model ...')
print(blocks)
# Setting up the filter size
filter_size = (3,3)
# Encoding
for block_id , n_block in enumerate(blocks):
with K.name_scope('Encoder_block_{0}'.format(block_id)):
encoder = Conv2D(filters = n_block, kernel_size = filter_size, activation = activation, padding = 'same',
kernel_initializer = filter_initializer)(encoder)
encoder = AlphaDropout(0,1*block_id, )(encoder)
encoder = Conv2D(filters = n_block, kernel_size = filter_size, dilation_rate = (2,2),
activation = activation, padding='same', kernel_initializer = filter_initializer)(encoder)
list_encoders.append(encoder)
# maxpooling 'BETWEEN' every 2 blocks
if block_id < len(blocks)-1:
encoder = MaxPooling2D(pool_size = (2,2))(encoder)
# Decoding
decoder = encoder
decoder_blocks = blocks[::-1][1:]
for block_id, n_block in enumerate(decoder_blocks):
with K.name_scope('Decoder_block_{0}'.format(block_id)):
block_id_inv = len(blocks) - 1 - block_id
decoder = concatenate([decoder, list_encoders[block_id_inv]], axis = ch_axis) # concatenate the first decoder with the last encoder and so on, according to the channell axis
decoder = Conv2D(filters=n_block, kernel_size = filter_size, activation = activation, padding = 'same',
dilation_rate = (2,2), kernel_initializer = filter_initializer)(decoder)
decoder = AlphaDropout(0,1*block_id, )(decoder)
decoder = Conv2D(filters=n_block, kernel_size = filter_size, activation = activation, padding = 'same',
kernel_initializer = filter_initializer)(decoder)
decoder = Conv2DTranspose(filters=n_block, kernel_size = filter_size, kernel_initializer = filter_initializer,
padding='same', strides=(2,2))(decoder)
# Last Layer...
outp = Conv2DTranspose(filters=1, kernel_size = filter_size, activation = 'sigmoid',
padding = 'same', kernel_initializer = 'glorot_normal')(decoder)
self.model = Model(inputs=[inp], outputs=[outp])
return
def selu_base_network(input_shape):
'''Base network to be shared (eq. to feature extraction).
'''
inputs = Input(shape=input_shape)
x = Dense(128, activation='selu',
kernel_initializer='lecun_normal')(inputs)
x = AlphaDropout(0.1)(x)
x = Dense(128, activation='selu', kernel_initializer='lecun_normal')(x)
x = AlphaDropout(0.1)(x)
x = Dense(128, activation='selu', kernel_initializer='lecun_normal')(x)
return Model(inputs, x)
def __autoencoder(size, encoded_size, l1, activation, dropout_rate,
use_batch_norm, coefficients):
# TODO activation fns: relu, elu, selu (AlphaDropout instead of dropout for selu)
# TODO dropout rate: 0 (no dropout), 0.1, 0.2, 0.3
# TODO batch norm: YES/NO
# TODO l1: 0 (no regularization), 1e-7, 1e-6
# TODO optimizers: rmsprop, adam, nadam
# TODO coefficients: [4, 8, 12], [4, 8], [2, 4] etc.
input_layer = Input(shape=(size, ), name='input_conformation')
x = input_layer
for i, c in enumerate(coefficients):
idx = i + 1
x = Dense(size // c, activation=activation, name="enc_%d" % idx)(x)
if use_batch_norm:
x = BatchNormalization(name="enc_%d_batch_norm" % idx)(x)
if dropout_rate > 0:
if activation == 'selu':
x = AlphaDropout(dropout_rate,
name="enc_%d_dropout" % idx)(x)
else:
x = Dropout(dropout_rate, name="enc_%d_dropout" % idx)(x)
x = Dense(encoded_size,
activation="linear",
name="encoded",
activity_regularizer=regularizers.l1(l1))(x)
for i, c in enumerate(reversed(coefficients)):
idx = len(coefficients) - i
x = Dense(size // c, activation=activation, name="dec_%d" % idx)(x)
if use_batch_norm:
x = BatchNormalization(name="dec_%d_batch_norm" % idx)(x)
if dropout_rate > 0:
if activation == 'selu':
x = AlphaDropout(dropout_rate,
name="dec_%d_dropout" % idx)(x)
else:
x = Dropout(dropout_rate, name="dec_%d_dropout" % idx)(x)
decoded = Dense(size, activation="linear",
name="decoded_conformation")(x)
autoencoder = Model(input_layer, decoded)
autoencoder.compile(optimizer=Adam(lr=0.001),
loss='mse',
metrics=['mae'])
autoencoder.summary()
return autoencoder
def getSELUClassifier(nIn, nOut, compileArgs): model = Sequential() model.add(Dense(16, input_dim=nIn, kernel_initializer='he_normal', activation='selu')) model.add(AlphaDropout(0.2)) model.add(Dense(32, kernel_initializer='he_normal', activation='selu')) model.add(AlphaDropout(0.2)) model.add(Dense(32, kernel_initializer='he_normal', activation='selu')) model.add(AlphaDropout(0.2)) model.add(Dense(32, kernel_initializer='he_normal', activation='selu')) model.add(AlphaDropout(0.2)) model.add(Dense(nOut, activation="sigmoid", kernel_initializer='glorot_normal')) model.compile(**compileArgs) return model
def cnn_model():
# create model
model = Sequential()
BatchNormalization()
model.add(
ZeroPadding2D(padding=(2, 2),
data_format=None,
input_shape=(1, 28, 28)))
model.add(Conv2D(32, (5, 5), activation='relu', use_bias=False))
BatchNormalization(axis=-1)
model.add(Conv2D(32, (5, 5), activation='relu', use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.3))
model.add(Conv2D(64, (3, 3), activation='relu', use_bias=False))
BatchNormalization(axis=-1)
model.add(Conv2D(64, (3, 3), activation='relu', use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.3))
model.add(Flatten())
BatchNormalization()
model.add(
Dense(512,
activation='selu',
kernel_initializer=lecun_uniform(seed=None),
use_bias=True,
bias_initializer=lecun_uniform(seed=None)))
BatchNormalization()
model.add(AlphaDropout(0.50))
model.add(
Dense(100,
activation='selu',
kernel_initializer=lecun_uniform(seed=None),
use_bias=True,
bias_initializer=lecun_uniform(seed=None)))
model.add(AlphaDropout(0.50))
model.add(Dense(num_classes, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
return model
def f(input_features):
conv_name_base, bn_name_base = _block_name_base(stage, block)
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
x = Conv2D(filters=filters,
kernel_size=(3, 3),
strides=transition_strides,
dilation_rate=dilation_rate,
padding="same",
kernel_initializer="lecun_normal",
kernel_regularizer=l2(1e-4),
name=conv_name_base + '2a')(input_features)
else:
x = residual_unit(filters=filters,
kernel_size=(3, 3),
strides=transition_strides,
dilation_rate=dilation_rate,
conv_name_base=conv_name_base + '2a',
bn_name_base=bn_name_base + '2a')(input_features)
if dropout is not None:
x = AlphaDropout(dropout)(x)
x = residual_unit(filters=filters,
kernel_size=(3, 3),
conv_name_base=conv_name_base + '2b',
bn_name_base=bn_name_base + '2b')(x)
return _shortcut(input_features, x)
def model_design(self):
convolution_output = []
inindex = 0
for filter_width in self.filter_sizes:
conv = Convolution1D(filters=256,
kernel_size=filter_width,
activation='relu',
name='Conv1D_{}_{}_{}'.format(
256, filter_width, inindex))(self.x)
pool = GlobalMaxPooling1D(
name='MaxPoolingOverTime_{}_{}_{}'.format(
256, filter_width, inindex))(conv)
convolution_output.append(pool)
inindex = inindex + 1
self.x = Concatenate()(convolution_output)
for fl in self.fully_connected_layers:
self.x = Dense(fl,
activation='relu',
kernel_initializer='lecun_normal')(self.x)
self.x = AlphaDropout(self.dropout_p)(self.x)
predictions = Dense(self.label_size, activation='softmax')(self.x)
self.model = Model(inputs=self.inputs, outputs=predictions)
self.model.compile(optimizer=self.optimizer, loss=self.loss)
self.model.summary()
def build_model(input_dim, hidden1, hidden2, output_dim, drpout_rate, lr):
inputs = Input(shape=(input_dim,))
dense_layer1 = Dense(name='first_dense_layer',
units=hidden1,
activation='relu')
dense_layer2 = Dense(name='second_hidden_layer',
units=hidden2,
activation='relu')
dense_layer3 = Dense(name='final_layer',
units=output_dim,
activation='softmax')
input_bn_layer = BatchNormalization(name='input_batchnorm_layer')
bn_layer1 = BatchNormalization(name='first_batchnorm_layer')
bn_layer2 = BatchNormalization(name='second_batchnorm_layer')
dropout_layer = AlphaDropout(rate=drpout_rate)
y = input_bn_layer(inputs)
y = dense_layer1(y)
y = bn_layer1(y)
y = dropout_layer(y)
y = dense_layer2(y)
y = bn_layer2(y)
y = dropout_layer(y)
outputs = dense_layer3(y)
model = Model(inputs=inputs, outputs=outputs)
adam = Adam(lr=lr)
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
return model
def base_cnn():
x = Input(shape=(IMAGE_SIZE, IMAGE_SIZE, 3))
i = Conv2D(64, (3, 3),
activation='selu',
kernel_initializer='lecun_normal',
name='conv1')(x)
i = MaxPooling2D()(i)
i = Conv2D(64, (3, 3),
activation='selu',
kernel_initializer='lecun_normal',
name='conv2')(i)
i = MaxPooling2D()(i)
i = Conv2D(128, (3, 3),
activation='selu',
kernel_initializer='lecun_normal',
name='conv3')(i)
i = MaxPooling2D()(i)
i = Conv2D(128, (3, 3),
activation='selu',
kernel_initializer='lecun_normal',
name='conv4')(i)
i = MaxPooling2D()(i)
i = Conv2D(256, (3, 3),
activation='selu',
kernel_initializer='lecun_normal',
name='conv5')(i)
i = MaxPooling2D()(i)
i = Conv2D(256, (3, 3),
activation='selu',
kernel_initializer='lecun_normal',
name='conv6')(i)
# i = Conv2D(512, (3, 3), activation='selu', kernel_initializer='lecun_normal')(i)
i = AlphaDropout(0.3)(i)
i = Flatten()(i)
return x, i
def alpha_dropout(layer, layer_in, layerId, tensor=True):
rate = layer['params']['rate']
seed = layer['params']['seed']
out = {layerId: AlphaDropout(rate=rate, seed=seed)}
if tensor:
out[layerId] = out[layerId](*layer_in)
return out
def _fully_connected_layers(num_layers, num_neurons, x):
x = _default_dense_layer(num_neurons, x)
for _ in range(num_layers // 4):
x = _dense_indentity(4, num_neurons, x)
x = _dense_indentity(max(num_layers % 4 - 1, 0), num_neurons, x)
x = AlphaDropout(0.5)(x)
return x
def _build_model(self):
"""
Build and compile the Character Level CNN model
Returns: None
"""
# Input layer
inputs = Input(shape=(self.input_size,), name='sent_input', dtype='int64')
# Embedding layers
x = Embedding(self.alphabet_size + 1, self.embedding_size, input_length=self.input_size)(inputs)
# Convolution layers
convolution_output = []
for num_filters, filter_width in self.conv_layers:
conv = Convolution1D(filters=num_filters,
kernel_size=filter_width,
activation='tanh',
name='Conv1D_{}_{}'.format(num_filters, filter_width))(x)
pool = GlobalMaxPooling1D(name='MaxPoolingOverTime_{}_{}'.format(num_filters, filter_width))(conv)
convolution_output.append(pool)
x = Concatenate()(convolution_output)
# Fully connected layers
for fl in self.fully_connected_layers:
x = Dense(fl, activation='selu', kernel_initializer='lecun_normal')(x)
x = AlphaDropout(self.dropout_p)(x)
# Output layer
predictions = Dense(self.num_of_classes, activation='softmax')(x)
# Build and compile model
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer=self.optimizer, loss=self.loss,
metrics=["accuracy"])
self.model = model
print("CharCNNKim model built: ")
self.model.summary()
def f(input):
residuals = []
for i in range(num_residuals):
residual = _norm_relu_conv(filters,
kernel_size=1,
subsample=subsample,
normalization=normalization,
weight_norm=weight_norm,
weight_decay=weight_decay,
norm_kwargs=norm_kwargs,
init=init,
nonlinearity=nonlinearity,
ndim=ndim,
name=name)(input)
residual = _norm_relu_conv(filters,
kernel_size=3,
normalization=normalization,
weight_norm=weight_norm,
weight_decay=weight_decay,
norm_kwargs=norm_kwargs,
init=init,
nonlinearity=nonlinearity,
ndim=ndim,
name=name)(residual)
residual = _norm_relu_conv(filters * 4,
kernel_size=1,
upsample=upsample,
normalization=normalization,
weight_norm=weight_norm,
weight_decay=weight_decay,
norm_kwargs=norm_kwargs,
init=init,
nonlinearity=nonlinearity,
ndim=ndim,
name=name)(residual)
if dropout > 0:
if nonlinearity == 'selu':
residual = AlphaDropout(dropout)(residual)
else:
residual = Dropout(dropout)(residual)
residiuals.append(residual)
if len(residuals) > 1:
output = merge_add(residuals)
else:
output = residuals[0]
if skip:
output = _shortcut(input,
output,
subsample=subsample,
upsample=upsample,
weight_norm=weight_norm,
normalization=normalization,
weight_decay=weight_decay,
init=init,
ndim=ndim,
name=name)
return output
def generator_model(input_size=100,
filter_width=5,
min_data_width=4,
min_conv_filters=64,
output_size=(32, 32, 1),
stride=2,
activation="relu",
output_activation="linear",
dropout_alpha=0):
"""
Creates a generator convolutional neural network for a generative adversarial network set. The keyword arguments
allow aspects of the structure of the generator to be tuned for optimal performance.
Args:
input_size (int): Number of nodes in the input layer.
filter_width (int): Width of each convolutional filter
min_data_width (int): Width of the first convolved layer after the input layer
min_conv_filters (int): Number of convolutional filters in the last convolutional layer
output_size (tuple of size 3): Dimensions of the output
stride (int): Number of pixels that the convolution filter shifts between operations.
activation (str): Type of activation used for convolutional layers. Use "leaky" for Leaky ReLU.
output_activation (str): Type of activation used on the output layer
dropout_alpha (float): proportion of nodes dropped out
Returns:
Model output graph, model input
"""
num_layers = int(np.log2(output_size[0]) - np.log2(min_data_width))
max_conv_filters = int(min_conv_filters * 2**(num_layers - 1))
curr_conv_filters = max_conv_filters
vector_input = Input(shape=(input_size, ), name="gen_input")
model = Dense(units=max_conv_filters * min_data_width * min_data_width,
kernel_regularizer=l2())(vector_input)
model = Reshape((min_data_width, min_data_width, max_conv_filters))(model)
if activation == "leaky":
model = LeakyReLU(alpha=0.2)(model)
else:
model = Activation(activation)(model)
for i in range(1, num_layers):
curr_conv_filters //= 2
model = Conv2DTranspose(curr_conv_filters,
filter_width,
strides=(stride, stride),
padding="same")(model)
if activation == "leaky":
model = LeakyReLU(alpha=0.2)(model)
else:
model = Activation(activation)(model)
if activation == "selu":
model = AlphaDropout(dropout_alpha)(model)
else:
model = Dropout(dropout_alpha)(model)
model = Conv2DTranspose(output_size[-1],
filter_width,
strides=(stride, stride),
padding="same")(model)
model = Activation(output_activation)(model)
return model, vector_input
def encoder_disc_model(input_size=(32, 32, 1),
filter_width=5,
min_data_width=4,
min_conv_filters=64,
output_size=100,
stride=2,
activation="relu",
encoder_output_activation="linear",
dropout_alpha=0):
"""
Creates an encoder/discriminator convolutional neural network that reproduces the generator input vector.
The keyword arguments allow aspects of the structure of the enocder/discriminator to be tuned
for optimal performance.
Args:
input_size (tuple of ints): Number of nodes in the input layer.
filter_width (int): Width of each convolutional filter
min_data_width (int): Width of the last convolved layer
min_conv_filters (int): Number of convolutional filters in the first convolutional layer
output_size (int): Dimensions of the output
stride (int): Number of pixels that the convolution filter shifts between operations.
activation (str): Type of activation used for convolutional layers. Use "leaky" for Leaky ReLU.
encoder_output_activation (str): Type of activation used on the output layer
dropout_alpha (float): Proportion of nodes dropped out during training.
Returns:
discriminator model output, encoder model output, image input
"""
num_layers = int(np.log2(input_size[0]) - np.log2(min_data_width))
curr_conv_filters = min_conv_filters
image_input = Input(shape=input_size, name="enc_input")
model = image_input
for c in range(num_layers):
model = Conv2D(curr_conv_filters,
filter_width,
strides=(stride, stride),
padding="same")(model)
if activation == "leaky":
model = LeakyReLU(0.2)(model)
else:
model = Activation(activation)(model)
if activation == "selu":
model = AlphaDropout(dropout_alpha)(model)
else:
model = Dropout(dropout_alpha)(model)
curr_conv_filters *= 2
model = Flatten()(model)
enc_model = Dense(256, kernel_regularizer=l2())(model)
if activation == "leaky":
enc_model = LeakyReLU(0.2)(enc_model)
else:
enc_model = Activation(activation)(enc_model)
enc_model = Dense(output_size, kernel_regularizer=l2())(enc_model)
enc_model = Activation(encoder_output_activation)(enc_model)
disc_model = Dense(1, kernel_regularizer=l2())(model)
disc_model = Activation("sigmoid")(disc_model)
return disc_model, enc_model, image_input
def add_drops(model, drop_out, k):
if DG[k].upper() == 'D':
model.add(Dropout(drop_out[0]))
elif DG[k].upper() == 'G':
model.add(GaussianNoise(drop_out[k]))
elif DG[k].upper() == "A":
model.add(AlphaDropout(drop_out[k]))
else:
pass
return model
def up(input_layer, residual, filters):
filters = int(filters)
upsample = UpSampling2D()(input_layer)
upconv = Conv2D(filters, (2, 2), padding="same")(upsample)
concat = Concatenate(axis=3)([residual, upconv])
# drop = Dropout(0.3)(concat)
drop = AlphaDropout(rate=0.3)(concat)
conv1 = Conv2D(filters, (3, 3), padding='same', activation='elu')(drop)
conv2 = Conv2D(filters, (3, 3), padding='same', activation='elu')(conv1)
return conv2
def build_sequential_model(input_rate, rate, shape):
model = Sequential()
model.add(AlphaDropout(input_rate, input_shape=(shape,)))
model.add(Dense(6, activation="linear", kernel_initializer="lecun_normal"))
model.add(Activation('selu'))
model.add(AlphaDropout(rate))
model.add(Dense(3, activation="linear", kernel_initializer="lecun_normal"))
model.add(Activation('selu'))
model.add(AlphaDropout(rate))
model.add(Dense(units=1, activation="linear", kernel_initializer="lecun_normal"))
optim = Adam(lr=0.01, beta_1=0.95)
model.compile(loss='mean_squared_error',
optimizer=optim)
return model
def double_conv_layer(x, size, dropout, batch_norm):
conv = Conv2D(size, (3, 3), padding='same')(x)
if batch_norm is True:
conv = BatchNormalization()(conv)
conv = Activation('selu')(conv)
conv = Conv2D(size, (3, 3), padding='same')(conv)
if batch_norm is True:
conv = BatchNormalization()(conv)
conv = Activation('selu')(conv)
if dropout > 0:
conv = AlphaDropout(dropout)(conv)
return conv
def get_model_selu(input_size, output, crossentropy=True):
model = Sequential()
model.add(
Dense(500,
input_shape=(input_size, ),
bias_initializer='zeros',
kernel_initializer=keras.initializers.lecun_normal()))
model.add(Activation('selu'))
model.add(BatchNormalization(axis=1))
model.add(AlphaDropout(0.25))
for index in range(5):
model.add(
Dense(500,
bias_initializer='zeros',
kernel_initializer=keras.initializers.lecun_normal()))
model.add(Activation('selu'))
model.add(BatchNormalization(axis=1))
model.add(AlphaDropout(0.25))
model.add(
Dense(500,
bias_initializer='zeros',
kernel_initializer=keras.initializers.lecun_normal()))
model.add(Activation('selu'))
model.add(BatchNormalization(axis=1))
model.add(AlphaDropout(0.5))
model.add(
Dense(output,
bias_initializer='zeros',
kernel_initializer=keras.initializers.lecun_normal()))
if crossentropy:
model.add(Activation("softmax"))
else:
model.add(Activation("sigmoid"))
model.summary()
return model
def prepare_processor(input_dim, output_dim, hidden_dim, dense_depth,
optimizer_type, loss_type) -> Model:
"""Prepares the processing LSTM decoder network
Args:
:param input_dim: The dimensions of the input
:param output_dim: The dimensions of the output
:param output_timesteps: The number of timesteps to predict for the output
:param hidden_dim: The dimensionality of internal network layers
:param dense_depth: The number of dense hidden layers
:param lstm_depth: The number of LSTM hidden layers
:param optimizer_type: The type of optimizer to use (eg. "rmsprop")
:param loss_type: The type of loss to use (eg. "mse")
Returns:
Processing decoder model
"""
model = Sequential()
model.add(
Dense(input_dim,
input_shape=(input_dim, ),
activation="selu",
kernel_initializer='lecun_normal'))
model.add(AlphaDropout(0.2))
for i in range(dense_depth):
model.add(
Dense(hidden_dim,
activation="selu",
kernel_initializer='lecun_normal'))
model.add(AlphaDropout(0.2))
model.add(Dense(output_dim, activation="softmax"))
model.compile(optimizer=optimizer_type, loss=loss_type, metrics=['acc'])
print(model.summary())
return model
def build_sequential_model(input_rate, rate, shape):
model = Sequential()
model.add(AlphaDropout(input_rate, input_shape=(shape, )))
model.add(Dense(6, activation="linear", kernel_initializer="lecun_normal"))
#model.add(BatchNormalization())
model.add(Activation('selu'))
model.add(AlphaDropout(rate))
model.add(Dense(3, activation="linear", kernel_initializer="lecun_normal"))
#model.add(BatchNormalization())
model.add(Activation('selu'))
model.add(AlphaDropout(rate))
model.add(
Dense(activation="sigmoid", units=1,
kernel_initializer="lecun_normal"))
optim = Adam(lr=0.01, beta_1=0.95)
model.compile(loss='binary_crossentropy', optimizer=optim)
return model
def define_model_1(conv_layers,
fully_connected_layers,
input_size=MAX_INPUT_LEN,
embedding_size=32,
alphabet_size=ALPHABET_SIZE,
num_classes=2,
optimizer='adam',
dropout_proba=0.5,
fl_activation='selu',
fl_initializer='lecun_normal',
conv_activations='tanh',
loss='categorical_crossentropy'):
"""
Based on: https://arxiv.org/abs/1508.06615
"""
inputs = Input(shape=(input_size, ), name='input_layer', dtype='int64')
embeds = Embedding(alphabet_size + 1,
embedding_size,
input_length=input_size)(inputs)
convs = list()
for num_filters, filter_width in conv_layers:
conv = Convolution1D(filters=num_filters,
kernel_size=filter_width,
activation=conv_activations,
name='ConvLayer{}{}'.format(
num_filters, filter_width))(embeds)
pool = GlobalMaxPooling1D(
name='MaxPoolLayer{}{}'.format(num_filters, filter_width))(conv)
convs.append(pool)
x = Concatenate()(convs)
for units in fully_connected_layers:
x = Dense(units,
activation=fl_activation,
kernel_initializer=fl_initializer)(x)
x = AlphaDropout(dropout_proba)(x)
predictions = Dense(num_classes, activation='softmax')(x)
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
return model
def f(input):
residuals = []
for i in range(num_residuals):
residual = input
if normalization is not None:
residual = normalization(name=name + "_norm_" + str(i),
**norm_kwargs)(residual)
residual = get_nonlinearity(nonlinearity)(residual)
if subsample:
residual = MaxPooling(pool_size=2, ndim=ndim)(residual)
residual = Convolution(filters=filters,
kernel_size=3,
ndim=ndim,
weight_norm=weight_norm,
kernel_initializer=init,
padding='same',
kernel_regularizer=_l2(weight_decay),
name=name + "_conv2d_" + str(i))(residual)
if dropout > 0:
if nonlinearity == 'selu':
residual = AlphaDropout(dropout)(residual)
else:
residual = Dropout(dropout)(residual)
if upsample:
residual = UpSampling(size=2, ndim=ndim)(residual)
residuals.append(residual)
if len(residuals) > 1:
output = merge_add(residuals)
else:
output = residuals[0]
if skip:
output = _shortcut(input,
output,
subsample=subsample,
upsample=upsample,
normalization=normalization,
weight_norm=weight_norm,
weight_decay=weight_decay,
init=init,
ndim=ndim,
name=name)
return output
def create_snn_classif(input_size,
n_classes,
layer_sizes=None,
actFunc='selu',
learning_rate=0.001,
alpha_dropout_rate=0,
nb_epoch=200,
loss='categorical_crossentropy',
metrics=None,
optimizer='adam',
gpu_device='/gpu:0'):
if layer_sizes is None:
layer_sizes = []
if metrics is None:
metrics = ['accuracy']
model = Sequential()
input_dim = input_size
for i, dim in enumerate(layer_sizes):
with tf.device(gpu_device):
if i == 0:
model.add(
Dense(dim,
input_dim=input_dim,
kernel_initializer='lecun_normal'))
else:
model.add(Dense(dim, kernel_initializer='lecun_normal'))
with tf.device(gpu_device):
model.add(Activation(actFunc))
if alpha_dropout_rate > 0:
model.add(AlphaDropout(alpha_dropout_rate))
with tf.device(gpu_device):
model.add(Dense(n_classes, activation='softmax'))
# Compile model
optimizer = define_optimizer(optimizer_type=optimizer, lr=learning_rate)
model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
return model
