def get_mlp(input_shape, n_outputs):
from tensorflow import keras
import tensorflow.keras.layers as layers
inputs = layers.Input(shape=input_shape)
# x = layers.Dense(500, kernel_initializer="he_normal")(inputs)
# x = layers.LeakyReLU(alpha=0.2)(x)
# x = layers.Dense(500, kernel_initializer="he_normal")(x)
# x = layers.LeakyReLU(alpha=0.2)(x)
x = layers.Dense(500,
kernel_initializer="he_normal",
kernel_regularizer=keras.regularizers.l2(0.01),
use_bias=False)(inputs)
x = layers.BatchNormalization()(x)
x = layers.ELU()(x)
x = layers.Dense(500,
kernel_initializer="he_normal",
kernel_regularizer=keras.regularizers.l2(0.01),
use_bias=False)(x)
x = layers.BatchNormalization()(x)
x = layers.ELU()(x)
# x = layers.Dense(50, activation="relu")(inputs)
# x = layers.Dense(10, activation="relu")(x)
outputs = layers.Dense(n_outputs)(x)
model = keras.Model(inputs=[inputs], outputs=[outputs])
return model
def __init__(self,
in_channels,
out_channels,
data_format="channels_last",
**kwargs):
super(LwopEncoderFinalBlock, self).__init__(**kwargs)
self.pre_conv = conv1x1_block(in_channels=in_channels,
out_channels=out_channels,
use_bias=True,
use_bn=False,
data_format=data_format,
name="pre_conv")
self.body = SimpleSequential(name="body")
for i in range(3):
self.body.add(
dwsconv3x3_block(in_channels=out_channels,
out_channels=out_channels,
use_bn=False,
dw_activation=(lambda: nn.ELU()),
pw_activation=(lambda: nn.ELU()),
data_format=data_format,
name="block{}".format(i + 1)))
self.post_conv = conv3x3_block(in_channels=out_channels,
out_channels=out_channels,
use_bias=True,
use_bn=False,
data_format=data_format,
name="post_conv")
def conv_layer(x):
import tensorflow.keras.layers as layers
from tensorflow import keras
# x = layers.Conv2D(32, 8, kernel_initializer=initializer, strides=4, activation='relu')(x)
# x = layers.Conv2D(64, 4, kernel_initializer=initializer, strides=2, activation='relu')(x)
# x = layers.Conv2D(64, 3, kernel_initializer=initializer, strides=1, activation='relu')(x)
initializer = keras.initializers.VarianceScaling(
scale=2.0, mode='fan_in', distribution='truncated_normal')
x = layers.Conv2D(64, 5, kernel_initializer=initializer, padding='same')(x)
x = layers.BatchNormalization()(x)
x = layers.ELU()(x)
x = layers.Conv2D(64, 3, kernel_initializer=initializer, padding='valid')(x)
x = layers.BatchNormalization()(x)
x = layers.ELU()(x)
x = layers.Conv2D(128, 3, kernel_initializer=initializer, padding='valid')(x)
x = layers.BatchNormalization()(x)
x = layers.ELU()(x)
x = layers.Conv2D(128, 3, kernel_initializer=initializer, padding='valid')(x)
x = layers.BatchNormalization()(x)
x = layers.ELU()(x)
return x
def mlp_layer(x):
from tensorflow import keras
import tensorflow.keras.layers as layers
# initializer = "he_normal"
# x = layers.Dense(512, kernel_initializer=initializer, activation='relu')(x)
initializer = keras.initializers.VarianceScaling(
scale=2.0, mode='fan_in', distribution='truncated_normal')
x = layers.Dense(1000, kernel_initializer=initializer,
kernel_regularizer=keras.regularizers.l2(0.01),
use_bias=False)(x)
x = layers.BatchNormalization()(x)
x = layers.ELU()(x)
# x = layers.ReLU()(x)
x = layers.Dense(1000, kernel_initializer=initializer,
kernel_regularizer=keras.regularizers.l2(0.01),
use_bias=False)(x)
x = layers.BatchNormalization()(x)
x = layers.ELU()(x)
# x = layers.ReLU()(x)
return x
def __init__(self, params):
name = params['model_name']
super(discrete_10000_ec_4, self).__init__(name=name)
# float
dropout_rate = params['dropout_rate']
# list with 2 element: start and end
# [0:1] = V
# [1:2] = W
# [0:2] = V and W
self.pchoice = params['pchoice']
self.d = layers.Conv1D(1, 2, strides=1, padding='same')
self.d2 = layers.Conv1D(1, 3, strides=1, padding='same')
self.semilocal = layers.Conv1D(1, 16, strides=1, padding='same')
self.decider = keras.Sequential()
self.decider.add(layers.Conv1D(6, 2, strides=1, padding='same'))
self.decider.add(layers.ELU())
self.decider.add(layers.Dropout(rate=dropout_rate))
self.decider.add(layers.Conv1D(7, 4, strides=1, padding='same'))
self.decider.add(layers.LeakyReLU())
self.decider.add(layers.Dropout(rate=dropout_rate))
self.decider.add(layers.Conv1D(8, 6, strides=1, padding='same'))
self.counters = []
for i in range(1, 8):
model = keras.Sequential()
model.add(layers.Conv1D(8, 2**i, 2**(i - 1), padding='same'))
model.add(layers.ELU())
model.add(layers.Dropout(rate=dropout_rate))
pool_size = (10000 + 2**(i - 1) - 1) // 2**(i - 1)
model.add(layers.AveragePooling1D(pool_size=pool_size))
model.add(layers.LeakyReLU())
self.counters.append(model)
self.conv_down = layers.Dense(1)
self.weighters = keras.Sequential()
self.weighters.add(layers.BatchNormalization())
self.weighters.add(layers.Dense(21))
self.weighters.add(layers.ELU())
self.weighters.add(layers.Dropout(rate=dropout_rate))
self.weighters.add(layers.Dense(14))
self.weighters.add(layers.LeakyReLU())
self.weighters.add(layers.Dropout(rate=dropout_rate))
self.weighters.add(layers.Dense(7))
self.weighters.add(layers.Activation('softmax'))
self.out_layer = layers.Dense(1)
def __init__(self, **kwargs):
super(Q, self).__init__(**kwargs)
self.dim_reduction1 = FFC(out_length=2, groups=2, as_matrix=False)
self.layer_normal1 = layers.LayerNormalization()
self.activation1 = layers.ELU()
self.dim_reduction2 = FFC(out_length=1, groups=1, as_matrix=False)
self.layer_normal2 = layers.LayerNormalization()
self.activation2 = layers.ELU()
self.reshape = None
self.squeeze = None
self.relu = layers.ReLU()
self.excitation = None
self.reverse_reshape = None
self.layer_normal3 = layers.LayerNormalization()
def create_model(trainable=True):
image_input = Input(shape=(160, 120, 3))
model = MobileNetV2(input_tensor=image_input, include_top=False, alpha=ALPHA, weights = None)
last_layer = model.layers[-1].output
x = GlobalAveragePooling2D()(last_layer)
x = layers.GaussianDropout(0.3)(x)
x = Dense(512, activation=layers.ELU(alpha=1.0), name='fc1')(x)
x = layers.GaussianDropout(0.1)(x)
x = Dense(64, activation=layers.ELU(alpha=1.0), name='fc2')(x)
x = layers.GaussianDropout(0.05)(x)
x = Dense(NUM_CLASSES, activation='linear', name='output')(x)
return Model(inputs=model.input, outputs=x)
def __init__(self, params):
name = params['model_name']
super(cnn_landscape, self).__init__(name=name)
# choice of potential
self.pchoice = params['pchoice']
# residual
self.res = keras.Sequential(name='residual')
self.res.add(layers.Conv1D(self.pchoice[1]-self.pchoice[0], 20, strides=1, padding='same'))
self.res.add(layers.ELU())
self.res.add(layers.Conv1D(self.pchoice[1]-self.pchoice[0], 20, strides=1, padding='same'))
self.res.add(layers.LeakyReLU())
self.res.add(layers.Dropout(rate=0.1))
# embedding layers
self.emb = keras.Sequential(name='embedding')
self.emb.add(layers.BatchNormalization())
self.emb.add(layers.Conv1D(5, 50, strides=5, padding='same'))
self.emb.add(layers.ELU())
self.emb.add(layers.Dropout(rate=0.1))
self.emb.add(layers.Conv1D(5, 50, strides=5, padding='same'))
self.emb.add(layers.LeakyReLU())
self.emb.add(layers.Dropout(rate=0.1))
# reduction of input
self.conv1 = layers.Conv1D(5, 50, strides=25, padding='same')
self.weighters = keras.Sequential(name='integral_weight')
self.weighters.add(layers.Conv1D(10, 20, strides=10, padding='same'))
self.weighters.add(layers.ELU())
self.weighters.add(layers.Dropout(rate=0.1))
self.weighters.add(layers.Conv1D(20, 20, strides=10, padding='same'))
self.weighters.add(layers.LeakyReLU())
self.weighters.add(layers.Dropout(rate=0.1))
self.weighters.add(layers.Conv1D(30, 4, strides=4, padding='same'))
self.weighters.add(layers.Flatten())
self.out_layer1 = keras.Sequential(name='out_layer1')
self.out_layer1.add(layers.Dense(30))
self.out_layer1.add(layers.LeakyReLU())
self.out_layer1.add(layers.Dense(30))
self.out_layer1.add(layers.LeakyReLU())
self.out_layer2 = keras.Sequential(name='out_layer2')
self.out_layer2.add(layers.Dense(10))
self.out_layer2.add(layers.LeakyReLU())
self.out_layer2.add(layers.Dense(1))
def do_activation(input_values, function_name, alpha=0.2):
"""Runs input array through activation function.
:param input_values: numpy array (any shape).
:param function_name: Name of activation function (must be accepted by ``).
:param alpha: Slope parameter (alpha) for activation function. This applies
only for eLU and ReLU.
:return: output_values: Same as `input_values` but post-activation.
"""
architecture_utils.check_activation_function(
activation_function_string=function_name,
alpha_for_elu=alpha,
alpha_for_relu=alpha)
input_object = K.placeholder()
if function_name == architecture_utils.ELU_FUNCTION_STRING:
function_object = K.function([input_object],
[layers.ELU(alpha=alpha)(input_object)])
elif function_name == architecture_utils.RELU_FUNCTION_STRING:
function_object = K.function(
[input_object], [layers.LeakyReLU(alpha=alpha)(input_object)])
else:
function_object = K.function(
[input_object], [layers.Activation(function_name)(input_object)])
return function_object([input_values])[0]
def __init__(self, num_actions):
super().__init__()
self.conv1 = layers.Conv2D(
32,
8,
4,
padding="same",
input_shape=((84, 84, stack_size)),
kernel_initializer=tf.keras.initializers.glorot_normal())
self.activation = layers.ELU()
self.conv2 = layers.Conv2D(
64,
4,
2,
padding="same",
kernel_initializer=tf.keras.initializers.glorot_normal())
self.conv3 = layers.Conv2D(
128,
2,
2,
padding="valid",
kernel_initializer=tf.keras.initializers.glorot_normal())
self.normalization = layers.BatchNormalization(epsilon=1e-5,
name="batch_norm")
self.normalization1 = layers.BatchNormalization(epsilon=1e-5,
name="batch_norm")
self.flatten = layers.Flatten()
self.dense1 = layers.Dense(512, activation="relu")
self.actor = layers.Dense(num_actions)
self.critic = layers.Dense(1)
def __init__(self, rate=2, matrix_shape=(4, 4), regularize=1e-5, **kwargs):
super(CondenseTiny, self).__init__(**kwargs)
self.sparse_extraction = PartialMatrix(rate=rate,
matrix_shape=matrix_shape,
regularize=regularize)
self.normal = layers.LayerNormalization()
self.activation = layers.ELU()
def build_model_Dense():
model = keras.Sequential()
model.add(
layers.Dense(200,
input_dim=375,
kernel_initializer=keras.initializers.Zeros()))
model.add(layers.Activation('sigmoid'))
model.add(layers.Dense(185))
model.add(layers.Activation('sigmoid'))
model.add(layers.Dense(92))
model.add(layers.ELU(alpha=1.0))
model.add(layers.BatchNormalization())
model.add(layers.Dense(1, activation='relu'))
optimizer = keras.optimizers.Adam(lr=0.0001,
beta_1=0.99,
beta_2=0.999,
epsilon=0.01,
decay=0.01,
amsgrad=False)
model.compile(loss='mean_absolute_error',
optimizer=optimizer,
metrics=['acc'])
model.summary()
return model
def get_activation_fn(activation: str, name: str) -> tf.Tensor:
"""Returns the activation function"""
if activation == 'relu':
return layers.ReLU(name=name+'_relu')
if activation == 'elu':
return layers.ELU(name=name+'_elu')
raise ValueError('Unknown activation function')
def __init__(self, **kwargs):
super(CapsSimilarity, self).__init__(**kwargs)
self.layer_normal1 = layers.LayerNormalization()
# self.dot = layers.Dot((2, 2), normalize=True)
self.dot = layers.Dot((2, 2))
self.layer_normal2 = layers.LayerNormalization()
self.activation = layers.ELU()
def create_wdmodel(n_cat_input, n_num_input, hidden_layer, num_classes):
input_layer1 = tf.keras.Input(shape=(n_num_input, ))
hidden = layers.Dense(
hidden_layer[0],
kernel_regularizer=regularizers.l2(0.1),
activity_regularizer=regularizers.l1(0.1))(input_layer1)
hidden = layers.ReLU(max_value=None, negative_slope=0.0,
threshold=0.0)(hidden)
hidden = layers.Dropout(0.1)(hidden)
hidden = layers.Dense(hidden_layer[1],
kernel_regularizer=regularizers.l2(0.1),
activity_regularizer=regularizers.l1(0.1))(hidden)
hidden = layers.ReLU(max_value=None, negative_slope=0.0,
threshold=0.0)(hidden)
hidden = layers.Dropout(0.1)(hidden)
hidden = layers.Dense(hidden_layer[2],
kernel_regularizer=regularizers.l2(0.1),
activity_regularizer=regularizers.l1(0.1))(hidden)
hidden = layers.ReLU(max_value=None, negative_slope=0.0,
threshold=0.0)(hidden)
hidden = layers.Dropout(0.1)(hidden)
if num_classes > 2:
output_layer1 = layers.Dense(num_classes, activation="softmax")(hidden)
else:
output_layer1 = layers.Dense(1, activation="sigmoid")(hidden)
input_layer2 = tf.keras.Input(shape=(n_cat_input, ))
output_layer2 = layers.ELU(alpha=1)(input_layer2)
inputs = []
inputs.append(input_layer1)
inputs.append(input_layer2)
outputs = []
outputs.append(output_layer1)
outputs.append(output_layer2)
concat = layers.Concatenate()(outputs)
if num_classes > 2:
model_out = layers.Dense(num_classes, activation="softmax")(concat)
else:
model_out = layers.Dense(1, activation="sigmoid")(concat)
#model_out = layers.Dense( 2, activation = "softmax" )( concat )
model = models.Model(inputs, model_out)
if num_classes > 2:
loss_func = 'sparse_categorical_crossentropy'
else:
loss_func = 'binary_crossentropy'
opt = optimizers.Adam(learning_rate=0.005,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
model.compile(optimizer=opt, loss=loss_func, metrics=['accuracy'])
return model
def get_activation(name):
"""
Parses a string to extract the activation function.
"""
string = name.split(':')
if string[0] == 'RELU':
activation = Layers.ReLU()
elif string[0] == 'TANH':
activation = Layers.Activation(tf.keras.activations.tanh)
elif string[0] == 'SIGMOID':
activation = Layers.Activation(tf.keras.activations.sigmoid)
elif string[0] == 'LRELU':
try:
activation = Layers.LeakyReLU(float(string[1]))
except:
warnings.warn('Using default alpha parameter : 0.1.',
SyntaxWarning)
warnings.warn(
'To set a custom alpha value type LRELU:alpha instead of LRELU',
SyntaxWarning)
activation = Layers.LeakyReLU(0.1)
elif string[0] == 'PRELU':
activation = Layers.PReLU()
elif string[0] == 'ELU':
try:
activation = Layers.ELU(float(string[1]))
except:
warnings.warn('Using default alpha parameter : 1.0.',
SyntaxWarning)
warnings.warn(
'To set a custom alpha value type LRELU:alpha instead of LRELU',
SyntaxWarning)
activation = Layers.ELU(1.0)
elif string[0] == 'SELU':
activation = Layers.Activation(tf.keras.activations.selu)
elif string[0] == 'SWISH':
activation = swish()
elif string[0] == 'CSWISH':
activation = Layers.Activation(cswish)
elif string[0] == 'MISH':
activation = Layers.Activation(mish)
else:
raise ValueError('error: unknown activation function')
return activation
def __init__(self,
num_capsule: int,
matrix_shape: tuple = (4, 4),
regularize=1e-4,
**kwargs):
super(Condense, self).__init__(**kwargs)
self.sparse_extraction = GlobalMatrix(num_capsule,
matrix_shape,
regularize=regularize)
self.normal = layers.LayerNormalization()
self.activation = layers.ELU()
def create_model(num_input, hidden_layer, num_classes):
input_layer = tf.keras.Input(shape=(num_input, ))
hidden = layers.Dense(
hidden_layer[0],
kernel_regularizer=regularizers.l2(0.1),
activity_regularizer=regularizers.l1(0.1))(input_layer)
hidden = layers.ELU(alpha=1)(hidden)
hidden = layers.Dropout(0.1)(hidden)
hidden = layers.Dense(hidden_layer[1],
kernel_regularizer=regularizers.l2(0.1),
activity_regularizer=regularizers.l1(0.1))(hidden)
hidden = layers.ELU(alpha=1)(hidden)
hidden = layers.Dropout(0.1)(hidden)
hidden = layers.Dense(hidden_layer[2],
kernel_regularizer=regularizers.l2(0.1),
activity_regularizer=regularizers.l1(0.1))(hidden)
hidden = layers.ELU(alpha=1)(hidden)
hidden = layers.Dropout(0.1)(hidden)
if num_classes > 2:
output_layer = layers.Dense(num_classes, activation="softmax")(hidden)
else:
output_layer = layers.Dense(1, activation="sigmoid")(hidden)
model = models.Model(input_layer, output_layer)
if num_classes > 2:
loss_func = 'sparse_categorical_crossentropy'
else:
loss_func = 'binary_crossentropy'
opt = optimizers.Adam(learning_rate=0.005,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
model.compile(optimizer=opt, loss=loss_func, metrics=['accuracy'])
return model
def createModelIII(reg1=1e-3, reg2=1e-3, dp=0.3, learning_rate=1e-4):
tsteps = 48
model = models.Sequential()
model.add(
layers.LSTM(32,
input_shape=[tsteps, 6],
return_sequences=True,
dropout=0.2,
use_bias=True))
model.add(
layers.TimeDistributed(
layers.Dense(tsteps, activation='relu', use_bias=True)))
model.add(layers.Reshape([tsteps, tsteps, 1]))
model.add(layers.Conv2D(16, (3, 3), padding="same", activation="relu"))
model.add(layers.Conv2D(32, (7, 7), padding="same", activation="relu"))
model.add(layers.Flatten())
model.add(
layers.Dense(64, kernel_regularizer=tf.keras.regularizers.l2(reg1)))
model.add(layers.ELU())
model.add(layers.Dropout(dp))
model.add(
layers.Dense(14,
activation='softmax',
kernel_regularizer=tf.keras.regularizers.l2(reg2)))
lr_decayed_fn = (tf.keras.experimental.CosineDecayRestarts(
learning_rate, 10))
opt = tf.keras.optimizers.Adam(learning_rate=lr_decayed_fn,
amsgrad=True,
beta_1=0.86,
beta_2=0.98,
epsilon=1e-9)
opt = tf.keras.optimizers.RMSprop(learning_rate=learning_rate, )
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
def __init__(self,
out_length,
rate=2,
strides=2,
regularize=1e-5,
**kwargs):
super(CondenseTiny, self).__init__(**kwargs)
self.sparse_extraction = layers.Conv1D(
out_length,
rate,
strides=strides,
use_bias=False,
kernel_regularizer=regularizers.L2(regularize))
self.normal = layers.LayerNormalization()
self.activation = layers.ELU()
def __init__(self,
num_hiddenlayers=3,
num_neurons=64,
activationlayer=kl.ELU()):
super().__init__('mlp_policy')
self.num_hiddenlayers = num_hiddenlayers
self.activationlayer = activationlayer
self.hidden2 = kl.Dense(num_neurons) #hidden layer for state-value
self.hidden22 = kl.Dense(num_neurons)
if self.num_hiddenlayers > 2:
self.hidden222 = kl.Dense(num_neurons)
if self.num_hiddenlayers > 3:
self.hidden2222 = kl.Dense(num_neurons)
self.val = kl.Dense(1, name='value')
def __init__(self,
n_classes,
channels=100,
kernel_size=1,
classifier_layers=3,
rate=0.0,
**kwargs):
super().__init__(**kwargs)
# define the layers
self.n_classes = n_classes
self.channels = channels
self.kernel_size = kernel_size
self.classifier_layers = classifier_layers
self.rate = rate
self.initial_conv1d = layers.Conv1D(channels, kernel_size)
self.inner_conv1d = layers.Conv1D(channels, kernel_size)
self.elu = layers.ELU()
self.outer_conv1d = layers.Conv1D(n_classes, kernel_size)
self.dropout = layers.Dropout(rate)
def __init__(self, params):
super(cnn_dense_0, self).__init__(name=params['model_name'])
self.pchoice = params['pchoice']
self.internal = tf.keras.Sequential(name=params['model_name'] +
'_layers')
self.internal.add(layers.BatchNormalization())
self.internal.add(layers.Conv1D(4, 20, strides=10, padding='same'))
self.internal.add(layers.ELU())
self.internal.add(layers.Dropout(0.1))
self.internal.add(layers.BatchNormalization())
self.internal.add(layers.Conv1D(8, 20, strides=10, padding='same'))
self.internal.add(layers.ELU(alpha=2.0))
self.internal.add(layers.Dropout(0.1))
self.internal.add(layers.BatchNormalization())
self.internal.add(layers.Conv1D(12, 20, strides=10, padding='same'))
self.internal.add(layers.ELU(alpha=4.0))
self.internal.add(layers.Dropout(0.1))
self.internal.add(layers.BatchNormalization())
self.internal.add(layers.Conv1D(32, 10, strides=10, padding='same'))
self.internal.add(layers.ELU(alpha=8.0))
self.internal.add(layers.Dropout(0.1))
self.internal.add(layers.Flatten())
self.internal.add(layers.Dense(32))
self.internal.add(layers.ELU(alpha=10.0))
self.internal.add(layers.Dropout(0.05))
self.internal.add(layers.Dense(16))
self.internal.add(layers.ELU(alpha=10.0))
self.internal.add(layers.Dense(1))
SPEC_SHAPE = (257, 384) # (height, width)
# Initializers
initializers = {
'glorot': tf.initializers.GlorotNormal(seed=RANDOM_SEED),
'he': k.initializers.he_normal(seed=RANDOM_SEED),
'random': k.initializers.RandomNormal(mean=0.0,
stddev=0.05,
seed=RANDOM_SEED),
'constant': k.initializers.Constant(value=0)
}
# Activations
activations = {
'relu': l.ReLU(max_value=None, negative_slope=0.0, threshold=0.0),
'elu': l.ELU(alpha=1.0),
'lrelu': l.LeakyReLU(alpha=0.3)
}
def resBlock(net_in,
filters,
kernel_size,
stride=1,
preactivated=True,
block_id=1,
name=''):
# Show input shape
print(' ' + name + ' IN SHAPE:', net_in.shape, end=' ')
def make_cnn_model(x_train,
y_train,
x_test,
y_test,
reg=0.001,
alpha=.7,
learning_rate=0.001,
dropout=0.5,
epochs=100,
relative_size=1.0,
optim='SGD'):
#policy = mixed_precision.Policy('mixed_float16')
#mixed_precision.set_policy(policy)
x_train = x_train.transpose((0, 2, 1))[:, :, :, None]
x_test = x_test.transpose((0, 2, 1))[:, :, :, None]
y_train -= 769
y_test -= 769
print(x_train.shape)
model = keras.models.Sequential()
size = int(25 * relative_size)
conv1 = layers.Conv2D(size,
kernel_size=(10, 1),
strides=1,
kernel_regularizer=regularizers.l2(reg))
conv2 = layers.Conv2D(size,
kernel_size=(1, 22),
kernel_regularizer=regularizers.l2(reg))
perm1 = layers.Permute((1, 3, 2))
pool1 = layers.AveragePooling2D(pool_size=(3, 1))
drop1 = layers.Dropout(dropout)
model.add(conv1)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(conv2)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(perm1)
model.add(pool1)
model.add(drop1)
conv3 = layers.Conv2D(2 * size,
kernel_size=(10, size),
kernel_regularizer=regularizers.l2(reg))
model.add(layers.ELU(alpha))
perm2 = layers.Permute((1, 3, 2))
pool2 = layers.AveragePooling2D(pool_size=(3, 1))
drop2 = layers.Dropout(dropout)
model.add(conv3)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(perm2)
model.add(pool2)
model.add(drop2)
conv4 = layers.Conv2D(4 * size,
kernel_size=(10, 2 * size),
kernel_regularizer=regularizers.l2(reg))
perm3 = layers.Permute((1, 3, 2))
pool3 = layers.AveragePooling2D(pool_size=(3, 1))
drop3 = layers.Dropout(dropout)
model.add(conv4)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(perm3)
model.add(pool3)
model.add(drop3)
conv5 = layers.Conv2D(8 * size,
kernel_size=(10, 4 * size),
kernel_regularizer=regularizers.l2(reg))
perm4 = layers.Permute((1, 3, 2))
pool4 = layers.AveragePooling2D(pool_size=(3, 1))
drop4 = layers.Dropout(dropout)
model.add(conv5)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(perm4)
model.add(pool4)
model.add(drop4)
model.add(layers.Flatten())
model.add(layers.Dense(4, name='dense_logits'))
model.add(layers.Activation('softmax', dtype='float32',
name='predictions'))
if optim == 'Adam':
optimizer = keras.optimizers.Adam(learning_rate,
beta_1=0.85,
beta_2=0.92,
amsgrad=True)
elif optim == 'RMSprop':
optimizer = keras.optimizers.RMSprop(learning_rate)
else:
optimizer = keras.optimizers.SGD(learning_rate, nesterov=True)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train,
y_train,
batch_size=20,
epochs=epochs,
validation_split=0.2,
verbose=1)
test_scores = model.evaluate(x_test, y_test, verbose=2)
print('Test loss:', test_scores[0])
print('Test accuracy:', test_scores[1])
plot(history)
return model
def make_vae_model(x_train,
y_train,
x_test,
y_test,
reg=0.001,
alpha=.7,
learning_rate=0.001,
dropout=0.5,
epochs=100,
relative_size=1.0,
optim='SGD'):
y_train -= 769
y_test -= 769
# latent_dim should be much smaller, but right now its equal to the original cnn input size
latent_dim = 500 * 22 # 2
original_dim = 22000
intermediate_dim = 512
batch_size = 100
epsilon_std = 1.0
x_train = x_train.reshape(-1, original_dim)
train_mean = np.mean(x_train)
train_std = np.std(x_train)
norm_x_train = (x_train - train_mean) / train_std
x_test = x_test.reshape(-1, original_dim)
test_mean = np.mean(x_test)
test_std = np.std(x_test)
norm_x_test = (x_test - test_mean) / test_std
decoder = kmodels.Sequential([
klayers.Dense(intermediate_dim,
input_dim=latent_dim,
activation='relu',
kernel_regularizer=regularizers.l2(reg)),
klayers.Dense(original_dim,
activation='sigmoid',
kernel_regularizer=regularizers.l2(reg))
])
x = klayers.Input(shape=(original_dim, ))
h = klayers.Dense(intermediate_dim,
activation='relu',
kernel_regularizer=regularizers.l2(reg))(x)
z_mu = klayers.Dense(latent_dim,
kernel_regularizer=regularizers.l2(reg))(h)
z_log_var = klayers.Dense(latent_dim,
kernel_regularizer=regularizers.l2(reg))(h)
z_mu, z_log_var = KLDivergenceLayer()([z_mu, z_log_var])
z_sigma = klayers.Lambda(lambda t: K.exp(.5 * t))(z_log_var)
eps = klayers.Input(tensor=K.random_normal(
stddev=epsilon_std, shape=(K.shape(x)[0], latent_dim)))
z_eps = klayers.Multiply()([z_sigma, eps])
z = klayers.Add()([z_mu, z_eps])
x_pred = decoder(z)
vae = kmodels.Model(inputs=[x, eps], outputs=x_pred)
vae.compile(optimizer='rmsprop', loss=nll)
history = vae.fit(norm_x_train,
norm_x_train,
shuffle=True,
epochs=epochs,
batch_size=batch_size,
validation_split=.2)
encoder = kmodels.Model(x, z_mu)
z_train = encoder.predict(norm_x_train, batch_size=batch_size)
z_test = encoder.predict(norm_x_test, batch_size=batch_size)
z_train = z_train.reshape(-1, 22, 500, 1).transpose(0, 2, 1, 3)
z_test = z_test.reshape(-1, 22, 500, 1).transpose(0, 2, 1, 3)
# now pass encoded input into cnn
size = int(25 * relative_size)
conv1 = layers.Conv2D(size,
kernel_size=(10, 1),
strides=1,
kernel_regularizer=regularizers.l2(reg))
conv2 = layers.Conv2D(size,
kernel_size=(1, 22),
kernel_regularizer=regularizers.l2(reg))
perm1 = layers.Permute((1, 3, 2))
pool1 = layers.AveragePooling2D(pool_size=(3, 1))
drop1 = layers.Dropout(dropout)
model = keras.models.Sequential()
model.add(conv1)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(conv2)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(perm1)
model.add(pool1)
model.add(drop1)
conv3 = layers.Conv2D(2 * size,
kernel_size=(10, size),
kernel_regularizer=regularizers.l2(reg))
model.add(layers.ELU(alpha))
perm2 = layers.Permute((1, 3, 2))
pool2 = layers.AveragePooling2D(pool_size=(3, 1))
drop2 = layers.Dropout(dropout)
model.add(conv3)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(perm2)
model.add(pool2)
model.add(drop2)
conv4 = layers.Conv2D(4 * size,
kernel_size=(10, 2 * size),
kernel_regularizer=regularizers.l2(reg))
perm3 = layers.Permute((1, 3, 2))
pool3 = layers.AveragePooling2D(pool_size=(3, 1))
drop3 = layers.Dropout(dropout)
model.add(conv4)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(perm3)
model.add(pool3)
model.add(drop3)
conv5 = layers.Conv2D(8 * size,
kernel_size=(10, 4 * size),
kernel_regularizer=regularizers.l2(reg))
perm4 = layers.Permute((1, 3, 2))
pool4 = layers.AveragePooling2D(pool_size=(3, 1))
drop4 = layers.Dropout(dropout)
model.add(conv5)
model.add(layers.ELU(alpha))
model.add(layers.BatchNormalization())
model.add(perm4)
model.add(pool4)
model.add(drop4)
model.add(layers.Flatten())
model.add(layers.Dense(4, name='dense_logits'))
model.add(layers.Activation('softmax', dtype='float32',
name='predictions'))
if optim == 'Adam':
optimizer = keras.optimizers.Adam(learning_rate,
beta_1=0.85,
beta_2=0.92,
amsgrad=True)
elif optim == 'RMSprop':
optimizer = keras.optimizers.RMSprop(learning_rate)
else:
optimizer = keras.optimizers.SGD(learning_rate, nesterov=True)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(z_train,
y_train,
batch_size=20,
epochs=epochs,
validation_split=0.2,
verbose=1)
test_scores = model.evaluate(z_test, y_test, verbose=2)
print('Test loss:', test_scores[0])
print('Test accuracy:', test_scores[1])
plot(history)
return model
def __init__(self, num_caps, out_length, **kwargs):
super(Condense, self).__init__(**kwargs)
self.sparse_extraction = CapsuleMapping(num_caps=num_caps,
caps_length=out_length)
self.normal = layers.LayerNormalization()
self.activation = layers.ELU()
def build_stixel_net(input_shape=(370, 800, 3)):
"""
input_shape -> (height, width, channel)
"""
img_input = keras.Input(shape=input_shape)
x = layers.Conv2D(
64, (3, 3), activation="relu", padding="same", name="block1_conv1"
)(img_input)
x = layers.Conv2D(
64, (3, 3), activation="relu", padding="same", name="block1_conv2"
)(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name="block1_pool")(x)
# Block 2
x = layers.Conv2D(
128, (3, 3), activation="relu", padding="same", name="block2_conv1"
)(x)
x = layers.Conv2D(
128, (3, 3), activation="relu", padding="same", name="block2_conv2"
)(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name="block2_pool")(x)
# Block 3
x = layers.Conv2D(
256, (3, 3), activation="relu", padding="same", name="block3_conv1"
)(x)
x = layers.Conv2D(
256, (3, 3), activation="relu", padding="same", name="block3_conv2"
)(x)
x = layers.Conv2D(
256, (3, 3), activation="relu", padding="same", name="block3_conv3"
)(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name="block3_pool")(x)
# Block 4
x = layers.Conv2D(
512, (3, 3), activation="relu", padding="same", name="block4_conv1"
)(x)
x = layers.Conv2D(
512, (3, 3), activation="relu", padding="same", name="block4_conv2"
)(x)
x = layers.Conv2D(
512, (3, 3), activation="relu", padding="same", name="block4_conv3"
)(x)
x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding="same")(x)
x = layers.ELU()(x)
x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding="same")(x)
x = layers.ELU()(x)
x = layers.MaxPooling2D((2, 1), strides=(2, 1))(x)
x = layers.Dropout(0.4)(x)
x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding="same")(x)
x = layers.ELU()(x)
x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding="same")(x)
x = layers.ELU()(x)
x = layers.MaxPooling2D((2, 1), strides=(2, 1), padding="same")(x)
x = layers.Dropout(0.4)(x)
x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding="same")(x)
x = layers.ELU()(x)
x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding="same")(x)
x = layers.ELU()(x)
x = layers.MaxPooling2D((2, 1), strides=(2, 1))(x)
x = layers.Dropout(0.4)(x)
x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding="same")(x)
x = layers.ELU()(x)
x = layers.Conv2D(256, (3, 3), strides=(1, 1), padding="same")(x)
x = layers.ELU()(x)
x = layers.MaxPooling2D((2, 1), strides=(2, 1))(x)
x = layers.Dropout(0.4)(x)
x = layers.Conv2D(2048, (3, 1), strides=(1, 1), padding="valid")(x)
x = layers.ELU()(x)
x = layers.Conv2D(2048, (1, 3), strides=(1, 1), padding="same")(x)
x = layers.ELU()(x)
x = layers.Conv2D(2048, (1, 1), strides=(1, 1))(x)
x = layers.ELU()(x)
x = layers.Dropout(0.4)(x)
x = layers.Conv2D(50, (1, 1), strides=(1, 1), activation="softmax")(x)
x = layers.Reshape((100, 50))(x)
model = models.Model(inputs=img_input, outputs=x)
return model
