def __init__(self, kernels: List[tuple], lstm_kernels: List[tuple], up_factor=2, data_format='NHWC', layer_ind_up = 0, pretraining =False, return_logits=False):
super(UpBlock2D, self).__init__()
self.data_format_keras = 'channels_first' if data_format[1] == 'C' else 'channels_last'
self.up_factor = up_factor
self.channel_axis = 1 if data_format[1] == 'C' else -1
self.ConvLSTM = []
self.Conv = []
self.BN = []
self.LReLU = []
self.return_logits = return_logits
self.pretraining = pretraining
if lstm_kernels is not None:
for kxy_lstm, kout_lstm, dropout, reg, kernel_init in lstm_kernels:
self.ConvLSTM.append(k.layers.ConvLSTM2D(filters=kout_lstm, kernel_size=kxy_lstm, strides=1,
padding='same', data_format=self.data_format_keras, kernel_initializer=kernel_init,
return_sequences=True, stateful=False, recurrent_dropout=dropout,
kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1])))
#initialization of Conv + Batch + ReLU
for l_ind, (kxy, kout, dropout, reg, kernel_init) in enumerate(kernels):
self.Conv.append(k.layers.Conv2D(filters=kout, kernel_size=kxy, strides=1, use_bias=True, kernel_initializer=kernel_init,
data_format=self.data_format_keras, padding='same',
kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1])))
self.BN.append(k.layers.BatchNormalization(axis=self.channel_axis))
self.LReLU.append(k.layers.LeakyReLU())
def __init__(self, conv_kernels: List[tuple], lstm_kernels: List[tuple], data_format='NHWC', pretraining = False):
super(DownBlock2D, self).__init__()
data_format_keras = 'channels_first' if data_format[1] == 'C' else 'channels_last'
channel_axis = 1 if data_format[1] == 'C' else -1
self.ConvLSTM = []
self.Conv = []
self.BN = []
self.LReLU = []
self.total_stride = 1
self.pretraining = pretraining
#initialization of convLSTM2D layers
for kxy_lstm, kout_lstm, dropout, reg, kernel_init in lstm_kernels:
self.ConvLSTM.append(k.layers.ConvLSTM2D(filters=kout_lstm, kernel_size=kxy_lstm, strides=1,
padding='same', data_format=data_format_keras, kernel_initializer=kernel_init,
return_sequences=True, stateful=False, recurrent_dropout=dropout,
kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1])))
#initialization of Conv + Batch + ReLU
for l_ind, (kxy, kout, dropout, reg, kernel_init) in enumerate(conv_kernels):
self.Conv.append(k.layers.Conv2D(filters=kout, kernel_size=kxy, strides=1, use_bias=True, kernel_initializer=kernel_init,
data_format=data_format_keras, padding='same',
kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1])))
self.BN.append(k.layers.BatchNormalization(axis=channel_axis))
self.LReLU.append(k.layers.LeakyReLU())
#initialization of maxpooling layer
self.MaxPool = k.layers.MaxPool2D(pool_size=(2, 2))
def model_graph(self, l1=1e-5, l2=1e-4, hidden_size=[256, 128]):
a = Input(shape=(self.node_size, ))
l = Input(shape=(None, ))
output = a
for i in range(len(hidden_size)):
if i == len(hidden_size) - 1:
output = Dense(hidden_size[i],
activation="relu",
kernel_regularizer=l1_l2(l1, l2),
name="1st")(output)
else:
output = Dense(hidden_size[i],
activation="relu",
kernel_regularizer=l1_l2(l1, l2))(output)
# 记录一阶向量
y = output
for i in reversed(range(len(hidden_size) - 1)):
output = Dense(hidden_size[i],
activation="relu",
kernel_regularizer=l1_l2(l1, l2))(output)
a_ = Dense(self.node_size, activation="relu", name="2nd")(output)
model = Model(inputs=[a, l], outputs=[a_, y])
emb = Model(inputs=a, outputs=y)
return model, emb
def create_model(node_size, hidden_size=[256, 128], l1=1e-5, l2=1e-4):
try:
pass
except ImportWarning:
print("tensorflow not found, please install")
from tensorflow.python.keras.layers import Dense
from tensorflow.python.keras.layers import Input
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.regularizers import l1_l2
A = Input(shape=(node_size, ))
L = Input(shape=(None, ))
fc = A
for i in range(len(hidden_size)):
if i == len(hidden_size) - 1:
fc = Dense(
hidden_size[i],
activation="relu",
kernel_regularizer=l1_l2(l1, l2),
name="1st",
)(fc)
else:
fc = Dense(hidden_size[i],
activation="relu",
kernel_regularizer=l1_l2(l1, l2))(fc)
Y = fc
for i in reversed(range(len(hidden_size) - 1)):
fc = Dense(hidden_size[i],
activation="relu",
kernel_regularizer=l1_l2(l1, l2))(fc)
A_ = Dense(node_size, "relu", name="2nd")(fc)
model = Model(inputs=[A, L], outputs=[A_, Y])
emb = Model(inputs=A, outputs=Y)
return model, emb
def EEGNet_old(nb_classes, Chans = 64, Samples = 128, regRate = 0.0001,
dropoutRate = 0.25, kernels = [(2, 32), (8, 4)], strides = (2, 4)):
""" Keras Implementation of EEGNet_v1 (https://arxiv.org/abs/1611.08024v2)
This model is the original EEGNet model proposed on arxiv
https://arxiv.org/abs/1611.08024v2
with a few modifications: we use striding instead of max-pooling as this
helped slightly in classification performance while also providing a
computational speed-up.
Note that we no longer recommend the use of this architecture, as the new
version of EEGNet performs much better overall and has nicer properties.
Inputs:
nb_classes : total number of final categories
Chans, Samples : number of EEG channels and samples, respectively
regRate : regularization rate for L1 and L2 regularizations
dropoutRate : dropout fraction
kernels : the 2nd and 3rd layer kernel dimensions (default is
the [2, 32] x [8, 4] configuration)
strides : the stride size (note that this replaces the max-pool
used in the original paper)
"""
# start the model
input_main = Input((1, Chans, Samples))
layer1 = Conv2D(16, (Chans, 1), input_shape=(1, Chans, Samples),
kernel_regularizer = l1_l2(l1=regRate, l2=regRate))(input_main)
layer1 = BatchNormalization(axis=1)(layer1)
layer1 = Activation('elu')(layer1)
layer1 = Dropout(dropoutRate)(layer1)
permute_dims = 2, 1, 3
permute1 = Permute(permute_dims)(layer1)
layer2 = Conv2D(4, kernels[0], padding = 'same',
kernel_regularizer=l1_l2(l1=0.0, l2=regRate),
strides = strides)(permute1)
layer2 = BatchNormalization(axis=1)(layer2)
layer2 = Activation('elu')(layer2)
layer2 = Dropout(dropoutRate)(layer2)
layer3 = Conv2D(4, kernels[1], padding = 'same',
kernel_regularizer=l1_l2(l1=0.0, l2=regRate),
strides = strides)(layer2)
layer3 = BatchNormalization(axis=1)(layer3)
layer3 = Activation('elu')(layer3)
layer3 = Dropout(dropoutRate)(layer3)
flatten = Flatten(name = 'flatten')(layer3)
dense = Dense(nb_classes, name = 'dense')(flatten)
softmax = Activation('softmax', name = 'softmax')(dense)
return Model(inputs=input_main, outputs=softmax)
def SepConv_BN(x,
filters,
prefix,
stride=1,
kernel_size=3,
rate=1,
depth_activation=False,
epsilon=1e-3,
regularizer_l1=0.0,
regularizer_l2=0.0):
""" SepConv with BN between depthwise & pointwise. Optionally add activation after BN
Implements right "same" padding for even kernel sizes
Args:
x: input tensor
filters: num of filters in pointwise convolution
prefix: prefix before name
stride: stride at depthwise conv
kernel_size: kernel size for depthwise convolution
rate: atrous rate for depthwise convolution
depth_activation: flag to use activation between depthwise & poinwise convs
epsilon: epsilon to use in BN layer
"""
if stride == 1:
depth_padding = 'same'
else:
kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1)
pad_total = kernel_size_effective - 1
pad_beg = pad_total // 2
pad_end = pad_total - pad_beg
x = ZeroPadding2D((pad_beg, pad_end))(x)
depth_padding = 'valid'
if not depth_activation:
x = Activation('relu')(x)
x = DepthwiseConv2D((kernel_size, kernel_size),
strides=(stride, stride),
dilation_rate=(rate, rate),
padding=depth_padding,
use_bias=False,
name=prefix + '_depthwise',
kernel_regularizer=l1_l2(regularizer_l1,
regularizer_l2))(x)
x = BatchNormalization(name=prefix + '_depthwise_BN', epsilon=epsilon)(x)
if depth_activation:
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1),
padding='same',
use_bias=False,
name=prefix + '_pointwise',
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2))(x)
x = BatchNormalization(name=prefix + '_pointwise_BN', epsilon=epsilon)(x)
if depth_activation:
x = Activation('relu')(x)
return x
def __init__(self, fl, hparams):
"""
Initialises new ANN model
:param fl: fl class
:param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function
"""
self.features_dim = fl.features_c_dim
self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output
self.hparams = hparams
self.normalise_labels = fl.normalise_labels
self.labels_scaler = fl.labels_scaler
features_in = Input(shape=(self.features_dim, ),
name='main_features_c_input')
# Build keras ANN model
x = Dense(units=hparams['nodes'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='Layer_' + str(0))(features_in)
x = Dense(units=hparams['nodes'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='Layer_' + str(1))(x)
x = Dense(units=hparams['nodes'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='Layer_' + str(2))(x)
# x = BatchNormalization()(x)
x = Dense(units=self.labels_dim,
activation='linear',
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='Final')(x)
self.model = Model(inputs=features_in, outputs=x)
optimizer = Adam(learning_rate=hparams['learning_rate'], clipnorm=1)
def mean_relative_error(y_true, y_pred):
diff = K.abs(
(y_true - y_pred) /
K.reshape(K.clip(K.abs(y_true[:, -1]), K.epsilon(), None),
(-1, 1)))
return 100. * K.mean(diff, axis=-1)
if hparams['loss'] == 'mape':
self.model.compile(optimizer=optimizer,
loss=MeanAbsolutePercentageError())
elif hparams['loss'] == 'mre':
self.model.compile(optimizer=optimizer, loss=mean_relative_error)
elif hparams['loss'] == 'mse':
self.model.compile(optimizer=optimizer, loss='mean_squared_error')
def create_model(self, **kwargs):
kwargs.setdefault('metric', 'accuracy')
model_params = _parse_params(self._model_params, return_as='nested')
# Why make it private? Alternate name?
# Move parsing to base model
model = Sequential()
if len(self.y.shape) == 1 or self.y.shape[1] == 1:
## Use OHE for all classification algorithms
## Check for class numbers in y
## Use that as units count
units = 1
else:
units = self.y.shape[1]
model_params['layer_1'].update({'input_dim': self.X.shape[1], 'units': units})
model.add(Dense(units=model_params['layer_1']['units'],
input_dim=model_params['layer_1']['input_dim'],
activation=model_params['layer_1']['activation'],
kernel_regularizer=l1_l2(l1=model_params['layer_1']['l1'],
l2=model_params['layer_1']['l2'])))
model.compile(optimizer=model_params['optimizer'],
loss=model_params['loss'],
metrics=[kwargs['metric']])
return model
def default_regularizer(*args, **kwargs):
l1, l2 = 0.0, 0.0
if (len(args) == 0 and len(kwargs) == 0) or args[0] is None:
return None
elif len(args) == 1:
if isinstance(args[0], (float, int)):
l1 = 0.0
l2 = args[0]
elif isinstance(args[0], list):
l1 = args[0][0]
l2 = args[0][1]
elif isinstance(args[0], dict):
l1 = 0.0 if 'l1' not in args[0] else args[0]['l1']
l2 = 0.0 if 'l2' not in args[0] else args[0]['l2']
elif len(args) == 2:
l1 = args[0]
l2 = args[1]
elif len(kwargs) > 0:
l1 = 0.0 if 'l1' not in kwargs else kwargs['l1']
l2 = 0.0 if 'l2' not in kwargs else kwargs['l2']
else:
raise ValueError(
'Unrecognized entry - input regularization values for l1 and l2.')
# print("regularization is used with l1={} and l2={}".format(l1, l2))
return l1_l2(l1=l1, l2=l2)
def _conv_single(inputs, filter_num, name=None):
conv = Conv2D(filters=filter_num,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer='he_uniform',
padding='valid',
kernel_regularizer=l1_l2(l1=FLAGS.l1, l2=FLAGS.l2),
name=name)(inputs)
return conv
def ann(features_dim: int, labels_dim: int, hparams: Dict):
"""
Neural network base model to be called by class Model to build a full network.
The base model provides a callable keras model that has input of shape specified by features_dim.
There are multiple output defined by the list provided by labels_dim.
:param features_dim: Defines the input shape of the callable keras model
:param labels_dim: Defines how many output there will be and its respective shapes.
:param hparams: Dict created by create_hparam function. Specifies the various hyperparameters for the neural network
:return: Callable Keras neural network
"""
# Hyperparameters
shared_layers = hparams['shared_layers']
assert shared_layers, 'shared_layers is an empty list. Ensure that it is not empty as there must at least be 1 ' \
'shared dense layer.'
# Input
f_in = Input(shape=(features_dim, ), name='ANN_Input')
# Shared layers
for idx, nodes in enumerate(shared_layers):
if idx == 0:
x = Dense(units=nodes,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='shared_' + str(idx))(f_in)
elif nodes != 0:
x = Dense(units=nodes,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='shared_' + str(idx))(x)
# Final output to the correct label dimension
single_task = Dense(units=labels_dim,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(x)
# Create input output model
model = Model(inputs=f_in, outputs=single_task)
return model
def modify_model(model: Model, class_index: int,
importance_type: ImportanceType) -> Model:
gamma_initializer: str = "zeros"
if importance_type & ImportanceType.GAMMA:
gamma_initializer = "ones"
gamma_regularizer = None
if importance_type & ImportanceType.L1 and not importance_type & ImportanceType.L2:
gamma_regularizer = l1()
if not importance_type & ImportanceType.L1 and importance_type & ImportanceType.L2:
gamma_regularizer = l2()
if importance_type & ImportanceType.L1 and importance_type & ImportanceType.L2:
gamma_regularizer = l1_l2()
max_layer: int = len(model.layers)
last_output: Input = None
network_input: Input = None
for i, layer in enumerate(model.layers):
if i == 0:
last_output = layer.output
network_input = layer.input
if 0 < i < max_layer:
new_layer: Union[BatchNormalization,
BatchNormalization] = BatchNormalization(
center=(importance_type
& ImportanceType.CENTERING),
gamma_initializer=gamma_initializer,
gamma_regularizer=gamma_regularizer)
last_output = new_layer(last_output)
if i == max_layer - 1:
new_end_layer: Dense = Dense(2,
activation="softmax",
name="binary_output_layer")
last_output = new_end_layer(last_output)
old_weights = layer.get_weights()
old_weights[0] = np.transpose(old_weights[0], (1, 0))
new_weights: List[np.array] = [
np.append(old_weights[0][class_index:class_index + 1],
np.subtract(
np.sum(old_weights[0], axis=0, keepdims=True),
old_weights[0][class_index:class_index + 1]),
axis=0),
np.append(old_weights[1][class_index:class_index + 1],
np.subtract(
np.sum(old_weights[1], axis=0, keepdims=True),
old_weights[1][class_index:class_index + 1]),
axis=0)
]
new_weights[0] = np.transpose(new_weights[0], (1, 0))
new_end_layer.set_weights(new_weights)
elif i > 0:
last_output = layer(last_output)
return Model(inputs=network_input, outputs=last_output)
def create_model(node_size, hidden_size=[256, 128], l1=1e-5, l2=1e-4):
A = Input(shape=(node_size,))
L = Input(shape=(None,))
fc = A
for i in range(len(hidden_size)):
if i == len(hidden_size) - 1:
fc = Dense(hidden_size[i], activation='relu',
kernel_regularizer=l1_l2(l1, l2), name='1st')(fc)
else:
fc = Dense(hidden_size[i], activation='relu',
kernel_regularizer=l1_l2(l1, l2))(fc)
Y = fc
for i in reversed(range(len(hidden_size) - 1)):
fc = Dense(hidden_size[i], activation='relu',
kernel_regularizer=l1_l2(l1, l2))(fc)
A_ = Dense(node_size, 'relu', name='2nd')(fc)
model = Model(inputs=[A, L], outputs=[A_, Y])
emb = Model(inputs=A, outputs=Y)
return model, emb
def _conv2d_3(inputs, filter_num, name=None):
conv = Conv2D(filters=filter_num,
kernel_size=(3, 3),
strides=(1, 1),
kernel_initializer='he_uniform',
padding='same',
kernel_regularizer=l1_l2(l1=FLAGS.l1, l2=FLAGS.l2),
name=name)(inputs)
in_norm = InstanceNormalization()(conv)
relu = ReLU()(in_norm)
return relu
def decoder_block(a, n_filters):
a = layers.Conv2DTranspose(
filters=n_filters,
kernel_size=(4, 4),
padding='same',
strides=(2, 2),
kernel_regularizer=regularizers.l1_l2(
l1=Config.l1_kernel_regularization,
l2=Config.l2_kernel_regularization))(a)
a = layers.BatchNormalization()(a)
a = layers.ReLU()(a)
return a
def _conv2d_same(x,
filters,
prefix,
stride=1,
kernel_size=3,
rate=1,
regularizer_l1=0.0,
regularizer_l2=0.0):
"""Implements right 'same' padding for even kernel sizes
Without this there is a 1 pixel drift when stride = 2
Args:
x: input tensor
filters: num of filters in pointwise convolution
prefix: prefix before name
stride: stride at depthwise conv
kernel_size: kernel size for depthwise convolution
rate: atrous rate for depthwise convolution
"""
if stride == 1:
return Conv2D(filters, (kernel_size, kernel_size),
strides=(stride, stride),
padding='same',
use_bias=False,
dilation_rate=(rate, rate),
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2),
name=prefix)(x)
else:
kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1)
pad_total = kernel_size_effective - 1
pad_beg = pad_total // 2
pad_end = pad_total - pad_beg
x = ZeroPadding2D((pad_beg, pad_end))(x)
return Conv2D(filters, (kernel_size, kernel_size),
strides=(stride, stride),
padding='valid',
use_bias=False,
dilation_rate=(rate, rate),
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2),
name=prefix)(x)
def encoder_block(a, n_filters):
a = layers.Conv2D(filters=n_filters,
kernel_size=(4, 4),
padding='same',
kernel_regularizer=regularizers.l1_l2(
l1=Config.l1_kernel_regularization,
l2=Config.l2_kernel_regularization))(a)
a = layers.BatchNormalization()(a)
a = layers.LeakyReLU()(a)
a = layers.MaxPool2D(pool_size=(2, 2))(a)
if Config.use_spatial_dropout:
a = layers.SpatialDropout2D(
rate=Config.spatial_dropout_rate)(a)
return a
def make_model(num_classes):
reg = regularizers.l1_l2(0.01, 0.01)
cb = EfficientNetB2(weights='imagenet', include_top=False, drop_connect_rate=0.4, pooling='avg', input_shape=(256, 256, 3))
#cb.trainable = False
freeze_model(cb, 3)
x = cb.output
#x = layers.GaussianNoise(0.5)(x)
#x = layers.BatchNormalization()(x)
#x = layers.Dropout(0.3)(x)
#x = layers.Dense(128, activation='relu', kernel_regularizer=reg)(x)
x = layers.Dropout(0.4)(x)
x = layers.Dense(num_classes, activation='softmax', kernel_regularizer=reg)(x)
model = Model(cb.input, x)
#model.summary()
#pr = tf.keras.metrics.AUC(name='PR', curve='PR')
model.compile(optimizer=optimizers.Adam(1e-4), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
return model
def __init__(self, model_conf: ModelConfig, mode: RunMode, outputs):
self.model_conf = model_conf
self.utils = NetworkUtils(mode)
self.dense = tf.keras.layers.Dense(
units=self.model_conf.category_num + 2,
kernel_initializer=tf.keras.initializers.he_normal(seed=None),
kernel_regularizer=l1_l2(l1=0.01, l2=0.001),
bias_initializer='zeros',
)
self.time_distributed = lambda: tf.keras.layers.TimeDistributed(
layer=self.dense,
name='predict',
)(inputs=outputs, training=self.utils.training)
self.outputs = self.time_distributed()
self.predict = tf.keras.backend.permute_dimensions(self.outputs,
pattern=(1, 0, 2))
def __init__(self):
self.batch_size = 15
self.learning_rate = 0.0001
self.epochs = 6000
self.time_stamps = 1524
self.num_steps = 64
self.dropout = 0.5
self.regularizer_type = 'l2'
if self.regularizer_type == 'l2':
self.regularizer = l2(1)
elif self.regularizer_type == 'l1':
self.regularizer = l1(1)
elif self.regularizer == 'l1_l2':
self.regularizer = l1_l2(1)
self.model_type = ''
self.data_type = 'all'
def build_nn(hidden_layer_sizes,
input_dim,
l1_reg,
l2_reg,
activation_func='tanh'):
input_layer = Input(shape=(input_dim, ))
x = input_layer
# Fully connected hidden layers
for i, size in enumerate(hidden_layer_sizes):
x = Dense(size,
activation=activation_func,
kernel_regularizer=regularizers.l1_l2(l1=l1_reg,
l2=l2_reg))(x)
x = Dropout(0.3)(x) if i != len(hidden_layer_sizes) - 1 else x
x = BatchNormalization()(x)
# Wrap base_nn into a model
base_nn = Model(inputs=input_layer, outputs=x, name="base_nn")
return (base_nn, x)
def cross_stitch(features_dim: int, labels_dim: List[int], hparams: Dict):
"""
Neural network base model to be called by class Model to build a full network.
The base model provides a callable keras model that has input of shape specified by features_dim.
There are multiple output defined by the list provided by labels_dim.
:param features_dim: Defines the input shape of the callable keras model
:param labels_dim: Defines how many output there will be and its respective shapes.
eg: [1, 1, 1, 2] ==> Multi-task learning with 4 separate outputs, each with shape 1, 1, 1, 2 respectively.
:param hparams: Dict created by create_hparam function. Specifies the various hyperparameters for the neural network
:return: Callable multi-task keras neural network
Cross-stitch neural network paper
https://ieeexplore.ieee.org/document/7780802
"""
cs_layers = hparams['cs_layers']
assert cs_layers, 'cs_layers is an empty list. Ensure that it is not empty as there must at least be 1 ' \
'cross stitch dense layer.'
# Input
f_in = Input(shape=(features_dim, ), name='CS_Input')
# Cross stitch layers
multi_task = []
for idx, nodes in enumerate(cs_layers):
if idx == 0:
for idx_task, _ in enumerate(labels_dim):
x = Dense(units=nodes,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='cs_task_' + str(idx_task) + '_layer_' +
str(idx))(f_in)
multi_task.append(x)
multi_task = CrossStitchLayer(name='cs_unit_' +
str(idx))(multi_task)
elif nodes != 0:
temp = []
for idx_task, single_task in enumerate(multi_task):
single_task = Dense(units=nodes,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='cs_task_' + str(idx_task) +
'_layer_' + str(idx))(single_task)
temp.append(single_task)
multi_task = CrossStitchLayer(name='cs_unit_' + str(idx))(temp)
temp = []
for idx_task, single_task_label in enumerate(labels_dim):
single_task = multi_task[idx_task]
single_task = Dense(
units=single_task_label,
activation='linear',
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='task_' + str(idx_task) + '_output_layer')(single_task)
temp.append(single_task)
multi_task = temp
model = Model(inputs=f_in, outputs=multi_task, name='CS')
return model
def hps(features_dim: int, labels_dim: List[int], hparams: Dict):
"""
Neural network base model to be called by class Model to build a full network.
The base model provides a callable keras model that has input of shape specified by features_dim.
There are multiple output defined by the list provided by labels_dim.
:param features_dim: Defines the input shape of the callable keras model
:param labels_dim: Defines how many output there will be and its respective shapes.
eg: [1, 1, 1, 2] ==> Multi-task learning with 4 separate outputs, each with shape 1, 1, 1, 2 respectively.
:param hparams: Dict created by create_hparam function. Specifies the various hyperparameters for the neural network
:return: Callable multi-task keras neural network
HPS = Hard Parameter Sharing.
For reference: https://towardsdatascience.com/multitask-learning-teach-your-ai-more-to-make-it-better-dde116c2cd40
Dense neural network base, till the splitting point where each separate task has its own task specific
dense neural network.
Most basic form of multi-task learning.
"""
# Hyperparameters
shared_layers = hparams['shared_layers']
ts_layers = hparams['ts_layers'] # ts stands for task specific
assert shared_layers, 'shared_layers is an empty list. Ensure that it is not empty as there must at least be 1 ' \
'shared dense layer.'
assert ts_layers, 'ts_layers is an empty list. Ensure that it is not empty as there must at least be 1 task' \
'task specific dense layer'
# Input
f_in = Input(shape=(features_dim, ), name='HPS_Input')
# Shared layers
for idx, nodes in enumerate(shared_layers):
if idx == 0:
x = Dense(units=nodes,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='shared_' + str(idx))(f_in)
elif nodes != 0:
x = Dense(units=nodes,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='shared_' + str(idx))(x)
# Task Specific Layers
multi_task = []
for idx_task, single_task_label in enumerate(labels_dim):
# Task Specific layers
for idx_layer, nodes in enumerate(ts_layers):
if idx_layer == 0:
single_task = Dense(units=nodes,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='task_' + str(idx_task) + '_layer_' +
str(idx_layer))(x)
else:
single_task = Dense(units=nodes,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='task_' + str(idx_task) + '_layer_' +
str(idx_layer))(single_task)
# Final output to the correct label dimension
single_task = Dense(
units=single_task_label,
activation='linear',
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='task_' + str(idx_task) + '_output_layer')(single_task)
multi_task.append(single_task)
# Create input output model
model = Model(inputs=f_in, outputs=multi_task, name='HPS')
return model
def __init__(self, fl, mode, hparams):
"""
Initialises new DNN model based on input features_dim, labels_dim, hparams
:param features_dim: Number of input feature nodes. Integer
:param labels_dim: Number of output label nodes. Integer
:param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function.
hparams includes: hidden_layers: List containing number of nodes in each hidden layer. [10, 20] means 10 then 20 nodes.
"""
# self.features_dim = fl.features_c_dim
# self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output
self.features_dim = fl.features_c_dim + 1 # 1 for the positional argument
self.labels_dim = 1
self.numel = fl.labels.shape[1] + 1
self.hparams = hparams
self.mode = mode
self.normalise_labels = fl.normalise_labels
self.labels_scaler = fl.labels_scaler
features_in = Input(shape=(self.features_dim, ),
name='main_features_c_input')
# Selection of model
if mode == 'ann':
model = ann(self.features_dim, self.labels_dim, self.hparams)
x = model(features_in)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'ann2':
model_1 = ann(self.features_dim, 50, self.hparams)
x = model_1(features_in)
model_end = ann(50, 50, self.hparams)
end = model_end(x)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
model_2 = ann(50, self.labels_dim - 1, self.hparams)
x = model_2(x)
self.model = Model(inputs=features_in, outputs=[end_node, x])
elif mode == 'ann3':
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(0))(features_in)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(2))(x)
# x = BatchNormalization()(x)
x = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_set_19')(x)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'conv1':
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='shared' + str(1))(features_in)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
#x = BatchNormalization()(x)
x = Dense(units=19,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_set_19')(x)
#x = BatchNormalization()(x)
x = Reshape(target_shape=(19, 1))(x)
x = Conv1D(filters=hparams['filters'],
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
#x = BatchNormalization()(x)
x = Conv1D(filters=hparams['filters'] * 2,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = Conv1D(filters=hparams['filters'] * 4,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
#x = Permute((2,1))(x)
#x = GlobalAveragePooling1D()(x)
x = TimeDistributed(Dense(1, activation='linear'))(x)
x = Reshape(target_shape=(19, ))(x)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'conv2':
x = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(1))(features_in)
x = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(2))(x)
end = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(1))(x)
end = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(2))(end)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
x = Dense(units=80,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Reshape(target_shape=(80, 1))(x)
x = Conv1D(filters=8,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
x = Conv1D(filters=16,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
#x = Permute((2,1))(x)
#x = GlobalAveragePooling1D()(x)
x = TimeDistributed(Dense(1, activation='linear'))(x)
x = Reshape(target_shape=(20, ))(x)
self.model = Model(inputs=features_in, outputs=[end_node, x])
elif mode == 'lstm':
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(1))(features_in)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(2))(x)
end = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(1))(x)
end = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(2))(end)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(2))(x)
x = RepeatVector(n=20)(x)
x = LSTM(units=30, activation='relu', return_sequences=True)(x)
x = LSTM(units=30, activation='relu', return_sequences=True)(x)
x = TimeDistributed(Dense(1))(x)
x = Reshape(target_shape=(20, ))(x)
'''
x = Permute((2,1))(x)
x = GlobalAveragePooling1D()(x)
'''
self.model = Model(inputs=features_in, outputs=[end_node, x])
optimizer = Adam(clipnorm=1)
self.model.compile(optimizer=optimizer, loss='mean_squared_error')
def __init__(self, fl, mode, hparams):
"""
Initialises new DNN model based on input features_dim, labels_dim, hparams
:param features_dim: Number of input feature nodes. Integer
:param labels_dim: Number of output label nodes. Integer
:param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function.
hparams includes: hidden_layers: List containing number of nodes in each hidden layer. [10, 20] means 10 then 20 nodes.
"""
self.features_dim = fl.features_c_dim
self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output
self.hparams = hparams
self.mode = mode
self.normalise_labels = fl.normalise_labels
self.labels_scaler = fl.labels_scaler
features_in = Input(shape=(self.features_dim, ),
name='main_features_c_input')
# Selection of model
if mode == 'ann':
model = ann(self.features_dim, self.labels_dim, self.hparams)
x = model(features_in)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'ann2':
model_1 = ann(self.features_dim, 50, self.hparams)
x = model_1(features_in)
model_end = ann(50, 50, self.hparams)
end = model_end(x)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
model_2 = ann(50, self.labels_dim - 1, self.hparams)
x = model_2(x)
self.model = Model(inputs=features_in, outputs=[end_node, x])
elif mode == 'ann3':
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(0))(features_in)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(2))(x)
# x = BatchNormalization()(x)
x = Dense(units=self.labels_dim,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Final')(x)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'conv1':
if fl.label_type == 'gf20':
final_dim = 20
else:
final_dim = 19
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='shared' + str(1))(features_in)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
#x = BatchNormalization()(x)
x = Dense(units=final_dim,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_set_19')(x)
#x = BatchNormalization()(x)
x = Reshape(target_shape=(final_dim, 1))(x)
x = Conv1D(filters=hparams['filters'],
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
#x = BatchNormalization()(x)
x = Conv1D(filters=hparams['filters'] * 2,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = Conv1D(filters=hparams['filters'] * 4,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
#x = Permute((2,1))(x)
#x = GlobalAveragePooling1D()(x)
x = TimeDistributed(Dense(1, activation='linear'))(x)
x = Reshape(target_shape=(final_dim, ))(x)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'conv2':
x = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(1))(features_in)
x = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(2))(x)
end = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(1))(x)
end = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(2))(end)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
x = Dense(units=80,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Reshape(target_shape=(80, 1))(x)
x = Conv1D(filters=8,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
x = Conv1D(filters=16,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
#x = Permute((2,1))(x)
#x = GlobalAveragePooling1D()(x)
x = TimeDistributed(Dense(1, activation='linear'))(x)
x = Reshape(target_shape=(20, ))(x)
self.model = Model(inputs=features_in, outputs=[end_node, x])
elif mode == 'lstm':
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(1))(features_in)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(2))(x)
end = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(1))(x)
end = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(2))(end)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(2))(x)
x = RepeatVector(n=20)(x)
x = LSTM(units=30, activation='relu', return_sequences=True)(x)
x = LSTM(units=30, activation='relu', return_sequences=True)(x)
x = TimeDistributed(Dense(1))(x)
x = Reshape(target_shape=(20, ))(x)
'''
x = Permute((2,1))(x)
x = GlobalAveragePooling1D()(x)
'''
self.model = Model(inputs=features_in, outputs=[end_node, x])
optimizer = Adam(learning_rate=hparams['learning_rate'], clipnorm=1)
def weighted_mse(y_true, y_pred):
loss_weights = np.sqrt(np.arange(1, 20))
#loss_weights = np.arange(1, 20)
return K.mean(K.square(y_pred - y_true) * loss_weights, axis=-1)
def haitao_error(y_true, y_pred):
diff = K.abs(
(y_true - y_pred) /
K.reshape(K.clip(K.abs(y_true[:, -1]), K.epsilon(), None),
(-1, 1)))
return 100. * K.mean(diff, axis=-1)
if hparams['loss'] == 'mape':
self.model.compile(optimizer=optimizer,
loss=MeanAbsolutePercentageError())
elif hparams['loss'] == 'haitao':
self.model.compile(optimizer=optimizer, loss=haitao_error)
elif hparams['loss'] == 'mse':
self.model.compile(optimizer=optimizer, loss='mean_squared_error')
def __init__(self,
kernels: List[tuple],
up_factor=2,
data_format='NHWC',
layer_ind_up=0,
weights_list=0,
pretraining=False,
pretraining_type='full',
return_logits=False):
super(UpBlock2D, self).__init__()
self.data_format_keras = 'channels_first' if data_format[
1] == 'C' else 'channels_last'
self.up_factor = up_factor
self.channel_axis = 1 if data_format[1] == 'C' else -1
self.Conv = []
self.BN = []
self.LReLU = []
self.return_logits = return_logits
self.pretraining = pretraining
for l_ind, (kxy, kout, dropout, reg,
kernel_init) in enumerate(kernels):
if self.pretraining == 'cells' and pretraining_type == 'full':
C = tf.keras.initializers.Constant
if layer_ind_up == 3:
if l_ind != 2:
self.Conv.append(
k.layers.Conv2D(
filters=kout,
kernel_size=kxy,
strides=1,
use_bias=True,
data_format=self.data_format_keras,
padding='same',
kernel_initializer=C(weights_list[0 +
l_ind * 2]),
bias_initializer=C(weights_list[1 +
l_ind * 2]),
kernel_regularizer=regularizers.l1_l2(
l1=reg[0], l2=reg[1])))
self.BN.append(
k.layers.BatchNormalization(
axis=self.channel_axis,
beta_initializer=C(weights_list[6 +
l_ind * 4]),
gamma_initializer=C(weights_list[7 +
l_ind * 4]),
moving_mean_initializer=C(
weights_list[8 + l_ind * 4]),
moving_variance_initializer=C(
weights_list[9 + l_ind * 4])))
else:
self.Conv.append(
k.layers.Conv2D(
filters=kout,
kernel_size=kxy,
strides=1,
use_bias=True,
data_format=self.data_format_keras,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l1_l2(
l1=reg[0], l2=reg[1])))
self.BN.append(
k.layers.BatchNormalization(
axis=self.channel_axis))
else:
self.Conv.append(
k.layers.Conv2D(
filters=kout,
kernel_size=kxy,
strides=1,
use_bias=True,
data_format=self.data_format_keras,
padding='same',
kernel_initializer=C(weights_list[0 + l_ind * 2]),
bias_initializer=C(weights_list[1 + l_ind * 2]),
kernel_regularizer=regularizers.l1_l2(l1=reg[0],
l2=reg[1])))
self.BN.append(
k.layers.BatchNormalization(
axis=self.channel_axis,
beta_initializer=C(weights_list[4 + l_ind * 4]),
gamma_initializer=C(weights_list[5 + l_ind * 4]),
moving_mean_initializer=C(weights_list[6 +
l_ind * 4]),
moving_variance_initializer=C(
weights_list[7 + l_ind * 4])))
# elif self.pretraining == 'imagenet':
# C = tf.keras.initializers.Constant
# if l_ind != 2:
# self.Conv.append(k.layers.Conv2D(filters=kout, kernel_size=kxy, strides=1, use_bias=True,
# data_format=self.data_format_keras, padding='same', kernel_initializer=C(weights_list[0 + l_ind]),
# kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1])))
# self.BN.append(k.layers.BatchNormalization(axis=self.channel_axis, beta_initializer = C(weights_list[2 + l_ind*4]),
# gamma_initializer = C(weights_list[3 + l_ind*4]), moving_mean_initializer = C(weights_list[4 + l_ind*4]),
# moving_variance_initializer = C(weights_list[5 + l_ind*4])))
# else:
# self.Conv.append(k.layers.Conv2D(filters=kout, kernel_size=kxy, strides=1, use_bias=True,
# data_format=self.data_format_keras, padding='same', kernel_initializer='he_normal',
# kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1])))
# self.BN.append(k.layers.BatchNormalization(axis=self.channel_axis))
else:
self.Conv.append(
k.layers.Conv2D(filters=kout,
kernel_size=kxy,
strides=1,
use_bias=True,
kernel_initializer=kernel_init,
data_format=self.data_format_keras,
padding='same',
kernel_regularizer=regularizers.l1_l2(
l1=reg[0], l2=reg[1])))
self.BN.append(
k.layers.BatchNormalization(axis=self.channel_axis))
self.LReLU.append(k.layers.LeakyReLU())
class KerasRegularizersTest(keras_parameterized.TestCase,
parameterized.TestCase):
def create_model(self, kernel_regularizer=None, activity_regularizer=None):
model = keras.models.Sequential()
model.add(
keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=kernel_regularizer,
activity_regularizer=activity_regularizer,
input_shape=(DATA_DIM, )))
return model
def get_data(self):
(x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
train_samples=10,
test_samples=10,
input_shape=(DATA_DIM, ),
num_classes=NUM_CLASSES)
y_train = np_utils.to_categorical(y_train, NUM_CLASSES)
y_test = np_utils.to_categorical(y_test, NUM_CLASSES)
return (x_train, y_train), (x_test, y_test)
def create_multi_input_model_from(self, layer1, layer2):
input_1 = keras.layers.Input(shape=(DATA_DIM, ))
input_2 = keras.layers.Input(shape=(DATA_DIM, ))
out1 = layer1(input_1)
out2 = layer2(input_2)
out = keras.layers.Average()([out1, out2])
model = keras.models.Model([input_1, input_2], out)
model.add_loss(keras.backend.mean(out2))
model.add_loss(math_ops.reduce_sum(input_1))
return model
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
])
def test_kernel_regularization(self, regularizer):
(x_train, y_train), _ = self.get_data()
model = self.create_model(kernel_regularizer=regularizer)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.
should_run_tf_function())
self.assertEqual(len(model.losses), 1)
model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0)
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
('l2_zero', keras.regularizers.l2(0.)),
])
def test_activity_regularization(self, regularizer):
(x_train, y_train), _ = self.get_data()
model = self.create_model(activity_regularizer=regularizer)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.
should_run_tf_function())
self.assertEqual(len(model.losses),
1 if context.executing_eagerly() else 1)
model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0)
@keras_parameterized.run_all_keras_modes
@keras_parameterized.run_with_all_model_types
def test_zero_regularization(self):
# Verifies that training with zero regularization works.
x, y = np.ones((10, 10)), np.ones((10, 3))
model = testing_utils.get_model_from_layers([
keras.layers.Dense(3, kernel_regularizer=keras.regularizers.l2(0))
],
input_shape=(10, ))
model.compile('sgd',
'mse',
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.
should_run_tf_function())
model.fit(x, y, batch_size=5, epochs=1)
def test_custom_regularizer_saving(self):
def my_regularizer(weights):
return math_ops.reduce_sum(math_ops.abs(weights))
inputs = keras.Input((10, ))
outputs = keras.layers.Dense(1,
kernel_regularizer=my_regularizer)(inputs)
model = keras.Model(inputs, outputs)
model2 = model.from_config(
model.get_config(),
custom_objects={'my_regularizer': my_regularizer})
self.assertEqual(model2.layers[1].kernel_regularizer, my_regularizer)
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
])
def test_regularization_shared_layer(self, regularizer):
dense_layer = keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=regularizer,
activity_regularizer=regularizer)
model = self.create_multi_input_model_from(dense_layer, dense_layer)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.
should_run_tf_function())
self.assertEqual(len(model.losses), 5)
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
])
def test_regularization_shared_model(self, regularizer):
dense_layer = keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=regularizer,
activity_regularizer=regularizer)
input_tensor = keras.layers.Input(shape=(DATA_DIM, ))
dummy_model = keras.models.Model(input_tensor,
dense_layer(input_tensor))
model = self.create_multi_input_model_from(dummy_model, dummy_model)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.
should_run_tf_function())
self.assertEqual(len(model.losses), 6)
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
])
def test_regularization_shared_layer_in_different_models(
self, regularizer):
shared_dense = keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=regularizer,
activity_regularizer=regularizer)
models = []
for _ in range(2):
input_tensor = keras.layers.Input(shape=(DATA_DIM, ))
unshared_dense = keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=regularizer)
out = unshared_dense(shared_dense(input_tensor))
models.append(keras.models.Model(input_tensor, out))
model = self.create_multi_input_model_from(layer1=models[0],
layer2=models[1])
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.
should_run_tf_function())
self.assertEqual(len(model.losses), 14)
def _inverted_res_block(inputs,
expansion,
stride,
alpha,
filters,
block_id,
skip_connection,
rate=1,
regularizer_l1=0.0,
regularizer_l2=0.0):
in_channels = inputs.shape.as_list()[-1] # inputs._keras_shape[-1]
pointwise_conv_filters = int(filters * alpha)
pointwise_filters = _make_divisible(pointwise_conv_filters, 8)
x = inputs
prefix = 'expanded_conv_{}_'.format(block_id)
if block_id:
# Expand
x = Conv2D(expansion * in_channels,
kernel_size=1,
padding='same',
use_bias=False,
activation=None,
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2),
name=prefix + 'expand')(x)
x = BatchNormalization(epsilon=1e-3,
momentum=0.999,
name=prefix + 'expand_BN')(x)
x = Activation(relu6, name=prefix + 'expand_relu')(x)
else:
prefix = 'expanded_conv_'
# Depthwise
x = DepthwiseConv2D(kernel_size=3,
strides=stride,
activation=None,
use_bias=False,
padding='same',
dilation_rate=(rate, rate),
kernel_regularizer=l1_l2(regularizer_l1,
regularizer_l2),
name=prefix + 'depthwise')(x)
x = BatchNormalization(epsilon=1e-3,
momentum=0.999,
name=prefix + 'depthwise_BN')(x)
x = Activation(relu6, name=prefix + 'depthwise_relu')(x)
# Project
x = Conv2D(pointwise_filters,
kernel_size=1,
padding='same',
use_bias=False,
activation=None,
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2),
name=prefix + 'project')(x)
x = BatchNormalization(epsilon=1e-3,
momentum=0.999,
name=prefix + 'project_BN')(x)
if skip_connection:
return Add(name=prefix + 'add')([inputs, x])
# if in_channels == pointwise_filters and stride == 1:
# return Add(name='res_connect_' + str(block_id))([inputs, x])
return x
def Deeplabv3(weights='pascal_voc',
input_tensor=None,
input_shape=(512, 512, 3),
classes=21,
backbone='mobilenetv2',
OS=16,
alpha=1.,
activation=None,
regularizer_l1=0.0,
regularizer_l2=1e-4):
""" Instantiates the Deeplabv3+ architecture
Optionally loads weights pre-trained
on PASCAL VOC or Cityscapes. This model is available for TensorFlow only.
# Arguments
weights: one of 'pascal_voc' (pre-trained on pascal voc),
'cityscapes' (pre-trained on cityscape) or None (random initialization)
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: shape of input image. format HxWxC
PASCAL VOC model was trained on (512,512,3) images. None is allowed as shape/width
classes: number of desired classes. PASCAL VOC has 21 classes, Cityscapes has 19 classes.
If number of classes not aligned with the weights used, last layer is initialized randomly
backbone: backbone to use. one of {'xception','mobilenetv2'}
activation: optional activation to add to the top of the network.
One of 'softmax', 'sigmoid' or None
OS: determines input_shape/feature_extractor_output ratio. One of {8,16}.
Used only for xception backbone.
alpha: controls the width of the MobileNetV2 network. This is known as the
width multiplier in the MobileNetV2 paper.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
Used only for mobilenetv2 backbone. Pretrained is only available for alpha=1.
# Returns
A Keras model instance.
# Raises
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
ValueError: in case of invalid argument for `weights` or `backbone`
"""
if not (weights in {'pascal_voc', 'cityscapes', None}):
raise ValueError(
'The `weights` argument should be either '
'`None` (random initialization), `pascal_voc`, or `cityscapes` '
'(pre-trained on PASCAL VOC)')
if not (backbone in {'xception', 'mobilenetv2'}):
raise ValueError('The `backbone` argument should be either '
'`xception` or `mobilenetv2` ')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
img_input = input_tensor
if backbone == 'xception':
if OS == 8:
entry_block3_stride = 1
middle_block_rate = 2 # ! Not mentioned in paper, but required
exit_block_rates = (2, 4)
atrous_rates = (12, 24, 36)
else:
entry_block3_stride = 2
middle_block_rate = 1
exit_block_rates = (1, 2)
atrous_rates = (6, 12, 18)
x = Conv2D(32, (3, 3),
strides=(2, 2),
name='entry_flow_conv1_1',
use_bias=False,
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2),
padding='same')(img_input)
x = BatchNormalization(name='entry_flow_conv1_1_BN')(x)
x = Activation('relu')(x)
x = _conv2d_same(x,
64,
'entry_flow_conv1_2',
kernel_size=3,
stride=1,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = BatchNormalization(name='entry_flow_conv1_2_BN')(x)
x = Activation('relu')(x)
x = _xception_block(x, [128, 128, 128],
'entry_flow_block1',
skip_connection_type='conv',
stride=2,
depth_activation=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x, skip1 = _xception_block(x, [256, 256, 256],
'entry_flow_block2',
skip_connection_type='conv',
stride=2,
depth_activation=False,
return_skip=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _xception_block(x, [728, 728, 728],
'entry_flow_block3',
skip_connection_type='conv',
stride=entry_block3_stride,
depth_activation=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
for i in range(16):
x = _xception_block(x, [728, 728, 728],
'middle_flow_unit_{}'.format(i + 1),
skip_connection_type='sum',
stride=1,
rate=middle_block_rate,
depth_activation=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _xception_block(x, [728, 1024, 1024],
'exit_flow_block1',
skip_connection_type='conv',
stride=1,
rate=exit_block_rates[0],
depth_activation=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _xception_block(x, [1536, 1536, 2048],
'exit_flow_block2',
skip_connection_type='none',
stride=1,
rate=exit_block_rates[1],
depth_activation=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
else:
OS = 8
first_block_filters = _make_divisible(32 * alpha, 8)
x = Conv2D(first_block_filters,
kernel_size=3,
strides=(2, 2),
padding='same',
use_bias=False,
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2),
name='Conv')(img_input)
x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_BN')(x)
x = Activation(relu6, name='Conv_Relu6')(x)
x = _inverted_res_block(x,
filters=16,
alpha=alpha,
stride=1,
expansion=1,
block_id=0,
skip_connection=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=2,
expansion=6,
block_id=1,
skip_connection=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=1,
expansion=6,
block_id=2,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=2,
expansion=6,
block_id=3,
skip_connection=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=4,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=5,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
# stride in block 6 changed from 2 -> 1, so we need to use rate = 2
x = _inverted_res_block(
x,
filters=64,
alpha=alpha,
stride=1, # 1!
expansion=6,
block_id=6,
skip_connection=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=7,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=8,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=9,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=10,
skip_connection=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=11,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=12,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(
x,
filters=160,
alpha=alpha,
stride=1,
rate=2, # 1!
expansion=6,
block_id=13,
skip_connection=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=14,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=15,
skip_connection=True,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = _inverted_res_block(x,
filters=320,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=16,
skip_connection=False,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
# end of feature extractor
# branching for Atrous Spatial Pyramid Pooling
# Image Feature branch
shape_before = tf.shape(x)
b4 = GlobalAveragePooling2D()(x)
# from (b_size, channels)->(b_size, 1, 1, channels)
b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4)
b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4)
b4 = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2),
name='image_pooling')(b4)
b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4)
b4 = Activation('relu')(b4)
# upsample. have to use compat because of the option align_corners
size_before = tf.keras.backend.int_shape(x)
b4 = Lambda(lambda x: tf.compat.v1.image.resize(
x, size_before[1:3], align_corners=True))(b4)
# simple 1x1
b0 = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='aspp0',
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2))(x)
b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0)
b0 = Activation('relu', name='aspp0_activation')(b0)
# there are only 2 branches in mobilenetV2. not sure why
if backbone == 'xception':
# rate = 6 (12)
b1 = SepConv_BN(x,
256,
'aspp1',
rate=atrous_rates[0],
depth_activation=True,
epsilon=1e-5,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
# rate = 12 (24)
b2 = SepConv_BN(x,
256,
'aspp2',
rate=atrous_rates[1],
depth_activation=True,
epsilon=1e-5,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
# rate = 18 (36)
b3 = SepConv_BN(x,
256,
'aspp3',
rate=atrous_rates[2],
depth_activation=True,
epsilon=1e-5,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
# concatenate ASPP branches & project
x = Concatenate()([b4, b0, b1, b2, b3])
else:
x = Concatenate()([b4, b0])
x = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='concat_projection')(x)
x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x)
x = Activation('relu')(x)
x = Dropout(0.1)(x)
# DeepLab v.3+ decoder
if backbone == 'xception':
# Feature projection
# x4 (x2) block
size_before2 = tf.keras.backend.int_shape(x)
x = Lambda(lambda xx: tf.compat.v1.image.resize(
xx, size_before2[1:3] * tf.constant(OS // 4), align_corners=True))(
x)
dec_skip1 = Conv2D(48, (1, 1),
padding='same',
kernel_regularizer=l1_l2(regularizer_l1,
regularizer_l2),
use_bias=False,
name='feature_projection0')(skip1)
dec_skip1 = BatchNormalization(name='feature_projection0_BN',
epsilon=1e-5)(dec_skip1)
dec_skip1 = Activation('relu')(dec_skip1)
x = Concatenate()([x, dec_skip1])
x = SepConv_BN(x,
256,
'decoder_conv0',
depth_activation=True,
epsilon=1e-5,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
x = SepConv_BN(x,
256,
'decoder_conv1',
depth_activation=True,
epsilon=1e-5,
regularizer_l1=regularizer_l1,
regularizer_l2=regularizer_l2)
# you can use it with arbitary number of classes
if (weights == 'pascal_voc' and classes == 21) or (weights == 'cityscapes'
and classes == 19):
last_layer_name = 'logits_semantic'
else:
last_layer_name = 'custom_logits_semantic'
x = Conv2D(classes, (1, 1),
padding='same',
name=last_layer_name,
kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2))(x)
size_before3 = tf.keras.backend.int_shape(img_input)
x = Lambda(lambda xx: tf.compat.v1.image.resize(
xx, size_before3[1:3], align_corners=True))(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
if activation in {'softmax', 'sigmoid'}:
x = tf.keras.layers.Activation(activation)(x)
model = Model(inputs, x, name='deeplabv3plus')
# load weights
if weights == 'pascal_voc':
if backbone == 'xception':
weights_path = get_file(
'deeplabv3_xception_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH_X,
cache_subdir='models')
else:
weights_path = get_file(
'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH_MOBILE,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
elif weights == 'cityscapes':
if backbone == 'xception':
weights_path = get_file(
'deeplabv3_xception_tf_dim_ordering_tf_kernels_cityscapes.h5',
WEIGHTS_PATH_X_CS,
cache_subdir='models')
else:
weights_path = get_file(
'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels_cityscapes.h5',
WEIGHTS_PATH_MOBILE_CS,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
return model
def create_model(self, **kwargs):
model_params = _parse_params(self._model_params, return_as='nested')
model_params.update(
{'input_dims_users': self.x_train_user.shape[1], 'input_dims_items': self.x_train_questions.shape[1]})
user_input_layer = Input(
shape=(model_params['input_dims_users'],), name='user_id')
quest_input_layer = Input(shape=(
model_params['input_dims_items'],), name='questions/items')
if not self.l_traits == None:
pass
# provision to add latent traits to first layer
# latent_trait = Dense(1, use_bias=False,
# kernel_initializer= initializers.RandomNormal(mean=0, stddev=1.0, seed=None),
# kernel_regularizer=regularizers.l2(0.01), name='latent_trait')(user_input_layer)
else:
latent_trait = Dense(model_params['ability_params']['units'], use_bias=model_params['ability_params']['use_bias'],
bias_initializer= model_params['ability_params']['bias'],
kernel_initializer=model_params['ability_params']['kernel'],
kernel_regularizer=l1_l2(
l1=model_params['ability_params']['regularizers']['l1'],
l2=model_params['ability_params']['regularizers']['l2']),
activity_regularizer=l1_l2(
l1=model_params['ability_params']['group_lasso']['l1'],
l2=model_params['ability_params']['group_lasso']['l2']),
name='latent_trait/ability')(user_input_layer)
difficulty_level = Dense(model_params['diff_params']['units'], use_bias=model_params['diff_params']['use_bias'],
bias_initializer= model_params['diff_params']['bias'],
kernel_initializer=model_params['diff_params']['kernel'],
kernel_regularizer=l1_l2(
l1=model_params['diff_params']['regularizers']['l1'],
l2=model_params['diff_params']['regularizers']['l2']),
activity_regularizer=l1_l2(
l1=model_params['diff_params']['group_lasso']['l1'],
l2=model_params['diff_params']['group_lasso']['l2']),
name='difficulty_level')(quest_input_layer)
discrimination_param = Dense(model_params['disc_params']['units'], use_bias=model_params['disc_params']['use_bias'],
kernel_initializer=model_params['disc_params']['kernel'],
bias_initializer=model_params['disc_params']['bias'],
kernel_regularizer=l1_l2(
l1=model_params['disc_params']['regularizers']['l1'],
l2=model_params['disc_params']['regularizers']['l2']),
activity_regularizer=l1_l2(
l1=model_params['disc_params']['group_lasso']['l1'],
l2=model_params['disc_params']['group_lasso']['l2']),
trainable=model_params['disc_params']['train'],
#activation= model_params['disc_params']['act'],
name='disc_param')(quest_input_layer)
discrimination_param = Activation(model_params['disc_params']['act'], name= 'disc_activation')(discrimination_param)
disc_latent_interaction = keras.layers.Multiply(
name='lambda_latent_inter.')([discrimination_param, latent_trait])
disc_diff_interaction = keras.layers.Multiply(
name='alpha_param.')([discrimination_param, difficulty_level])
alpha_lambda_add = keras.layers.Subtract(name='alpha_lambda_add')(
[disc_latent_interaction, disc_diff_interaction])
sigmoid_layer = Activation(
'sigmoid', name='Sigmoid_func')(alpha_lambda_add)
guess_param = Dense(model_params['guess_params']['units'], use_bias=model_params['guess_params']['use_bias'],
kernel_initializer=model_params['guess_params']['kernel'],
bias_initializer=model_params['guess_params']['bias'],
kernel_regularizer=l1_l2(
l1=model_params['guess_params']['regularizers']['l1'],
l2=model_params['guess_params']['regularizers']['l2']),
activity_regularizer=l1_l2(
l1=model_params['guess_params']['group_lasso']['l1'],
l2=model_params['guess_params']['group_lasso']['l2']),
trainable=model_params['guess_params']['train'],
activation=model_params['guess_params']['act'], name='guessing_param')(quest_input_layer)
slip_param= Dense(model_params['slip_params']['units'], use_bias=model_params['slip_params']['use_bias'],
kernel_initializer=model_params['slip_params']['kernel'],
bias_initializer=model_params['slip_params']['bias'],
kernel_regularizer=l1_l2(
l1=model_params['slip_params']['regularizers']['l1'],
l2=model_params['slip_params']['regularizers']['l2']),
activity_regularizer=l1_l2(
l1=model_params['slip_params']['group_lasso']['l1'],
l2=model_params['slip_params']['group_lasso']['l2']),
trainable=model_params['slip_params']['train'],
activation=model_params['slip_params']['act'], name='slip_param')(quest_input_layer)
guess_param_interaction = Lambda(lambda x: 1 - x, name='slip_param_inter.')(slip_param)
guess_param_interaction = keras.layers.Subtract(name= 'slip/guess_interaction')([guess_param_interaction, guess_param])#2
#guess_param_interaction = Lambda(lambda x: K.constant(value=np.array(
#[1 - model_params['guess_params']['slip']])) - x, name='guess_param_inter.')(guess_param)
guess_param_interaction = keras.layers.Multiply(
name='disc/guess_param_inter.')([sigmoid_layer, guess_param_interaction])
guess_param_interaction = keras.layers.Add(
name='guess_param_inter/add')([guess_param, guess_param_interaction])
prediction_output = Dense(model_params['hyper_params']['units'], trainable=False, use_bias=False,
kernel_initializer=keras.initializers.Ones(), name='prediction_layer')(guess_param_interaction)
model_ = Model(
inputs=[user_input_layer, quest_input_layer], outputs=prediction_output)
model_.compile(loss=model_params['hyper_params']['loss'],
optimizer=model_params['hyper_params']['optimizer'], metrics=['mae', 'accuracy'])
return model_
