def __init__(self, kernels: List[tuple], lstm_kernels: List[tuple], up_factor=2, data_format='NHWC', layer_ind_up = 0, pretraining =False, return_logits=False): super(UpBlock2D, self).__init__() self.data_format_keras = 'channels_first' if data_format[1] == 'C' else 'channels_last' self.up_factor = up_factor self.channel_axis = 1 if data_format[1] == 'C' else -1 self.ConvLSTM = [] self.Conv = [] self.BN = [] self.LReLU = [] self.return_logits = return_logits self.pretraining = pretraining if lstm_kernels is not None: for kxy_lstm, kout_lstm, dropout, reg, kernel_init in lstm_kernels: self.ConvLSTM.append(k.layers.ConvLSTM2D(filters=kout_lstm, kernel_size=kxy_lstm, strides=1, padding='same', data_format=self.data_format_keras, kernel_initializer=kernel_init, return_sequences=True, stateful=False, recurrent_dropout=dropout, kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1]))) #initialization of Conv + Batch + ReLU for l_ind, (kxy, kout, dropout, reg, kernel_init) in enumerate(kernels): self.Conv.append(k.layers.Conv2D(filters=kout, kernel_size=kxy, strides=1, use_bias=True, kernel_initializer=kernel_init, data_format=self.data_format_keras, padding='same', kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1]))) self.BN.append(k.layers.BatchNormalization(axis=self.channel_axis)) self.LReLU.append(k.layers.LeakyReLU())
def __init__(self, conv_kernels: List[tuple], lstm_kernels: List[tuple], data_format='NHWC', pretraining = False): super(DownBlock2D, self).__init__() data_format_keras = 'channels_first' if data_format[1] == 'C' else 'channels_last' channel_axis = 1 if data_format[1] == 'C' else -1 self.ConvLSTM = [] self.Conv = [] self.BN = [] self.LReLU = [] self.total_stride = 1 self.pretraining = pretraining #initialization of convLSTM2D layers for kxy_lstm, kout_lstm, dropout, reg, kernel_init in lstm_kernels: self.ConvLSTM.append(k.layers.ConvLSTM2D(filters=kout_lstm, kernel_size=kxy_lstm, strides=1, padding='same', data_format=data_format_keras, kernel_initializer=kernel_init, return_sequences=True, stateful=False, recurrent_dropout=dropout, kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1]))) #initialization of Conv + Batch + ReLU for l_ind, (kxy, kout, dropout, reg, kernel_init) in enumerate(conv_kernels): self.Conv.append(k.layers.Conv2D(filters=kout, kernel_size=kxy, strides=1, use_bias=True, kernel_initializer=kernel_init, data_format=data_format_keras, padding='same', kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1]))) self.BN.append(k.layers.BatchNormalization(axis=channel_axis)) self.LReLU.append(k.layers.LeakyReLU()) #initialization of maxpooling layer self.MaxPool = k.layers.MaxPool2D(pool_size=(2, 2))
def model_graph(self, l1=1e-5, l2=1e-4, hidden_size=[256, 128]): a = Input(shape=(self.node_size, )) l = Input(shape=(None, )) output = a for i in range(len(hidden_size)): if i == len(hidden_size) - 1: output = Dense(hidden_size[i], activation="relu", kernel_regularizer=l1_l2(l1, l2), name="1st")(output) else: output = Dense(hidden_size[i], activation="relu", kernel_regularizer=l1_l2(l1, l2))(output) # 记录一阶向量 y = output for i in reversed(range(len(hidden_size) - 1)): output = Dense(hidden_size[i], activation="relu", kernel_regularizer=l1_l2(l1, l2))(output) a_ = Dense(self.node_size, activation="relu", name="2nd")(output) model = Model(inputs=[a, l], outputs=[a_, y]) emb = Model(inputs=a, outputs=y) return model, emb
def create_model(node_size, hidden_size=[256, 128], l1=1e-5, l2=1e-4): try: pass except ImportWarning: print("tensorflow not found, please install") from tensorflow.python.keras.layers import Dense from tensorflow.python.keras.layers import Input from tensorflow.python.keras.models import Model from tensorflow.python.keras.regularizers import l1_l2 A = Input(shape=(node_size, )) L = Input(shape=(None, )) fc = A for i in range(len(hidden_size)): if i == len(hidden_size) - 1: fc = Dense( hidden_size[i], activation="relu", kernel_regularizer=l1_l2(l1, l2), name="1st", )(fc) else: fc = Dense(hidden_size[i], activation="relu", kernel_regularizer=l1_l2(l1, l2))(fc) Y = fc for i in reversed(range(len(hidden_size) - 1)): fc = Dense(hidden_size[i], activation="relu", kernel_regularizer=l1_l2(l1, l2))(fc) A_ = Dense(node_size, "relu", name="2nd")(fc) model = Model(inputs=[A, L], outputs=[A_, Y]) emb = Model(inputs=A, outputs=Y) return model, emb
def EEGNet_old(nb_classes, Chans = 64, Samples = 128, regRate = 0.0001, dropoutRate = 0.25, kernels = [(2, 32), (8, 4)], strides = (2, 4)): """ Keras Implementation of EEGNet_v1 (https://arxiv.org/abs/1611.08024v2) This model is the original EEGNet model proposed on arxiv https://arxiv.org/abs/1611.08024v2 with a few modifications: we use striding instead of max-pooling as this helped slightly in classification performance while also providing a computational speed-up. Note that we no longer recommend the use of this architecture, as the new version of EEGNet performs much better overall and has nicer properties. Inputs: nb_classes : total number of final categories Chans, Samples : number of EEG channels and samples, respectively regRate : regularization rate for L1 and L2 regularizations dropoutRate : dropout fraction kernels : the 2nd and 3rd layer kernel dimensions (default is the [2, 32] x [8, 4] configuration) strides : the stride size (note that this replaces the max-pool used in the original paper) """ # start the model input_main = Input((1, Chans, Samples)) layer1 = Conv2D(16, (Chans, 1), input_shape=(1, Chans, Samples), kernel_regularizer = l1_l2(l1=regRate, l2=regRate))(input_main) layer1 = BatchNormalization(axis=1)(layer1) layer1 = Activation('elu')(layer1) layer1 = Dropout(dropoutRate)(layer1) permute_dims = 2, 1, 3 permute1 = Permute(permute_dims)(layer1) layer2 = Conv2D(4, kernels[0], padding = 'same', kernel_regularizer=l1_l2(l1=0.0, l2=regRate), strides = strides)(permute1) layer2 = BatchNormalization(axis=1)(layer2) layer2 = Activation('elu')(layer2) layer2 = Dropout(dropoutRate)(layer2) layer3 = Conv2D(4, kernels[1], padding = 'same', kernel_regularizer=l1_l2(l1=0.0, l2=regRate), strides = strides)(layer2) layer3 = BatchNormalization(axis=1)(layer3) layer3 = Activation('elu')(layer3) layer3 = Dropout(dropoutRate)(layer3) flatten = Flatten(name = 'flatten')(layer3) dense = Dense(nb_classes, name = 'dense')(flatten) softmax = Activation('softmax', name = 'softmax')(dense) return Model(inputs=input_main, outputs=softmax)
def SepConv_BN(x, filters, prefix, stride=1, kernel_size=3, rate=1, depth_activation=False, epsilon=1e-3, regularizer_l1=0.0, regularizer_l2=0.0): """ SepConv with BN between depthwise & pointwise. Optionally add activation after BN Implements right "same" padding for even kernel sizes Args: x: input tensor filters: num of filters in pointwise convolution prefix: prefix before name stride: stride at depthwise conv kernel_size: kernel size for depthwise convolution rate: atrous rate for depthwise convolution depth_activation: flag to use activation between depthwise & poinwise convs epsilon: epsilon to use in BN layer """ if stride == 1: depth_padding = 'same' else: kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1) pad_total = kernel_size_effective - 1 pad_beg = pad_total // 2 pad_end = pad_total - pad_beg x = ZeroPadding2D((pad_beg, pad_end))(x) depth_padding = 'valid' if not depth_activation: x = Activation('relu')(x) x = DepthwiseConv2D((kernel_size, kernel_size), strides=(stride, stride), dilation_rate=(rate, rate), padding=depth_padding, use_bias=False, name=prefix + '_depthwise', kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2))(x) x = BatchNormalization(name=prefix + '_depthwise_BN', epsilon=epsilon)(x) if depth_activation: x = Activation('relu')(x) x = Conv2D(filters, (1, 1), padding='same', use_bias=False, name=prefix + '_pointwise', kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2))(x) x = BatchNormalization(name=prefix + '_pointwise_BN', epsilon=epsilon)(x) if depth_activation: x = Activation('relu')(x) return x
def __init__(self, fl, hparams): """ Initialises new ANN model :param fl: fl class :param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function """ self.features_dim = fl.features_c_dim self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output self.hparams = hparams self.normalise_labels = fl.normalise_labels self.labels_scaler = fl.labels_scaler features_in = Input(shape=(self.features_dim, ), name='main_features_c_input') # Build keras ANN model x = Dense(units=hparams['nodes'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Layer_' + str(0))(features_in) x = Dense(units=hparams['nodes'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Layer_' + str(1))(x) x = Dense(units=hparams['nodes'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Layer_' + str(2))(x) # x = BatchNormalization()(x) x = Dense(units=self.labels_dim, activation='linear', kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Final')(x) self.model = Model(inputs=features_in, outputs=x) optimizer = Adam(learning_rate=hparams['learning_rate'], clipnorm=1) def mean_relative_error(y_true, y_pred): diff = K.abs( (y_true - y_pred) / K.reshape(K.clip(K.abs(y_true[:, -1]), K.epsilon(), None), (-1, 1))) return 100. * K.mean(diff, axis=-1) if hparams['loss'] == 'mape': self.model.compile(optimizer=optimizer, loss=MeanAbsolutePercentageError()) elif hparams['loss'] == 'mre': self.model.compile(optimizer=optimizer, loss=mean_relative_error) elif hparams['loss'] == 'mse': self.model.compile(optimizer=optimizer, loss='mean_squared_error')
def create_model(self, **kwargs): kwargs.setdefault('metric', 'accuracy') model_params = _parse_params(self._model_params, return_as='nested') # Why make it private? Alternate name? # Move parsing to base model model = Sequential() if len(self.y.shape) == 1 or self.y.shape[1] == 1: ## Use OHE for all classification algorithms ## Check for class numbers in y ## Use that as units count units = 1 else: units = self.y.shape[1] model_params['layer_1'].update({'input_dim': self.X.shape[1], 'units': units}) model.add(Dense(units=model_params['layer_1']['units'], input_dim=model_params['layer_1']['input_dim'], activation=model_params['layer_1']['activation'], kernel_regularizer=l1_l2(l1=model_params['layer_1']['l1'], l2=model_params['layer_1']['l2']))) model.compile(optimizer=model_params['optimizer'], loss=model_params['loss'], metrics=[kwargs['metric']]) return model
def default_regularizer(*args, **kwargs): l1, l2 = 0.0, 0.0 if (len(args) == 0 and len(kwargs) == 0) or args[0] is None: return None elif len(args) == 1: if isinstance(args[0], (float, int)): l1 = 0.0 l2 = args[0] elif isinstance(args[0], list): l1 = args[0][0] l2 = args[0][1] elif isinstance(args[0], dict): l1 = 0.0 if 'l1' not in args[0] else args[0]['l1'] l2 = 0.0 if 'l2' not in args[0] else args[0]['l2'] elif len(args) == 2: l1 = args[0] l2 = args[1] elif len(kwargs) > 0: l1 = 0.0 if 'l1' not in kwargs else kwargs['l1'] l2 = 0.0 if 'l2' not in kwargs else kwargs['l2'] else: raise ValueError( 'Unrecognized entry - input regularization values for l1 and l2.') # print("regularization is used with l1={} and l2={}".format(l1, l2)) return l1_l2(l1=l1, l2=l2)
def _conv_single(inputs, filter_num, name=None): conv = Conv2D(filters=filter_num, kernel_size=(1, 1), strides=(1, 1), kernel_initializer='he_uniform', padding='valid', kernel_regularizer=l1_l2(l1=FLAGS.l1, l2=FLAGS.l2), name=name)(inputs) return conv
def ann(features_dim: int, labels_dim: int, hparams: Dict): """ Neural network base model to be called by class Model to build a full network. The base model provides a callable keras model that has input of shape specified by features_dim. There are multiple output defined by the list provided by labels_dim. :param features_dim: Defines the input shape of the callable keras model :param labels_dim: Defines how many output there will be and its respective shapes. :param hparams: Dict created by create_hparam function. Specifies the various hyperparameters for the neural network :return: Callable Keras neural network """ # Hyperparameters shared_layers = hparams['shared_layers'] assert shared_layers, 'shared_layers is an empty list. Ensure that it is not empty as there must at least be 1 ' \ 'shared dense layer.' # Input f_in = Input(shape=(features_dim, ), name='ANN_Input') # Shared layers for idx, nodes in enumerate(shared_layers): if idx == 0: x = Dense(units=nodes, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='shared_' + str(idx))(f_in) elif nodes != 0: x = Dense(units=nodes, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='shared_' + str(idx))(x) # Final output to the correct label dimension single_task = Dense(units=labels_dim, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(x) # Create input output model model = Model(inputs=f_in, outputs=single_task) return model
def modify_model(model: Model, class_index: int, importance_type: ImportanceType) -> Model: gamma_initializer: str = "zeros" if importance_type & ImportanceType.GAMMA: gamma_initializer = "ones" gamma_regularizer = None if importance_type & ImportanceType.L1 and not importance_type & ImportanceType.L2: gamma_regularizer = l1() if not importance_type & ImportanceType.L1 and importance_type & ImportanceType.L2: gamma_regularizer = l2() if importance_type & ImportanceType.L1 and importance_type & ImportanceType.L2: gamma_regularizer = l1_l2() max_layer: int = len(model.layers) last_output: Input = None network_input: Input = None for i, layer in enumerate(model.layers): if i == 0: last_output = layer.output network_input = layer.input if 0 < i < max_layer: new_layer: Union[BatchNormalization, BatchNormalization] = BatchNormalization( center=(importance_type & ImportanceType.CENTERING), gamma_initializer=gamma_initializer, gamma_regularizer=gamma_regularizer) last_output = new_layer(last_output) if i == max_layer - 1: new_end_layer: Dense = Dense(2, activation="softmax", name="binary_output_layer") last_output = new_end_layer(last_output) old_weights = layer.get_weights() old_weights[0] = np.transpose(old_weights[0], (1, 0)) new_weights: List[np.array] = [ np.append(old_weights[0][class_index:class_index + 1], np.subtract( np.sum(old_weights[0], axis=0, keepdims=True), old_weights[0][class_index:class_index + 1]), axis=0), np.append(old_weights[1][class_index:class_index + 1], np.subtract( np.sum(old_weights[1], axis=0, keepdims=True), old_weights[1][class_index:class_index + 1]), axis=0) ] new_weights[0] = np.transpose(new_weights[0], (1, 0)) new_end_layer.set_weights(new_weights) elif i > 0: last_output = layer(last_output) return Model(inputs=network_input, outputs=last_output)
def create_model(node_size, hidden_size=[256, 128], l1=1e-5, l2=1e-4): A = Input(shape=(node_size,)) L = Input(shape=(None,)) fc = A for i in range(len(hidden_size)): if i == len(hidden_size) - 1: fc = Dense(hidden_size[i], activation='relu', kernel_regularizer=l1_l2(l1, l2), name='1st')(fc) else: fc = Dense(hidden_size[i], activation='relu', kernel_regularizer=l1_l2(l1, l2))(fc) Y = fc for i in reversed(range(len(hidden_size) - 1)): fc = Dense(hidden_size[i], activation='relu', kernel_regularizer=l1_l2(l1, l2))(fc) A_ = Dense(node_size, 'relu', name='2nd')(fc) model = Model(inputs=[A, L], outputs=[A_, Y]) emb = Model(inputs=A, outputs=Y) return model, emb
def _conv2d_3(inputs, filter_num, name=None): conv = Conv2D(filters=filter_num, kernel_size=(3, 3), strides=(1, 1), kernel_initializer='he_uniform', padding='same', kernel_regularizer=l1_l2(l1=FLAGS.l1, l2=FLAGS.l2), name=name)(inputs) in_norm = InstanceNormalization()(conv) relu = ReLU()(in_norm) return relu
def decoder_block(a, n_filters): a = layers.Conv2DTranspose( filters=n_filters, kernel_size=(4, 4), padding='same', strides=(2, 2), kernel_regularizer=regularizers.l1_l2( l1=Config.l1_kernel_regularization, l2=Config.l2_kernel_regularization))(a) a = layers.BatchNormalization()(a) a = layers.ReLU()(a) return a
def _conv2d_same(x, filters, prefix, stride=1, kernel_size=3, rate=1, regularizer_l1=0.0, regularizer_l2=0.0): """Implements right 'same' padding for even kernel sizes Without this there is a 1 pixel drift when stride = 2 Args: x: input tensor filters: num of filters in pointwise convolution prefix: prefix before name stride: stride at depthwise conv kernel_size: kernel size for depthwise convolution rate: atrous rate for depthwise convolution """ if stride == 1: return Conv2D(filters, (kernel_size, kernel_size), strides=(stride, stride), padding='same', use_bias=False, dilation_rate=(rate, rate), kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2), name=prefix)(x) else: kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1) pad_total = kernel_size_effective - 1 pad_beg = pad_total // 2 pad_end = pad_total - pad_beg x = ZeroPadding2D((pad_beg, pad_end))(x) return Conv2D(filters, (kernel_size, kernel_size), strides=(stride, stride), padding='valid', use_bias=False, dilation_rate=(rate, rate), kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2), name=prefix)(x)
def encoder_block(a, n_filters): a = layers.Conv2D(filters=n_filters, kernel_size=(4, 4), padding='same', kernel_regularizer=regularizers.l1_l2( l1=Config.l1_kernel_regularization, l2=Config.l2_kernel_regularization))(a) a = layers.BatchNormalization()(a) a = layers.LeakyReLU()(a) a = layers.MaxPool2D(pool_size=(2, 2))(a) if Config.use_spatial_dropout: a = layers.SpatialDropout2D( rate=Config.spatial_dropout_rate)(a) return a
def make_model(num_classes): reg = regularizers.l1_l2(0.01, 0.01) cb = EfficientNetB2(weights='imagenet', include_top=False, drop_connect_rate=0.4, pooling='avg', input_shape=(256, 256, 3)) #cb.trainable = False freeze_model(cb, 3) x = cb.output #x = layers.GaussianNoise(0.5)(x) #x = layers.BatchNormalization()(x) #x = layers.Dropout(0.3)(x) #x = layers.Dense(128, activation='relu', kernel_regularizer=reg)(x) x = layers.Dropout(0.4)(x) x = layers.Dense(num_classes, activation='softmax', kernel_regularizer=reg)(x) model = Model(cb.input, x) #model.summary() #pr = tf.keras.metrics.AUC(name='PR', curve='PR') model.compile(optimizer=optimizers.Adam(1e-4), loss='sparse_categorical_crossentropy', metrics=['accuracy']) return model
def __init__(self, model_conf: ModelConfig, mode: RunMode, outputs): self.model_conf = model_conf self.utils = NetworkUtils(mode) self.dense = tf.keras.layers.Dense( units=self.model_conf.category_num + 2, kernel_initializer=tf.keras.initializers.he_normal(seed=None), kernel_regularizer=l1_l2(l1=0.01, l2=0.001), bias_initializer='zeros', ) self.time_distributed = lambda: tf.keras.layers.TimeDistributed( layer=self.dense, name='predict', )(inputs=outputs, training=self.utils.training) self.outputs = self.time_distributed() self.predict = tf.keras.backend.permute_dimensions(self.outputs, pattern=(1, 0, 2))
def __init__(self): self.batch_size = 15 self.learning_rate = 0.0001 self.epochs = 6000 self.time_stamps = 1524 self.num_steps = 64 self.dropout = 0.5 self.regularizer_type = 'l2' if self.regularizer_type == 'l2': self.regularizer = l2(1) elif self.regularizer_type == 'l1': self.regularizer = l1(1) elif self.regularizer == 'l1_l2': self.regularizer = l1_l2(1) self.model_type = '' self.data_type = 'all'
def build_nn(hidden_layer_sizes, input_dim, l1_reg, l2_reg, activation_func='tanh'): input_layer = Input(shape=(input_dim, )) x = input_layer # Fully connected hidden layers for i, size in enumerate(hidden_layer_sizes): x = Dense(size, activation=activation_func, kernel_regularizer=regularizers.l1_l2(l1=l1_reg, l2=l2_reg))(x) x = Dropout(0.3)(x) if i != len(hidden_layer_sizes) - 1 else x x = BatchNormalization()(x) # Wrap base_nn into a model base_nn = Model(inputs=input_layer, outputs=x, name="base_nn") return (base_nn, x)
def cross_stitch(features_dim: int, labels_dim: List[int], hparams: Dict): """ Neural network base model to be called by class Model to build a full network. The base model provides a callable keras model that has input of shape specified by features_dim. There are multiple output defined by the list provided by labels_dim. :param features_dim: Defines the input shape of the callable keras model :param labels_dim: Defines how many output there will be and its respective shapes. eg: [1, 1, 1, 2] ==> Multi-task learning with 4 separate outputs, each with shape 1, 1, 1, 2 respectively. :param hparams: Dict created by create_hparam function. Specifies the various hyperparameters for the neural network :return: Callable multi-task keras neural network Cross-stitch neural network paper https://ieeexplore.ieee.org/document/7780802 """ cs_layers = hparams['cs_layers'] assert cs_layers, 'cs_layers is an empty list. Ensure that it is not empty as there must at least be 1 ' \ 'cross stitch dense layer.' # Input f_in = Input(shape=(features_dim, ), name='CS_Input') # Cross stitch layers multi_task = [] for idx, nodes in enumerate(cs_layers): if idx == 0: for idx_task, _ in enumerate(labels_dim): x = Dense(units=nodes, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='cs_task_' + str(idx_task) + '_layer_' + str(idx))(f_in) multi_task.append(x) multi_task = CrossStitchLayer(name='cs_unit_' + str(idx))(multi_task) elif nodes != 0: temp = [] for idx_task, single_task in enumerate(multi_task): single_task = Dense(units=nodes, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='cs_task_' + str(idx_task) + '_layer_' + str(idx))(single_task) temp.append(single_task) multi_task = CrossStitchLayer(name='cs_unit_' + str(idx))(temp) temp = [] for idx_task, single_task_label in enumerate(labels_dim): single_task = multi_task[idx_task] single_task = Dense( units=single_task_label, activation='linear', kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='task_' + str(idx_task) + '_output_layer')(single_task) temp.append(single_task) multi_task = temp model = Model(inputs=f_in, outputs=multi_task, name='CS') return model
def hps(features_dim: int, labels_dim: List[int], hparams: Dict): """ Neural network base model to be called by class Model to build a full network. The base model provides a callable keras model that has input of shape specified by features_dim. There are multiple output defined by the list provided by labels_dim. :param features_dim: Defines the input shape of the callable keras model :param labels_dim: Defines how many output there will be and its respective shapes. eg: [1, 1, 1, 2] ==> Multi-task learning with 4 separate outputs, each with shape 1, 1, 1, 2 respectively. :param hparams: Dict created by create_hparam function. Specifies the various hyperparameters for the neural network :return: Callable multi-task keras neural network HPS = Hard Parameter Sharing. For reference: https://towardsdatascience.com/multitask-learning-teach-your-ai-more-to-make-it-better-dde116c2cd40 Dense neural network base, till the splitting point where each separate task has its own task specific dense neural network. Most basic form of multi-task learning. """ # Hyperparameters shared_layers = hparams['shared_layers'] ts_layers = hparams['ts_layers'] # ts stands for task specific assert shared_layers, 'shared_layers is an empty list. Ensure that it is not empty as there must at least be 1 ' \ 'shared dense layer.' assert ts_layers, 'ts_layers is an empty list. Ensure that it is not empty as there must at least be 1 task' \ 'task specific dense layer' # Input f_in = Input(shape=(features_dim, ), name='HPS_Input') # Shared layers for idx, nodes in enumerate(shared_layers): if idx == 0: x = Dense(units=nodes, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='shared_' + str(idx))(f_in) elif nodes != 0: x = Dense(units=nodes, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='shared_' + str(idx))(x) # Task Specific Layers multi_task = [] for idx_task, single_task_label in enumerate(labels_dim): # Task Specific layers for idx_layer, nodes in enumerate(ts_layers): if idx_layer == 0: single_task = Dense(units=nodes, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='task_' + str(idx_task) + '_layer_' + str(idx_layer))(x) else: single_task = Dense(units=nodes, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='task_' + str(idx_task) + '_layer_' + str(idx_layer))(single_task) # Final output to the correct label dimension single_task = Dense( units=single_task_label, activation='linear', kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='task_' + str(idx_task) + '_output_layer')(single_task) multi_task.append(single_task) # Create input output model model = Model(inputs=f_in, outputs=multi_task, name='HPS') return model
def __init__(self, fl, mode, hparams): """ Initialises new DNN model based on input features_dim, labels_dim, hparams :param features_dim: Number of input feature nodes. Integer :param labels_dim: Number of output label nodes. Integer :param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function. hparams includes: hidden_layers: List containing number of nodes in each hidden layer. [10, 20] means 10 then 20 nodes. """ # self.features_dim = fl.features_c_dim # self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output self.features_dim = fl.features_c_dim + 1 # 1 for the positional argument self.labels_dim = 1 self.numel = fl.labels.shape[1] + 1 self.hparams = hparams self.mode = mode self.normalise_labels = fl.normalise_labels self.labels_scaler = fl.labels_scaler features_in = Input(shape=(self.features_dim, ), name='main_features_c_input') # Selection of model if mode == 'ann': model = ann(self.features_dim, self.labels_dim, self.hparams) x = model(features_in) self.model = Model(inputs=features_in, outputs=x) elif mode == 'ann2': model_1 = ann(self.features_dim, 50, self.hparams) x = model_1(features_in) model_end = ann(50, 50, self.hparams) end = model_end(x) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) model_2 = ann(50, self.labels_dim - 1, self.hparams) x = model_2(x) self.model = Model(inputs=features_in, outputs=[end_node, x]) elif mode == 'ann3': x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(0))(features_in) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(2))(x) # x = BatchNormalization()(x) x = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_set_19')(x) self.model = Model(inputs=features_in, outputs=x) elif mode == 'conv1': x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='shared' + str(1))(features_in) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) #x = BatchNormalization()(x) x = Dense(units=19, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_set_19')(x) #x = BatchNormalization()(x) x = Reshape(target_shape=(19, 1))(x) x = Conv1D(filters=hparams['filters'], kernel_size=3, strides=1, padding='same', activation='relu')(x) #x = BatchNormalization()(x) x = Conv1D(filters=hparams['filters'] * 2, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = Conv1D(filters=hparams['filters'] * 4, kernel_size=3, strides=1, padding='same', activation='relu')(x) #x = Permute((2,1))(x) #x = GlobalAveragePooling1D()(x) x = TimeDistributed(Dense(1, activation='linear'))(x) x = Reshape(target_shape=(19, ))(x) self.model = Model(inputs=features_in, outputs=x) elif mode == 'conv2': x = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(1))(features_in) x = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(2))(x) end = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(1))(x) end = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(2))(end) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) x = Dense(units=80, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Reshape(target_shape=(80, 1))(x) x = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) x = Conv1D(filters=16, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) #x = Permute((2,1))(x) #x = GlobalAveragePooling1D()(x) x = TimeDistributed(Dense(1, activation='linear'))(x) x = Reshape(target_shape=(20, ))(x) self.model = Model(inputs=features_in, outputs=[end_node, x]) elif mode == 'lstm': x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(1))(features_in) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(2))(x) end = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(1))(x) end = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(2))(end) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(2))(x) x = RepeatVector(n=20)(x) x = LSTM(units=30, activation='relu', return_sequences=True)(x) x = LSTM(units=30, activation='relu', return_sequences=True)(x) x = TimeDistributed(Dense(1))(x) x = Reshape(target_shape=(20, ))(x) ''' x = Permute((2,1))(x) x = GlobalAveragePooling1D()(x) ''' self.model = Model(inputs=features_in, outputs=[end_node, x]) optimizer = Adam(clipnorm=1) self.model.compile(optimizer=optimizer, loss='mean_squared_error')
def __init__(self, fl, mode, hparams): """ Initialises new DNN model based on input features_dim, labels_dim, hparams :param features_dim: Number of input feature nodes. Integer :param labels_dim: Number of output label nodes. Integer :param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function. hparams includes: hidden_layers: List containing number of nodes in each hidden layer. [10, 20] means 10 then 20 nodes. """ self.features_dim = fl.features_c_dim self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output self.hparams = hparams self.mode = mode self.normalise_labels = fl.normalise_labels self.labels_scaler = fl.labels_scaler features_in = Input(shape=(self.features_dim, ), name='main_features_c_input') # Selection of model if mode == 'ann': model = ann(self.features_dim, self.labels_dim, self.hparams) x = model(features_in) self.model = Model(inputs=features_in, outputs=x) elif mode == 'ann2': model_1 = ann(self.features_dim, 50, self.hparams) x = model_1(features_in) model_end = ann(50, 50, self.hparams) end = model_end(x) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) model_2 = ann(50, self.labels_dim - 1, self.hparams) x = model_2(x) self.model = Model(inputs=features_in, outputs=[end_node, x]) elif mode == 'ann3': x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(0))(features_in) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(2))(x) # x = BatchNormalization()(x) x = Dense(units=self.labels_dim, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Final')(x) self.model = Model(inputs=features_in, outputs=x) elif mode == 'conv1': if fl.label_type == 'gf20': final_dim = 20 else: final_dim = 19 x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='shared' + str(1))(features_in) x = Dense(units=hparams['pre'], activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) #x = BatchNormalization()(x) x = Dense(units=final_dim, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_set_19')(x) #x = BatchNormalization()(x) x = Reshape(target_shape=(final_dim, 1))(x) x = Conv1D(filters=hparams['filters'], kernel_size=3, strides=1, padding='same', activation='relu')(x) #x = BatchNormalization()(x) x = Conv1D(filters=hparams['filters'] * 2, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = Conv1D(filters=hparams['filters'] * 4, kernel_size=3, strides=1, padding='same', activation='relu')(x) #x = Permute((2,1))(x) #x = GlobalAveragePooling1D()(x) x = TimeDistributed(Dense(1, activation='linear'))(x) x = Reshape(target_shape=(final_dim, ))(x) self.model = Model(inputs=features_in, outputs=x) elif mode == 'conv2': x = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(1))(features_in) x = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(2))(x) end = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(1))(x) end = Dense(units=10, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(2))(end) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) x = Dense(units=80, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Reshape(target_shape=(80, 1))(x) x = Conv1D(filters=8, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) x = Conv1D(filters=16, kernel_size=3, strides=1, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) #x = Permute((2,1))(x) #x = GlobalAveragePooling1D()(x) x = TimeDistributed(Dense(1, activation='linear'))(x) x = Reshape(target_shape=(20, ))(x) self.model = Model(inputs=features_in, outputs=[end_node, x]) elif mode == 'lstm': x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(1))(features_in) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Shared_e_' + str(2))(x) end = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(1))(x) end = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Dense_e_' + str(2))(end) end_node = Dense(units=1, activation='linear', kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='output_layer')(end) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(1))(x) x = Dense(units=20, activation=hparams['activation'], kernel_regularizer=regularizers.l1_l2( l1=hparams['reg_l1'], l2=hparams['reg_l2']), name='Pre_' + str(2))(x) x = RepeatVector(n=20)(x) x = LSTM(units=30, activation='relu', return_sequences=True)(x) x = LSTM(units=30, activation='relu', return_sequences=True)(x) x = TimeDistributed(Dense(1))(x) x = Reshape(target_shape=(20, ))(x) ''' x = Permute((2,1))(x) x = GlobalAveragePooling1D()(x) ''' self.model = Model(inputs=features_in, outputs=[end_node, x]) optimizer = Adam(learning_rate=hparams['learning_rate'], clipnorm=1) def weighted_mse(y_true, y_pred): loss_weights = np.sqrt(np.arange(1, 20)) #loss_weights = np.arange(1, 20) return K.mean(K.square(y_pred - y_true) * loss_weights, axis=-1) def haitao_error(y_true, y_pred): diff = K.abs( (y_true - y_pred) / K.reshape(K.clip(K.abs(y_true[:, -1]), K.epsilon(), None), (-1, 1))) return 100. * K.mean(diff, axis=-1) if hparams['loss'] == 'mape': self.model.compile(optimizer=optimizer, loss=MeanAbsolutePercentageError()) elif hparams['loss'] == 'haitao': self.model.compile(optimizer=optimizer, loss=haitao_error) elif hparams['loss'] == 'mse': self.model.compile(optimizer=optimizer, loss='mean_squared_error')
def __init__(self, kernels: List[tuple], up_factor=2, data_format='NHWC', layer_ind_up=0, weights_list=0, pretraining=False, pretraining_type='full', return_logits=False): super(UpBlock2D, self).__init__() self.data_format_keras = 'channels_first' if data_format[ 1] == 'C' else 'channels_last' self.up_factor = up_factor self.channel_axis = 1 if data_format[1] == 'C' else -1 self.Conv = [] self.BN = [] self.LReLU = [] self.return_logits = return_logits self.pretraining = pretraining for l_ind, (kxy, kout, dropout, reg, kernel_init) in enumerate(kernels): if self.pretraining == 'cells' and pretraining_type == 'full': C = tf.keras.initializers.Constant if layer_ind_up == 3: if l_ind != 2: self.Conv.append( k.layers.Conv2D( filters=kout, kernel_size=kxy, strides=1, use_bias=True, data_format=self.data_format_keras, padding='same', kernel_initializer=C(weights_list[0 + l_ind * 2]), bias_initializer=C(weights_list[1 + l_ind * 2]), kernel_regularizer=regularizers.l1_l2( l1=reg[0], l2=reg[1]))) self.BN.append( k.layers.BatchNormalization( axis=self.channel_axis, beta_initializer=C(weights_list[6 + l_ind * 4]), gamma_initializer=C(weights_list[7 + l_ind * 4]), moving_mean_initializer=C( weights_list[8 + l_ind * 4]), moving_variance_initializer=C( weights_list[9 + l_ind * 4]))) else: self.Conv.append( k.layers.Conv2D( filters=kout, kernel_size=kxy, strides=1, use_bias=True, data_format=self.data_format_keras, padding='same', kernel_initializer='he_normal', kernel_regularizer=regularizers.l1_l2( l1=reg[0], l2=reg[1]))) self.BN.append( k.layers.BatchNormalization( axis=self.channel_axis)) else: self.Conv.append( k.layers.Conv2D( filters=kout, kernel_size=kxy, strides=1, use_bias=True, data_format=self.data_format_keras, padding='same', kernel_initializer=C(weights_list[0 + l_ind * 2]), bias_initializer=C(weights_list[1 + l_ind * 2]), kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1]))) self.BN.append( k.layers.BatchNormalization( axis=self.channel_axis, beta_initializer=C(weights_list[4 + l_ind * 4]), gamma_initializer=C(weights_list[5 + l_ind * 4]), moving_mean_initializer=C(weights_list[6 + l_ind * 4]), moving_variance_initializer=C( weights_list[7 + l_ind * 4]))) # elif self.pretraining == 'imagenet': # C = tf.keras.initializers.Constant # if l_ind != 2: # self.Conv.append(k.layers.Conv2D(filters=kout, kernel_size=kxy, strides=1, use_bias=True, # data_format=self.data_format_keras, padding='same', kernel_initializer=C(weights_list[0 + l_ind]), # kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1]))) # self.BN.append(k.layers.BatchNormalization(axis=self.channel_axis, beta_initializer = C(weights_list[2 + l_ind*4]), # gamma_initializer = C(weights_list[3 + l_ind*4]), moving_mean_initializer = C(weights_list[4 + l_ind*4]), # moving_variance_initializer = C(weights_list[5 + l_ind*4]))) # else: # self.Conv.append(k.layers.Conv2D(filters=kout, kernel_size=kxy, strides=1, use_bias=True, # data_format=self.data_format_keras, padding='same', kernel_initializer='he_normal', # kernel_regularizer=regularizers.l1_l2(l1=reg[0], l2=reg[1]))) # self.BN.append(k.layers.BatchNormalization(axis=self.channel_axis)) else: self.Conv.append( k.layers.Conv2D(filters=kout, kernel_size=kxy, strides=1, use_bias=True, kernel_initializer=kernel_init, data_format=self.data_format_keras, padding='same', kernel_regularizer=regularizers.l1_l2( l1=reg[0], l2=reg[1]))) self.BN.append( k.layers.BatchNormalization(axis=self.channel_axis)) self.LReLU.append(k.layers.LeakyReLU())
class KerasRegularizersTest(keras_parameterized.TestCase, parameterized.TestCase): def create_model(self, kernel_regularizer=None, activity_regularizer=None): model = keras.models.Sequential() model.add( keras.layers.Dense(NUM_CLASSES, kernel_regularizer=kernel_regularizer, activity_regularizer=activity_regularizer, input_shape=(DATA_DIM, ))) return model def get_data(self): (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( train_samples=10, test_samples=10, input_shape=(DATA_DIM, ), num_classes=NUM_CLASSES) y_train = np_utils.to_categorical(y_train, NUM_CLASSES) y_test = np_utils.to_categorical(y_test, NUM_CLASSES) return (x_train, y_train), (x_test, y_test) def create_multi_input_model_from(self, layer1, layer2): input_1 = keras.layers.Input(shape=(DATA_DIM, )) input_2 = keras.layers.Input(shape=(DATA_DIM, )) out1 = layer1(input_1) out2 = layer2(input_2) out = keras.layers.Average()([out1, out2]) model = keras.models.Model([input_1, input_2], out) model.add_loss(keras.backend.mean(out2)) model.add_loss(math_ops.reduce_sum(input_1)) return model @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_kernel_regularization(self, regularizer): (x_train, y_train), _ = self.get_data() model = self.create_model(kernel_regularizer=regularizer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) self.assertEqual(len(model.losses), 1) model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0) @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ('l2_zero', keras.regularizers.l2(0.)), ]) def test_activity_regularization(self, regularizer): (x_train, y_train), _ = self.get_data() model = self.create_model(activity_regularizer=regularizer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) self.assertEqual(len(model.losses), 1 if context.executing_eagerly() else 1) model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0) @keras_parameterized.run_all_keras_modes @keras_parameterized.run_with_all_model_types def test_zero_regularization(self): # Verifies that training with zero regularization works. x, y = np.ones((10, 10)), np.ones((10, 3)) model = testing_utils.get_model_from_layers([ keras.layers.Dense(3, kernel_regularizer=keras.regularizers.l2(0)) ], input_shape=(10, )) model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) model.fit(x, y, batch_size=5, epochs=1) def test_custom_regularizer_saving(self): def my_regularizer(weights): return math_ops.reduce_sum(math_ops.abs(weights)) inputs = keras.Input((10, )) outputs = keras.layers.Dense(1, kernel_regularizer=my_regularizer)(inputs) model = keras.Model(inputs, outputs) model2 = model.from_config( model.get_config(), custom_objects={'my_regularizer': my_regularizer}) self.assertEqual(model2.layers[1].kernel_regularizer, my_regularizer) @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_regularization_shared_layer(self, regularizer): dense_layer = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer, activity_regularizer=regularizer) model = self.create_multi_input_model_from(dense_layer, dense_layer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) self.assertEqual(len(model.losses), 5) @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_regularization_shared_model(self, regularizer): dense_layer = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer, activity_regularizer=regularizer) input_tensor = keras.layers.Input(shape=(DATA_DIM, )) dummy_model = keras.models.Model(input_tensor, dense_layer(input_tensor)) model = self.create_multi_input_model_from(dummy_model, dummy_model) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) self.assertEqual(len(model.losses), 6) @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_regularization_shared_layer_in_different_models( self, regularizer): shared_dense = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer, activity_regularizer=regularizer) models = [] for _ in range(2): input_tensor = keras.layers.Input(shape=(DATA_DIM, )) unshared_dense = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer) out = unshared_dense(shared_dense(input_tensor)) models.append(keras.models.Model(input_tensor, out)) model = self.create_multi_input_model_from(layer1=models[0], layer2=models[1]) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) self.assertEqual(len(model.losses), 14)
def _inverted_res_block(inputs, expansion, stride, alpha, filters, block_id, skip_connection, rate=1, regularizer_l1=0.0, regularizer_l2=0.0): in_channels = inputs.shape.as_list()[-1] # inputs._keras_shape[-1] pointwise_conv_filters = int(filters * alpha) pointwise_filters = _make_divisible(pointwise_conv_filters, 8) x = inputs prefix = 'expanded_conv_{}_'.format(block_id) if block_id: # Expand x = Conv2D(expansion * in_channels, kernel_size=1, padding='same', use_bias=False, activation=None, kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2), name=prefix + 'expand')(x) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'expand_BN')(x) x = Activation(relu6, name=prefix + 'expand_relu')(x) else: prefix = 'expanded_conv_' # Depthwise x = DepthwiseConv2D(kernel_size=3, strides=stride, activation=None, use_bias=False, padding='same', dilation_rate=(rate, rate), kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2), name=prefix + 'depthwise')(x) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'depthwise_BN')(x) x = Activation(relu6, name=prefix + 'depthwise_relu')(x) # Project x = Conv2D(pointwise_filters, kernel_size=1, padding='same', use_bias=False, activation=None, kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2), name=prefix + 'project')(x) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name=prefix + 'project_BN')(x) if skip_connection: return Add(name=prefix + 'add')([inputs, x]) # if in_channels == pointwise_filters and stride == 1: # return Add(name='res_connect_' + str(block_id))([inputs, x]) return x
def Deeplabv3(weights='pascal_voc', input_tensor=None, input_shape=(512, 512, 3), classes=21, backbone='mobilenetv2', OS=16, alpha=1., activation=None, regularizer_l1=0.0, regularizer_l2=1e-4): """ Instantiates the Deeplabv3+ architecture Optionally loads weights pre-trained on PASCAL VOC or Cityscapes. This model is available for TensorFlow only. # Arguments weights: one of 'pascal_voc' (pre-trained on pascal voc), 'cityscapes' (pre-trained on cityscape) or None (random initialization) input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: shape of input image. format HxWxC PASCAL VOC model was trained on (512,512,3) images. None is allowed as shape/width classes: number of desired classes. PASCAL VOC has 21 classes, Cityscapes has 19 classes. If number of classes not aligned with the weights used, last layer is initialized randomly backbone: backbone to use. one of {'xception','mobilenetv2'} activation: optional activation to add to the top of the network. One of 'softmax', 'sigmoid' or None OS: determines input_shape/feature_extractor_output ratio. One of {8,16}. Used only for xception backbone. alpha: controls the width of the MobileNetV2 network. This is known as the width multiplier in the MobileNetV2 paper. - If `alpha` < 1.0, proportionally decreases the number of filters in each layer. - If `alpha` > 1.0, proportionally increases the number of filters in each layer. - If `alpha` = 1, default number of filters from the paper are used at each layer. Used only for mobilenetv2 backbone. Pretrained is only available for alpha=1. # Returns A Keras model instance. # Raises RuntimeError: If attempting to run this model with a backend that does not support separable convolutions. ValueError: in case of invalid argument for `weights` or `backbone` """ if not (weights in {'pascal_voc', 'cityscapes', None}): raise ValueError( 'The `weights` argument should be either ' '`None` (random initialization), `pascal_voc`, or `cityscapes` ' '(pre-trained on PASCAL VOC)') if not (backbone in {'xception', 'mobilenetv2'}): raise ValueError('The `backbone` argument should be either ' '`xception` or `mobilenetv2` ') if input_tensor is None: img_input = Input(shape=input_shape) else: img_input = input_tensor if backbone == 'xception': if OS == 8: entry_block3_stride = 1 middle_block_rate = 2 # ! Not mentioned in paper, but required exit_block_rates = (2, 4) atrous_rates = (12, 24, 36) else: entry_block3_stride = 2 middle_block_rate = 1 exit_block_rates = (1, 2) atrous_rates = (6, 12, 18) x = Conv2D(32, (3, 3), strides=(2, 2), name='entry_flow_conv1_1', use_bias=False, kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2), padding='same')(img_input) x = BatchNormalization(name='entry_flow_conv1_1_BN')(x) x = Activation('relu')(x) x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = BatchNormalization(name='entry_flow_conv1_2_BN')(x) x = Activation('relu')(x) x = _xception_block(x, [128, 128, 128], 'entry_flow_block1', skip_connection_type='conv', stride=2, depth_activation=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x, skip1 = _xception_block(x, [256, 256, 256], 'entry_flow_block2', skip_connection_type='conv', stride=2, depth_activation=False, return_skip=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _xception_block(x, [728, 728, 728], 'entry_flow_block3', skip_connection_type='conv', stride=entry_block3_stride, depth_activation=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) for i in range(16): x = _xception_block(x, [728, 728, 728], 'middle_flow_unit_{}'.format(i + 1), skip_connection_type='sum', stride=1, rate=middle_block_rate, depth_activation=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _xception_block(x, [728, 1024, 1024], 'exit_flow_block1', skip_connection_type='conv', stride=1, rate=exit_block_rates[0], depth_activation=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _xception_block(x, [1536, 1536, 2048], 'exit_flow_block2', skip_connection_type='none', stride=1, rate=exit_block_rates[1], depth_activation=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) else: OS = 8 first_block_filters = _make_divisible(32 * alpha, 8) x = Conv2D(first_block_filters, kernel_size=3, strides=(2, 2), padding='same', use_bias=False, kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2), name='Conv')(img_input) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_BN')(x) x = Activation(relu6, name='Conv_Relu6')(x) x = _inverted_res_block(x, filters=16, alpha=alpha, stride=1, expansion=1, block_id=0, skip_connection=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=24, alpha=alpha, stride=2, expansion=6, block_id=1, skip_connection=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=24, alpha=alpha, stride=1, expansion=6, block_id=2, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=32, alpha=alpha, stride=2, expansion=6, block_id=3, skip_connection=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=32, alpha=alpha, stride=1, expansion=6, block_id=4, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=32, alpha=alpha, stride=1, expansion=6, block_id=5, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) # stride in block 6 changed from 2 -> 1, so we need to use rate = 2 x = _inverted_res_block( x, filters=64, alpha=alpha, stride=1, # 1! expansion=6, block_id=6, skip_connection=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2, expansion=6, block_id=7, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2, expansion=6, block_id=8, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2, expansion=6, block_id=9, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2, expansion=6, block_id=10, skip_connection=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2, expansion=6, block_id=11, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2, expansion=6, block_id=12, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block( x, filters=160, alpha=alpha, stride=1, rate=2, # 1! expansion=6, block_id=13, skip_connection=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=160, alpha=alpha, stride=1, rate=4, expansion=6, block_id=14, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=160, alpha=alpha, stride=1, rate=4, expansion=6, block_id=15, skip_connection=True, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = _inverted_res_block(x, filters=320, alpha=alpha, stride=1, rate=4, expansion=6, block_id=16, skip_connection=False, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) # end of feature extractor # branching for Atrous Spatial Pyramid Pooling # Image Feature branch shape_before = tf.shape(x) b4 = GlobalAveragePooling2D()(x) # from (b_size, channels)->(b_size, 1, 1, channels) b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4) b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4) b4 = Conv2D(256, (1, 1), padding='same', use_bias=False, kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2), name='image_pooling')(b4) b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4) b4 = Activation('relu')(b4) # upsample. have to use compat because of the option align_corners size_before = tf.keras.backend.int_shape(x) b4 = Lambda(lambda x: tf.compat.v1.image.resize( x, size_before[1:3], align_corners=True))(b4) # simple 1x1 b0 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='aspp0', kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2))(x) b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0) b0 = Activation('relu', name='aspp0_activation')(b0) # there are only 2 branches in mobilenetV2. not sure why if backbone == 'xception': # rate = 6 (12) b1 = SepConv_BN(x, 256, 'aspp1', rate=atrous_rates[0], depth_activation=True, epsilon=1e-5, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) # rate = 12 (24) b2 = SepConv_BN(x, 256, 'aspp2', rate=atrous_rates[1], depth_activation=True, epsilon=1e-5, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) # rate = 18 (36) b3 = SepConv_BN(x, 256, 'aspp3', rate=atrous_rates[2], depth_activation=True, epsilon=1e-5, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) # concatenate ASPP branches & project x = Concatenate()([b4, b0, b1, b2, b3]) else: x = Concatenate()([b4, b0]) x = Conv2D(256, (1, 1), padding='same', use_bias=False, name='concat_projection')(x) x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x) x = Activation('relu')(x) x = Dropout(0.1)(x) # DeepLab v.3+ decoder if backbone == 'xception': # Feature projection # x4 (x2) block size_before2 = tf.keras.backend.int_shape(x) x = Lambda(lambda xx: tf.compat.v1.image.resize( xx, size_before2[1:3] * tf.constant(OS // 4), align_corners=True))( x) dec_skip1 = Conv2D(48, (1, 1), padding='same', kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2), use_bias=False, name='feature_projection0')(skip1) dec_skip1 = BatchNormalization(name='feature_projection0_BN', epsilon=1e-5)(dec_skip1) dec_skip1 = Activation('relu')(dec_skip1) x = Concatenate()([x, dec_skip1]) x = SepConv_BN(x, 256, 'decoder_conv0', depth_activation=True, epsilon=1e-5, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) x = SepConv_BN(x, 256, 'decoder_conv1', depth_activation=True, epsilon=1e-5, regularizer_l1=regularizer_l1, regularizer_l2=regularizer_l2) # you can use it with arbitary number of classes if (weights == 'pascal_voc' and classes == 21) or (weights == 'cityscapes' and classes == 19): last_layer_name = 'logits_semantic' else: last_layer_name = 'custom_logits_semantic' x = Conv2D(classes, (1, 1), padding='same', name=last_layer_name, kernel_regularizer=l1_l2(regularizer_l1, regularizer_l2))(x) size_before3 = tf.keras.backend.int_shape(img_input) x = Lambda(lambda xx: tf.compat.v1.image.resize( xx, size_before3[1:3], align_corners=True))(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input if activation in {'softmax', 'sigmoid'}: x = tf.keras.layers.Activation(activation)(x) model = Model(inputs, x, name='deeplabv3plus') # load weights if weights == 'pascal_voc': if backbone == 'xception': weights_path = get_file( 'deeplabv3_xception_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH_X, cache_subdir='models') else: weights_path = get_file( 'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH_MOBILE, cache_subdir='models') model.load_weights(weights_path, by_name=True) elif weights == 'cityscapes': if backbone == 'xception': weights_path = get_file( 'deeplabv3_xception_tf_dim_ordering_tf_kernels_cityscapes.h5', WEIGHTS_PATH_X_CS, cache_subdir='models') else: weights_path = get_file( 'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels_cityscapes.h5', WEIGHTS_PATH_MOBILE_CS, cache_subdir='models') model.load_weights(weights_path, by_name=True) return model
def create_model(self, **kwargs): model_params = _parse_params(self._model_params, return_as='nested') model_params.update( {'input_dims_users': self.x_train_user.shape[1], 'input_dims_items': self.x_train_questions.shape[1]}) user_input_layer = Input( shape=(model_params['input_dims_users'],), name='user_id') quest_input_layer = Input(shape=( model_params['input_dims_items'],), name='questions/items') if not self.l_traits == None: pass # provision to add latent traits to first layer # latent_trait = Dense(1, use_bias=False, # kernel_initializer= initializers.RandomNormal(mean=0, stddev=1.0, seed=None), # kernel_regularizer=regularizers.l2(0.01), name='latent_trait')(user_input_layer) else: latent_trait = Dense(model_params['ability_params']['units'], use_bias=model_params['ability_params']['use_bias'], bias_initializer= model_params['ability_params']['bias'], kernel_initializer=model_params['ability_params']['kernel'], kernel_regularizer=l1_l2( l1=model_params['ability_params']['regularizers']['l1'], l2=model_params['ability_params']['regularizers']['l2']), activity_regularizer=l1_l2( l1=model_params['ability_params']['group_lasso']['l1'], l2=model_params['ability_params']['group_lasso']['l2']), name='latent_trait/ability')(user_input_layer) difficulty_level = Dense(model_params['diff_params']['units'], use_bias=model_params['diff_params']['use_bias'], bias_initializer= model_params['diff_params']['bias'], kernel_initializer=model_params['diff_params']['kernel'], kernel_regularizer=l1_l2( l1=model_params['diff_params']['regularizers']['l1'], l2=model_params['diff_params']['regularizers']['l2']), activity_regularizer=l1_l2( l1=model_params['diff_params']['group_lasso']['l1'], l2=model_params['diff_params']['group_lasso']['l2']), name='difficulty_level')(quest_input_layer) discrimination_param = Dense(model_params['disc_params']['units'], use_bias=model_params['disc_params']['use_bias'], kernel_initializer=model_params['disc_params']['kernel'], bias_initializer=model_params['disc_params']['bias'], kernel_regularizer=l1_l2( l1=model_params['disc_params']['regularizers']['l1'], l2=model_params['disc_params']['regularizers']['l2']), activity_regularizer=l1_l2( l1=model_params['disc_params']['group_lasso']['l1'], l2=model_params['disc_params']['group_lasso']['l2']), trainable=model_params['disc_params']['train'], #activation= model_params['disc_params']['act'], name='disc_param')(quest_input_layer) discrimination_param = Activation(model_params['disc_params']['act'], name= 'disc_activation')(discrimination_param) disc_latent_interaction = keras.layers.Multiply( name='lambda_latent_inter.')([discrimination_param, latent_trait]) disc_diff_interaction = keras.layers.Multiply( name='alpha_param.')([discrimination_param, difficulty_level]) alpha_lambda_add = keras.layers.Subtract(name='alpha_lambda_add')( [disc_latent_interaction, disc_diff_interaction]) sigmoid_layer = Activation( 'sigmoid', name='Sigmoid_func')(alpha_lambda_add) guess_param = Dense(model_params['guess_params']['units'], use_bias=model_params['guess_params']['use_bias'], kernel_initializer=model_params['guess_params']['kernel'], bias_initializer=model_params['guess_params']['bias'], kernel_regularizer=l1_l2( l1=model_params['guess_params']['regularizers']['l1'], l2=model_params['guess_params']['regularizers']['l2']), activity_regularizer=l1_l2( l1=model_params['guess_params']['group_lasso']['l1'], l2=model_params['guess_params']['group_lasso']['l2']), trainable=model_params['guess_params']['train'], activation=model_params['guess_params']['act'], name='guessing_param')(quest_input_layer) slip_param= Dense(model_params['slip_params']['units'], use_bias=model_params['slip_params']['use_bias'], kernel_initializer=model_params['slip_params']['kernel'], bias_initializer=model_params['slip_params']['bias'], kernel_regularizer=l1_l2( l1=model_params['slip_params']['regularizers']['l1'], l2=model_params['slip_params']['regularizers']['l2']), activity_regularizer=l1_l2( l1=model_params['slip_params']['group_lasso']['l1'], l2=model_params['slip_params']['group_lasso']['l2']), trainable=model_params['slip_params']['train'], activation=model_params['slip_params']['act'], name='slip_param')(quest_input_layer) guess_param_interaction = Lambda(lambda x: 1 - x, name='slip_param_inter.')(slip_param) guess_param_interaction = keras.layers.Subtract(name= 'slip/guess_interaction')([guess_param_interaction, guess_param])#2 #guess_param_interaction = Lambda(lambda x: K.constant(value=np.array( #[1 - model_params['guess_params']['slip']])) - x, name='guess_param_inter.')(guess_param) guess_param_interaction = keras.layers.Multiply( name='disc/guess_param_inter.')([sigmoid_layer, guess_param_interaction]) guess_param_interaction = keras.layers.Add( name='guess_param_inter/add')([guess_param, guess_param_interaction]) prediction_output = Dense(model_params['hyper_params']['units'], trainable=False, use_bias=False, kernel_initializer=keras.initializers.Ones(), name='prediction_layer')(guess_param_interaction) model_ = Model( inputs=[user_input_layer, quest_input_layer], outputs=prediction_output) model_.compile(loss=model_params['hyper_params']['loss'], optimizer=model_params['hyper_params']['optimizer'], metrics=['mae', 'accuracy']) return model_