def add_activation_layer(model, activation, activation_param): """Adds the specified activation layer to the given model. Parameters ---------- model: tf.keras.model Tensorflow.keras model to which to add an activation layer activation: str Activation to add to the model activation_param: float Parameter related to the activation (when needed) """ if activation == "relu": model.add(ReLU()) elif activation == "leaky_relu": model.add(LeakyReLU(alpha=activation_param)) elif activation == "prelu": model.add(PReLU()) elif activation == "elu": model.add(ELU(alpha=activation_param)) elif activation == "selu": model.add(Activation("selu")) elif activation == "thresholded_relu": model.add(ThresholdedReLU(theta=activation_param)) elif activation == "softmax": model.add(Softmax()) elif activation == "softplus": model.add(Activation("softplus")) # elif activation == "rrelu": # model.add(tfa.activations.rrelu()) else: print(f"Selected activation function ({activation}) is not available!")
def avg_dice_2(y_true, y_pred): y_pred = ThresholdedReLU(theta=0.5)(y_pred) loss = 0 num_label = 2 y_true_f = K.flatten(y_true[..., num_label]) y_pred_f = K.flatten(y_pred[..., num_label]) intersection = K.sum(y_true_f * y_pred_f) loss += (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth) return loss # 1>= loss >0
def simple_block_2d(input, number_of_filters, downsample=False, upsample=False, convolution_kernel_size=(3, 3), deconvolution_kernel_size=(2, 2), weight_decay=0.0, dropout_rate=0.0): number_of_output_filters = number_of_filters output = BatchNormalization()(input) output = ThresholdedReLU(theta=0)(output) if downsample: output = MaxPooling2D(pool_size=(2, 2))(output) output = Conv2D( filters=number_of_filters, kernel_size=convolution_kernel_size, padding='same', kernel_regularizer=regularizers.l2(weight_decay))(output) if upsample: output = Conv2DTranspose( filters=number_of_filters, kernel_size=deconvolution_kernel_size, padding='same', kernel_initializer=initializers.he_normal(), kernel_regularizer=regularizers.l2(weight_decay))(output) output = UpSampling2D(size=(2, 2))(output) if dropout_rate > 0.0: output = Dropout(rate=dropout_rate)(output) # Modify the input so that it has the same size as the output if downsample: input = Conv2D(filters=number_of_output_filters, kernel_size=(1, 1), strides=(2, 2), padding='same')(input) elif upsample: input = Conv2DTranspose(filters=number_of_output_filters, kernel_size=(1, 1), padding='same')(input) input = UpSampling2D(size=(2, 2))(input) elif number_of_filters != number_of_output_filters: input = Conv2D(filters=number_of_output_filters, kernel_size=(1, 1), padding='same')(input) output = skip_connection(input, output) return (output)
def average_dice_coef(y_true, y_pred): y_pred = ThresholdedReLU(theta=0.5)(y_pred) loss = 0 label_length = y_pred.get_shape().as_list()[-1] for num_label in range(label_length): y_true_f = K.flatten(y_true[..., num_label]) y_pred_f = K.flatten(y_pred[..., num_label]) intersection = K.sum(y_true_f * y_pred_f) loss += (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth) return loss / label_length # 1>= loss >0
def nyu_model(max_features, max_len, use_gap=True): main_input = Input(shape=(75, ), dtype='int32 ', name='main_input') embedding = Embedding(input_dim=128, output_dim=128, input_length=75)(main_input) conv1 = Conv1D(filters=128, kernel_size=3, padding='same', strides=1)(embedding) thresh1 = ThresholdedReLU(1e-6)(conv1) max_pool1 = MaxPooling1D(pool_size=2, padding='same')(thresh1) conv2 = Conv1D(filters=128, kernel_size=2, padding='same', strides=1)(max_pool1) thresh2 = ThresholdedReLU(1e-6)(conv2) max_pool2 = MaxPooling1D(pool_size=2, padding='same')(thresh2) flatten = Flatten()(max_pool2) fc = Dense(64)(flatten) thresh_fc = ThresholdedReLU(1e-6)(fc) drop = Dropout(0.5)(thresh_fc) output = Dense(1, activation='sigmoid')(drop) model = Model(inputs=main_input, outputs=output) model.compile(loss=' binary_crossentropy ', optimizer='adam', metrics=['accuracy']) return model
def _build_model(self) -> None: """ Build and compile the Character Level CNN model :return: None """ # Input layer inputs = Input(shape=(self.input_sz,), name='sent_input', dtype='int16') # Embedding layers x = Embedding(self.alphabet_sz + 1, self.emb_sz, input_length=self.input_sz)(inputs) x = Reshape((self.input_sz, self.emb_sz))(x) # Convolutional layers for cl in self.conv_layers: x = Conv1D(cl[0], cl[1], kernel_initializer="lecun_normal", padding="causal", use_bias=False)(x) x = BatchNormalization(scale=False)(x) x = Activation('selu')(x) x = AlphaDropout(0.5)(x) if cl[2] != -1: x = MaxPooling1D(cl[2], cl[3])(x) if cl[4] != -1: x = self._squeeze_and_excitation_block(input_data = x, ratio = cl[4]) # Flatten the features x = Flatten()(x) # Fully connected layers for fl in self.fc_layers: x = Dense(fl)(x) x = ThresholdedReLU(self.threshold)(x) x = Dropout(self.dropout_p)(x) # Output layer predictions = Dense(self.num_of_classes, activation="softmax")(x) # Build and coompile the model model = Model(inputs, predictions) # Compile model.compile(optimizer='nadam', loss=self.loss, metrics=['accuracy']) self.model = model self.model.summary()
def block(x): if norm == "batch": x = BatchNormalization(momentum=0.8, scale=False)(x) if activation == 'leakyrelu': x = LeakyReLU(0.2)(x) elif activation == 'relu': x = ReLU()(x) elif activation == 'thresholdedrelu': x = ThresholdedReLU()(x) elif activation == 'elu': x = ELU()(x) if padding == 'reflect': pad = tuple((s - 1) // 2 for s in conv_size) x = TD(ReflectionPadding2D(padding=pad))(x) x = TD( Conv(channels, conv_size, padding='valid' if padding == 'reflect' else padding, strides=(stride, stride), kernel_regularizer=(l2(1e-4) if norm != "spectral" else None)))(x) return x
def load_model(input_shape, num_labels, axis=-1, base_filter=32, depth_size=4, se_res_block=True, se_ratio=16, noise=0.1, last_relu=False, atten_gate=False): def conv3d(layer_input, filters, axis=-1, se_res_block=True, se_ratio=16, down_sizing=True): if down_sizing == True: layer_input = MaxPooling3D(pool_size=(2, 2, 2))(layer_input) d = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(layer_input) d = InstanceNormalization(axis=axis)(d) d = LeakyReLU(alpha=0.3)(d) d = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(d) d = InstanceNormalization(axis=axis)(d) if se_res_block == True: se = GlobalAveragePooling3D()(d) se = Dense(filters // se_ratio, activation='relu')(se) se = Dense(filters, activation='sigmoid')(se) se = Reshape([1, 1, 1, filters])(se) d = Multiply()([d, se]) shortcut = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(layer_input) shortcut = InstanceNormalization(axis=axis)(shortcut) d = add([d, shortcut]) d = LeakyReLU(alpha=0.3)(d) return d def deconv3d(layer_input, skip_input, filters, axis=-1, se_res_block=True, se_ratio=16, atten_gate=False): if atten_gate == True: gating = Conv3D(filters, (1, 1, 1), use_bias=False, padding='same')(layer_input) gating = InstanceNormalization(axis=axis)(gating) attention = Conv3D(filters, (2, 2, 2), strides=(2, 2, 2), use_bias=False, padding='valid')(skip_input) attention = InstanceNormalization(axis=axis)(attention) attention = add([gating, attention]) attention = Conv3D(1, (1, 1, 1), use_bias=False, padding='same', activation='sigmoid')(attention) # attention = Lambda(resize_by_axis, arguments={'dim_1':skip_input.get_shape().as_list()[1],'dim_2':skip_input.get_shape().as_list()[2],'ax':3})(attention) # error when None dimension is feeded. # attention = Lambda(resize_by_axis, arguments={'dim_1':skip_input.get_shape().as_list()[1],'dim_2':skip_input.get_shape().as_list()[3],'ax':2})(attention) attention = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(attention) attention = UpSampling3D((2, 2, 2))(attention) attention = CropToConcat3D(mode='crop')([attention, skip_input]) attention = Lambda(lambda x: K.tile(x, [1, 1, 1, 1, filters]))( attention) skip_input = multiply([skip_input, attention]) u1 = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(layer_input) u1 = Conv3DTranspose(filters, (2, 2, 2), strides=(2, 2, 2), use_bias=False, padding='same')(u1) u1 = InstanceNormalization(axis=axis)(u1) u1 = LeakyReLU(alpha=0.3)(u1) u1 = CropToConcat3D()([u1, skip_input]) u2 = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u1) u2 = InstanceNormalization(axis=axis)(u2) u2 = LeakyReLU(alpha=0.3)(u2) u2 = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u2) u2 = InstanceNormalization(axis=axis)(u2) if se_res_block == True: se = GlobalAveragePooling3D()(u2) se = Dense(filters // se_ratio, activation='relu')(se) se = Dense(filters, activation='sigmoid')(se) se = Reshape([1, 1, 1, filters])(se) u2 = Multiply()([u2, se]) shortcut = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u1) shortcut = InstanceNormalization(axis=axis)(shortcut) u2 = add([u2, shortcut]) u2 = LeakyReLU(alpha=0.3)(u2) return u2 def CropToConcat3D(mode='concat'): def crop_to_concat_3D(concat_layers, axis=-1): bigger_input, smaller_input = concat_layers bigger_shape, smaller_shape = tf.shape(bigger_input), \ tf.shape(smaller_input) sh, sw, sd = smaller_shape[1], smaller_shape[2], smaller_shape[3] bh, bw, bd = bigger_shape[1], bigger_shape[2], bigger_shape[3] dh, dw, dd = bh - sh, bw - sw, bd - sd cropped_to_smaller_input = bigger_input[:, :-dh, :-dw, :-dd, :] if mode == 'concat': return K.concatenate([smaller_input, cropped_to_smaller_input], axis=axis) elif mode == 'add': return smaller_input + cropped_to_smaller_input elif mode == 'crop': return cropped_to_smaller_input return Lambda(crop_to_concat_3D) def resize_by_axis(image, dim_1, dim_2, ax): # it is available only for 1 channel 3D resized_list = [] unstack_img_depth_list = tf.unstack(image, axis=ax) for i in unstack_img_depth_list: resized_list.append(tf.image.resize_images( i, [dim_1, dim_2])) # defaults to ResizeMethod.BILINEAR stack_img = tf.stack(resized_list, axis=ax + 1) return stack_img input_img = Input(shape=input_shape, name='Input') d0 = GaussianNoise(noise)(input_img) d1 = Conv3D(base_filter, (3, 3, 3), use_bias=False, padding='same')(d0) d1 = InstanceNormalization(axis=axis)(d1) d1 = LeakyReLU(alpha=0.3)(d1) d2 = conv3d(d1, base_filter * 2, se_res_block=se_res_block) d3 = conv3d(d2, base_filter * 4, se_res_block=se_res_block) d4 = conv3d(d3, base_filter * 8, se_res_block=se_res_block) if depth_size == 4: d5 = conv3d(d4, base_filter * 16, se_res_block=se_res_block) u4 = deconv3d(d5, d4, base_filter * 8, se_res_block=se_res_block, atten_gate=atten_gate) u3 = deconv3d(u4, d3, base_filter * 4, se_res_block=se_res_block, atten_gate=atten_gate) elif depth_size == 3: u3 = deconv3d(d4, d3, base_filter * 4, se_res_block=se_res_block, atten_gate=atten_gate) else: raise Exception('depth size must be 3 or 4. you put ', depth_size) u2 = deconv3d(u3, d2, base_filter * 2, se_res_block=se_res_block, atten_gate=atten_gate) u1 = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(u2) u1 = Conv3DTranspose(base_filter, (2, 2, 2), strides=(2, 2, 2), use_bias=False, padding='same')(u1) u1 = InstanceNormalization(axis=axis)(u1) u1 = LeakyReLU(alpha=0.3)(u1) u1 = CropToConcat3D()([u1, d1]) # output_img = Conv3D(num_labels, (3, 3, 3), use_bias=True, padding='same', activation='sigmoid', name='lastconv')(u1) u1 = Conv3D(base_filter, (3, 3, 3), use_bias=False, padding='same')(u1) u1 = InstanceNormalization(axis=axis)(u1) u1 = LeakyReLU(alpha=0.3)(u1) output_img = Conv3D(num_labels, kernel_size=1, strides=1, padding='same', activation='sigmoid', name='lastconv')(u1) if last_relu == True: output_img = ThresholdedReLU(theta=0.5)(output_img) model = Model(inputs=input_img, outputs=output_img) return model
(Subtract(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Multiply(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Average(), [Input( (6, 6, 5)), Input( (6, 6, 5)), Input((6, 6, 5))], 32), (Maximum(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Minimum(), [Input((6, 6, 5)), Input((6, 6, 5))], 32), (Activation('relu'), Input((6, 6, 5)), 32), (Reshape((36, 5)), Input((6, 6, 5)), 32), (Reshape((20, 6, 5)), Input((6, 5, 4, 5)), 32), (GatherLayer([1, 4, 2], axis=1), Input((6, 5)), 32), (GatherLayer([1, 4, 2], axis=2), Input((6, 5)), 32), (GatherLayer([1, 4, 2], axis=2), Input((6, 7, 5)), 32), (LeakyReLU(), Input((6, 6, 5)), 32), (ELU(), Input((6, 6, 5)), 32), (ThresholdedReLU(), Input((6, 6, 5)), 32), (Softmax(), Input((6, 6, 5)), 32), (DownShiftLayer(), Input((6, 6, 5)), 32), (DownShiftLayer(), Input((6, 5)), 32), (RightShiftLayer(), Input((6, 6, 5)), 32), (VectorToComplexNumber(), Input((6, 6, 2)), 32), (LambdaWithOneToOneTopology(lambda x: tf.exp(x)), Input((6, 6, 2)), 32), (ToComplex64(), Input((6, 6, 2)), 32), (Conv2D( 8, kernel_size=3, data_format='channels_last', bias_initializer='ones'), Input((6, 6, 5)), 1), (Conv2D( 8, kernel_size=3, data_format='channels_last', bias_initializer='ones'), Input((6, 6, 1)), 32), (Conv2D( 8, kernel_size=3, data_format='channels_last',
def create_resunet_model_3d(input_image_size, number_of_outputs=1, number_of_filters_at_base_layer=32, bottle_neck_block_depth_schedule=(3, 4), convolution_kernel_size=(3, 3, 3), deconvolution_kernel_size=(2, 2, 2), dropout_rate=0.0, weight_decay=0.0, mode='classification'): """ 3-D implementation of the Resnet + U-net deep learning architecture. Creates a keras model of the U-net + ResNet deep learning architecture for image segmentation and regression with the paper available here: https://arxiv.org/abs/1608.04117 This particular implementation was ported from the following python implementation: https://github.com/veugene/fcn_maker/ Arguments --------- input_image_size : tuple of length 4 Used for specifying the input tensor shape. The shape (or dimension) of that tensor is the image dimensions followed by the number of channels (e.g., red, green, and blue). The batch size (i.e., number of training images) is not specified a priori. number_of_outputs : integer Meaning depends on the mode. For 'classification' this is the number of segmentation labels. For 'regression' this is the number of outputs. number_of_filters_at_base_layer : integer Number of filters at the beginning and end of the 'U'. Doubles at each descending/ascending layer. bottle_neck_block_depth_schedule : tuple Tuple that provides the encoding layer schedule for the number of bottleneck blocks per long skip connection. convolution_kernel_size : tuple of length 3 3-d vector defining the kernel size during the encoding path deconvolution_kernel_size : tuple of length 3 3-d vector defining the kernel size during the decoding dropout_rate : scalar Float between 0 and 1 to use between dense layers. weight_decay : scalar Weighting parameter for L2 regularization of the kernel weights of the convolution layers. Default = 0.0. mode : string 'classification' or 'regression'. Default = 'classification'. Returns ------- Keras model A 3-D Keras model defining the network. Example ------- >>> model = create_resunet_model_3d((128, 128, 128, 1)) >>> model.summary() """ def simple_block_3d(input, number_of_filters, downsample=False, upsample=False, convolution_kernel_size=(3, 3, 3), deconvolution_kernel_size=(2, 2, 2), weight_decay=0.0, dropout_rate=0.0): number_of_output_filters = number_of_filters output = BatchNormalization()(input) output = ThresholdedReLU(theta=0)(output) if downsample: output = MaxPooling3D(pool_size=(2, 2, 2))(output) output = Conv3D( filters=number_of_filters, kernel_size=convolution_kernel_size, padding='same', kernel_regularizer=regularizers.l2(weight_decay))(output) if upsample: output = Conv3DTranspose( filters=number_of_filters, kernel_size=deconvolution_kernel_size, padding='same', kernel_initializer=initializers.he_normal(), kernel_regularizer=regularizers.l2(weight_decay))(output) output = UpSampling3D(size=(2, 2, 2))(output) if dropout_rate > 0.0: output = Dropout(rate=dropout_rate)(output) # Modify the input so that it has the same size as the output if downsample: input = Conv3D(filters=number_of_output_filters, kernel_size=(1, 1, 1), strides=(2, 2, 2), padding='same')(input) elif upsample: input = Conv3DTranspose(filters=number_of_output_filters, kernel_size=(1, 1, 1), padding='same')(input) input = UpSampling3D(size=(2, 2, 2))(input) elif number_of_filters != number_of_output_filters: input = Conv3D(filters=number_of_output_filters, kernel_size=(1, 1, 1), padding='same')(input) output = skip_connection(input, output) return (output) def bottle_neck_block_3d(input, number_of_filters, downsample=False, upsample=False, deconvolution_kernel_size=(2, 2, 2), weight_decay=0.0, dropout_rate=0.0): output = input number_of_output_filters = number_of_filters if downsample: output = BatchNormalization()(output) output = ThresholdedReLU(theta=0)(output) output = Conv3D( filters=number_of_filters, kernel_size=(1, 1, 1), strides=(2, 2, 2), kernel_initializer=initializers.he_normal(), kernel_regularizer=regularizers.l2(weight_decay))(output) output = BatchNormalization()(output) output = ThresholdedReLU(theta=0)(output) output = Conv3D( filters=number_of_filters, kernel_size=(1, 1, 1), kernel_initializer=initializers.he_normal(), kernel_regularizer=regularizers.l2(weight_decay))(output) output = BatchNormalization()(output) output = ThresholdedReLU(theta=0)(output) if upsample: output = Conv3DTranspose( filters=number_of_filters, kernel_size=deconvolution_kernel_size, padding='same', kernel_initializer=initializers.he_normal(), kernel_regularizer=regularizers.l2(weight_decay))(output) output = UpSampling3D(size=(2, 2, 2))(output) output = Conv3D( filters=(number_of_filters * 4), kernel_size=(1, 1, 1), kernel_initializer=initializers.he_normal(), kernel_regularizer=regularizers.l2(weight_decay))(output) number_of_output_filters = number_of_filters * 4 if dropout_rate > 0.0: output = Dropout(rate=dropout_rate)(output) # Modify the input so that it has the same size as the output if downsample: input = Conv3D(filters=number_of_output_filters, kernel_size=(1, 1, 1), strides=(2, 2, 2), padding='same')(input) elif upsample: input = Conv3DTranspose(filters=number_of_output_filters, kernel_size=(1, 1, 1), padding='same')(input) input = UpSampling3D(size=(2, 2, 2))(input) elif number_of_filters != number_of_output_filters: input = Conv3D(filters=number_of_output_filters, kernel_size=(1, 1, 1), padding='valid')(input) output = skip_connection(input, output) return (output) def skip_connection(source, target, merge_mode='sum'): layer_list = [source, target] output = None if merge_mode == 'sum': output = Add()(layer_list) else: channel_axis = 0 if K.image_data_format() == 'channels_last': channel_axis = -1 output = Concatenate(axis=channel_axis)(layer_list) return (output) inputs = Input(shape=input_image_size) encoding_layers_with_long_skip_connections = [] encoding_layer_count = 1 # Preprocessing layer model = Conv3D(filters=number_of_filters_at_base_layer, kernel_size=convolution_kernel_size, activation='relu', padding='same', kernel_initializer=initializers.he_normal(), kernel_regularizer=regularizers.l2(weight_decay))(inputs) encoding_layers_with_long_skip_connections.append(model) encoding_layer_count += 1 # Encoding initialization path model = simple_block_3d( model, number_of_filters_at_base_layer, downsample=True, convolution_kernel_size=convolution_kernel_size, deconvolution_kernel_size=deconvolution_kernel_size, weight_decay=weight_decay, dropout_rate=dropout_rate) encoding_layers_with_long_skip_connections.append(model) encoding_layer_count += 1 # Encoding main path number_of_bottle_neck_layers = len(bottle_neck_block_depth_schedule) for i in range(number_of_bottle_neck_layers): number_of_filters = number_of_filters_at_base_layer * 2**i for j in range(bottle_neck_block_depth_schedule[i]): do_downsample = False if j == 0: do_downsample = True else: do_downsample = False model = bottle_neck_block_3d( model, number_of_filters=number_of_filters, downsample=do_downsample, deconvolution_kernel_size=deconvolution_kernel_size, weight_decay=weight_decay, dropout_rate=dropout_rate) if j == (bottle_neck_block_depth_schedule[i] - 1): encoding_layers_with_long_skip_connections.append(model) encoding_layer_count += 1 encoding_layer_count -= 1 # Transition path number_of_filters = number_of_filters_at_base_layer * 2**number_of_bottle_neck_layers model = bottle_neck_block_3d( model, number_of_filters=number_of_filters, downsample=True, deconvolution_kernel_size=deconvolution_kernel_size, weight_decay=weight_decay, dropout_rate=dropout_rate) model = bottle_neck_block_3d( model, number_of_filters=number_of_filters, upsample=True, deconvolution_kernel_size=deconvolution_kernel_size, weight_decay=weight_decay, dropout_rate=dropout_rate) # Decoding main path number_of_bottle_neck_layers = len(bottle_neck_block_depth_schedule) for i in range(number_of_bottle_neck_layers): number_of_filters = (number_of_filters_at_base_layer * 2**(number_of_bottle_neck_layers - i - 1)) for j in range( bottle_neck_block_depth_schedule[number_of_bottle_neck_layers - i - 1]): do_upsample = False if j == bottle_neck_block_depth_schedule[ number_of_bottle_neck_layers - i - 1] - 1: do_upsample = True else: do_upsample = False model = bottle_neck_block_3d( model, number_of_filters=number_of_filters, upsample=do_upsample, deconvolution_kernel_size=deconvolution_kernel_size, weight_decay=weight_decay, dropout_rate=dropout_rate) if j == 0: model = Conv3D(filters=(number_of_filters * 4), kernel_size=(1, 1, 1), padding='same')(model) model = skip_connection( encoding_layers_with_long_skip_connections[ encoding_layer_count - 1], model) encoding_layer_count -= 1 # Decoding initialization path model = simple_block_3d( model, number_of_filters_at_base_layer, upsample=True, convolution_kernel_size=convolution_kernel_size, deconvolution_kernel_size=deconvolution_kernel_size, weight_decay=weight_decay, dropout_rate=dropout_rate) # Postprocessing layer model = Conv3D(filters=number_of_filters_at_base_layer, kernel_size=convolution_kernel_size, activation='relu', padding='same', kernel_initializer=initializers.he_normal(), kernel_regularizer=regularizers.l2(weight_decay))(model) encoding_layer_count -= 1 model = skip_connection( encoding_layers_with_long_skip_connections[encoding_layer_count - 1], model) model = BatchNormalization()(model) model = ThresholdedReLU(theta=0)(model) convActivation = '' if mode == 'classification': convActivation = 'softmax' elif mode == 'regression': convActivation = 'linear' else: raise ValueError( 'mode must be either `classification` or `regression`.') outputs = Conv3D(filters=number_of_outputs, kernel_size=(1, 1, 1), activation=convActivation, kernel_regularizer=regularizers.l2(weight_decay))(model) resunet_model = Model(inputs=inputs, outputs=outputs) return resunet_model
def build_character_cnn(model_hyperparameters=None, verbose=None): """ Create a language model based on the Crepe model in Zhang et. al. Creates a model that uses only Convolutional Layers (max pooling in between), and dense layers for prediction tasks. For this model, the final output layer will predict the next character in the sequence. Parameters ---------- model_hyperparameters : dict A dictionary containing values necessary to build the model. verbose : bool A flag to print additional information. Returns ------- model : Sequential A Keras Sequential that represents the model. Notes ----- See https://gab41.lab41.org/deep-learning-sentiment-one-character-at-a-t-i-m-e-6cd96e4f780d for more information. See https://github.com/mhjabreel/CharCnn_Keras for the Keras code this is based on See https://github.com/zhangxiangxiao/Crepe for the original (Torch) Crepe code """ if model_hyperparameters is None: model_hyperparameters = _dutils.load_dictionary( 'model_hyperparameters.json') ''' Load hyperparameter-specific values from JSON file. ''' #The size of the characater vocabulary vocabulary_size = model_hyperparameters.get("vocabulary_size") #The max length of the text. Set as 1014 in the original. text_length = model_hyperparameters.get("text_length") #Number of filters for each convolutional layer num_filters = model_hyperparameters.get("num_filters") #The threshold for the ReLU activation layers threshold = model_hyperparameters.get("relu_threshold") #Dropout probability for Dropout layers dropout_p = model_hyperparameters.get("dropout_percent") #Embedding output dimension. Implementation sets it equal to vocabulary_size embed_dim = model_hyperparameters.get("embedding_dimension") ''' Values below specify the architecture. These aren't stored in the JSON file due to architectutre constraints with layers and kernel sizes. ''' #The number of units for each dense layer minus output layer fully_connected_layers = [128, 64] ''' conv_layers is a list of pairs. First component refers to kernel size. Second component refers to the size of the MaxPooling1D layer (-1 indicates said layer is not present). ''' conv_layers = [[7, 3], [3, -1], [3, -1], [3, -1], [3, 3]] #Input layer inputs = Input(shape=(text_length, ), name='sent_input', dtype='int32') #Embedding layers x = Embedding(vocabulary_size + 1, embed_dim, input_length=text_length, mask_zero=True)(inputs) #Convolution layers ''' First Conv1D layer + MaxPooling is separate in case changes are made upstream. Also it was used to test out TimeDistributed functionality. ''' x = (Convolution1D(num_filters, 7))(x) x = (MaxPooling1D(3))(x) for cl in conv_layers: x = (Convolution1D(num_filters, cl[0]))(x) x = ThresholdedReLU(threshold)(x) if cl[1] != -1: x = (MaxPooling1D(cl[1]))(x) x = Flatten()(x) # #Fully connected layers for fl in fully_connected_layers: ''' Original architecture did not use L2 regularization. However, empirical results show that, for my dataset it works well in handling overfitting. ''' x = Dense(fl, kernel_regularizer=regularizers.l2(0.0001))(x) x = ThresholdedReLU(threshold)(x) ''' Original architecture had dropout at 50%. This seemed to be too high for my dataset, and it resulted in underfitting. ''' x = Dropout(dropout_p)(x) # #Output layer predictions = Dense(vocabulary_size, activation='softmax')(x) # Build and compile model model = Model(inputs=inputs, outputs=predictions) if verbose: model.summary() return model
def test_delete_channels_advanced_activations(channel_index, data_format): layer_test_helper_flatten_2d(LeakyReLU(), channel_index, data_format) layer_test_helper_flatten_2d(ELU(), channel_index, data_format) layer_test_helper_flatten_2d(ThresholdedReLU(), channel_index, data_format)
#input does not has fixed shape. Changed to fixed shape only for layer illustration purpose. #inp=Input(shape=(256,256,1,)) inpp = Input(shape=( None, None, 1, )) #negInp=Input(shape=(256,256,1,)) #preprocessing layers - #maintaining scan points by creating mask. Masking output prediction for scanned point and re-insert back the values #clip the max value to 1 #Using threshold Relu for low lying points. mask = Lambda(lambda x: 1 - x)(inpp) mask1 = ThresholdedReLU(theta=11 / 256)(mask) mask1 = Lambda(lambda x: x * 25)(mask1) mask1 = Lambda(lambda x: backend.clip(x, 0, 1))(mask1) mask = Lambda(lambda x: 1 - x)(mask1) inp0 = Multiply()([mask, inpp]) inp1 = Multiply()([mask1, inpp]) inp = inpp #Stage 1-5 : Area identifier #Stage 1 # Stage 1 MAx Pooling2D layer rul = MaxPooling2D(pool_size=(64, 64), strides=(64, 64), name="max_layer1")(inp) xa0 = rul #preprocessing layers - #maintaining scan points by creating mask. Masking output prediction for scanned point and re-insert back the values
def call(self, inputs): input_sign = K.sign(inputs) hard_thresh_unsigned = ThresholdedReLU(self.threshold)(input_sign * inputs) hard_thresh = hard_thresh_unsigned * input_sign return hard_thresh