def two_head_ensemble_model(dense1=None, dense2=None, dropout=0.25, k_reg=0.00000001): """Not used due to no performance gains""" k_regularizer = keras.regularizers.l2(k_reg) input_tensor = keras.layers.Input(shape=(TWO_HEAD_SHAPE, )) x = layers.Dense(dense1, activation=None, kernel_initializer='he_uniform', kernel_regularizer=k_regularizer)(input_tensor) x = layers.PReLU()(x) out = layers.Dropout(dropout)(x) half_model = keras.models.Model(inputs=input_tensor, outputs=out) inp_a = keras.layers.Input(shape=(TWO_HEAD_SHAPE, )) inp_b = keras.layers.Input(shape=(TWO_HEAD_SHAPE, )) out_a = half_model(inp_a) out_b = half_model(inp_b) concatenated = keras.layers.concatenate([out_a, out_b]) x = layers.Dense(dense2, activation=None, kernel_initializer='he_uniform', kernel_regularizer=k_regularizer)(concatenated) x = layers.PReLU()(x) predictions = layers.Dense(N_CLASSES, activation='softmax', kernel_regularizer=k_regularizer)(x) model = keras.models.Model(inputs=[inp_a, inp_b], outputs=predictions) return model
def enet_upsample(tensor, nfilters, scale = 8, name=''): y = tensor skip = tensor skip = layers.Conv2D(filters=nfilters, kernel_size=(1, 1), kernel_initializer='he_normal', strides=(1, 1), padding='same', use_bias=False, name=f'1x1_conv_skip_{name}')(skip) skip = layers.UpSampling2D(size=(scale, scale), interpolation='bilinear', name=f'upsample_skip_{name}')(skip) # 1*1 dimensionality reduction y = layers.Conv2D(filters=nfilters // 4, kernel_size=(1, 1), kernel_initializer='he_normal', strides=(1, 1), padding='same', use_bias=False, name=f'1x1_conv_{name}')(y) y = layers.BatchNormalization(momentum=0.1, name=f'bn_1x1_{name}')(y) y = layers.PReLU(shared_axes=[1, 2], name=f'prelu_1x1_{name}')(y) # main conv y = layers.Conv2DTranspose(filters=nfilters // 4, kernel_size=(3, 3), kernel_initializer='he_normal', strides=(scale, scale), padding='same', name=f'3x3_deconv_{name}')(y) y = layers.BatchNormalization(momentum=0.1, name=f'bn_main_{name}')(y) y = layers.PReLU(shared_axes=[1, 2], name=f'prelu_{name}')(y) y = layers.Conv2D(filters=nfilters, kernel_size=(1, 1), kernel_initializer='he_normal', use_bias=False, name=f'final_1x1_{name}')(y) y = layers.BatchNormalization(momentum=0.1, name=f'bn_final_{name}')(y) print(f'add_{name}') y = layers.Add(name=f'add_{name}')([y, skip]) y = layers.ReLU(name=f'relu_out_{name}')(y) return y
def p_net(training=False): x = Input(shape=(12, 12, 3)) if training else Input(shape=(None, None, 3)) y = layers.Conv2D(10, 3, padding='valid', strides=(1, 1), name='p_conv1')(x) y = layers.PReLU(shared_axes=(1, 2), name='p_prelu1')(y) y = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='p_max_pooling1')(y) y = layers.Conv2D(16, 3, padding='valid', strides=(1, 1), name='p_conv2')(y) y = layers.PReLU(shared_axes=(1, 2), name='p_prelu2')(y) y = layers.Conv2D(32, 3, padding='valid', strides=(1, 1), name='p_conv3')(y) y = layers.PReLU(shared_axes=(1, 2), name='p_prelu3')(y) classifier = layers.Conv2D(2, 1, activation='softmax', name='p_classifier')(y) bbox = layers.Conv2D(4, 1, name='p_bbox')(y) landmark = layers.Conv2D(10, 1, padding='valid', name='p_landmark')(y) if training: classifier = layers.Reshape((2, ), name='p_classifier1')(classifier) bbox = layers.Reshape((4, ), name='p_bbox1')(bbox) landmark = layers.Reshape((10, ), name='p_landmark1')(landmark) outputs = layers.concatenate([classifier, bbox, landmark]) model = Model(inputs=[x], outputs=[outputs], name='P_Net') else: model = Model(inputs=[x], outputs=[classifier, bbox, landmark], name='P_Net') return model
def _create_Kao_Onet(self, weight_path='./models/mtcnn/48net.h5'): ''' ''' input = KL.Input(shape=[48, 48, 3]) x = KL.Conv2D(32, (3, 3), strides=1, padding='valid', name='conv1', data_format="channels_last")(input) x = KL.PReLU(shared_axes=[1, 2], name='prelu1')(x) x = KL.MaxPool2D(pool_size=3, strides=2, padding='same', data_format="channels_last")(x) x = KL.Conv2D(64, (3, 3), strides=1, padding='valid', name='conv2', data_format="channels_last")(x) x = KL.PReLU(shared_axes=[1, 2], name='prelu2')(x) x = KL.MaxPool2D(pool_size=3, strides=2, data_format="channels_last")(x) x = KL.Conv2D(64, (3, 3), strides=1, padding='valid', name='conv3', data_format="channels_last")(x) x = PReLU(shared_axes=[1, 2], name='prelu3')(x) x = KL.MaxPool2D(pool_size=2, data_format="channels_last")(x) x = KL.Conv2D(128, (2, 2), strides=1, padding='valid', name='conv4', data_format="channels_last")(x) x = KL.PReLU(shared_axes=[1, 2], name='prelu4')(x) x = KL.Permute((3, 2, 1))(x) x = KL.Flatten()(x) x = KL.Dense(256, name='conv5') (x) x = KL.PReLU(name='prelu5')(x) classifier = KL.Dense(2, activation='softmax', name='conv6-1')(x) bbox_regress = KL.Dense(4, name='conv6-2')(x) landmark_regress = KL.Dense(10, name='conv6-3')(x) model = Model([input], [classifier, bbox_regress, landmark_regress]) model.load_weights(weight_path, by_name=True) return model
def create_generator(): img = L.Input(shape=(32, 32, 3)) x = L.Conv2D(filters=64, kernel_size=(9, 9), strides=(1, 1), padding='same')(img) x = L.PReLU(shared_axes=[1, 2])(x) res = x for i in range(16): x = ResBlock(x, 64) x = L.Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x) x = L.BatchNormalization()(x) x = L.Add()([res, x]) x = L.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='same')(x) x = L.UpSampling2D()(x) x = L.PReLU(shared_axes=[1, 2])(x) x = L.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='same')(x) x = L.PReLU(shared_axes=[1, 2])(x) x = L.Conv2D(filters=3, kernel_size=(9, 9), strides=(1, 1), padding='same', activation='relu')(x) gen = Model(inputs=img, outputs=x) gen.compile(loss=mean_squared_error, optimizer=Adam(), metrics=['accuracy']) return gen
def r_net(training=False): x = Input(shape=(24, 24, 3)) y = layers.Conv2D(28, 3, padding='same', strides=(1, 1), name='r_conv1')(x) y = layers.PReLU(shared_axes=(1, 2), name='r_prelu1')(y) y = layers.MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='p_max_pooling1')(y) y = layers.Conv2D(48, 3, padding='valid', strides=(1, 1), name='r_conv2')(y) y = layers.PReLU(shared_axes=(1, 2), name='r_prelu2')(y) y = layers.MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='p_max_pooling2')(y) y = layers.Conv2D(64, 2, padding='valid', name='r_conv3')(y) y = layers.PReLU(shared_axes=(1, 2), name='r_prelu3')(y) y = layers.Dense(128, name='r_dense')(y) y = layers.PReLU(name='r_prelu4')(y) y = layers.Flatten()(y) classifier = layers.Dense(2, activation='softmax', name='r_classifier')(y) bbox = layers.Dense(4, name='r_bbox')(y) landmark = layers.Dense(10, name='r_landmark')(y) if training: outputs = layers.concatenate([classifier, bbox, landmark]) model = Model(inputs=[x], outputs=[outputs], name='R_Net') else: model = Model(inputs=[x], outputs=[classifier, bbox, landmark], name='R_Net') return model
def bottleneck_identity_block(inputs, filters=(64, 64, 256), activation='relu', kernel_size=(3, 3)): """ Follows the bottleneck architecture implementation of an identity block. :param inputs: :param filters: :param activation: :param kernel_size: :return: """ # Set residual and shortcut as inputs for clarity shortcut = inputs residual = inputs f1, f2, f3 = filters # Filters take specific numbers only # First convolution layer -> BN -> activation residual = Conv2D(f1, kernel_size=(1, 1), strides=(1, 1), padding='valid')(residual) residual = BatchNormalization()(residual) if activation == 'relu': residual = layers.ReLU()(residual) elif activation == 'prelu': residual = layers.PReLU()(residual) else: raise NotImplementedError # Second convolution layer -> BN -> activation # Padding is 'same' to ensure same dims residual = Conv2D(f2, kernel_size=kernel_size, strides=(1, 1), padding='same')(residual) residual = BatchNormalization()(residual) if activation == 'relu': output = layers.ReLU()(residual) elif activation == 'prelu': output = layers.PReLU()(residual) else: raise NotImplementedError residual = Conv2D(f3, kernel_size=(1, 1), strides=(1, 1), padding='valid')(residual) residual = BatchNormalization()(residual) output = layers.Add()([shortcut, residual]) if activation == 'relu': output = layers.ReLU()(output) elif activation == 'prelu': output = layers.PReLU()(output) else: raise NotImplementedError return output
def bottleneck_projection_block(inputs, filters, activation='relu', kernel_size=(3, 3), strides=(2, 2)): # Set residual and shortcut as inputs for clarity shortcut = inputs residual = inputs f1, f2, f3 = filters # Filters take specific numbers only # First convolution layer -> BN -> activation residual = Conv2D(f1, kernel_size=(1, 1), strides=strides, padding='valid')(residual) residual = BatchNormalization()(residual) if activation == 'relu': residual = layers.ReLU()(residual) elif activation == 'prelu': residual = layers.PReLU()(residual) else: raise NotImplementedError # Second convolution layer -> BN -> activation # Padding is 'same' to ensure same dims residual = Conv2D(f2, kernel_size=kernel_size, strides=(1, 1), padding='same')(residual) residual = BatchNormalization()(residual) if activation == 'relu': output = layers.ReLU()(residual) elif activation == 'prelu': output = layers.PReLU()(residual) else: raise NotImplementedError residual = Conv2D(f3, kernel_size=(1, 1), strides=(1, 1), padding='valid')(residual) residual = BatchNormalization()(residual) # Projection of shortcut through convolutional block with same strides as first conv layer shortcut = Conv2D(f3, kernel_size=(1, 1), strides=strides, padding='valid')(shortcut) shortcut = BatchNormalization()(shortcut) # Add shortcut to residual, pass through activation output = layers.Add()([shortcut, residual]) if activation == 'relu': output = layers.ReLU()(output) elif activation == 'prelu': output = layers.PReLU()(output) else: raise NotImplementedError return output
def identity_block(inputs, activation='relu', kernel_size=3, strides=(1, 1)): """ Regular residual block, not bottlenecked. :param inputs: :param activation: :param kernel_size: :param strides: :return: """ # Set residual and shortcut as inputs for clarity shortcut = inputs residual = inputs f = K.int_shape(inputs)[ 3] # Filters must be fixed to input shape for dimension adding later # First convolution layer -> BN -> activation # Padding is 'same' to ensure shortcut/residual have same dims residual = Conv2D(f, kernel_size=kernel_size, strides=strides, padding='same')(residual) residual = BatchNormalization()(residual) if activation == 'relu': residual = layers.ReLU()(residual) elif activation == 'prelu': residual = layers.PReLU()(residual) else: raise NotImplementedError # Second convolution layer -> BN. Activation follows after adding shortcut residual = Conv2D(f, kernel_size=kernel_size, strides=strides, padding='same')(residual) residual = BatchNormalization()(residual) add = layers.Add()([ shortcut, residual ]) # Add shortcut and residual, then send through activation out if activation == 'relu': output = layers.ReLU()(add) elif activation == 'prelu': output = layers.PReLU()(add) else: raise NotImplementedError return output
def convBN_pool(input_layer, conv_channels): convlayer = Conv1D(conv_channels, 1, padding = 'valid', strides = 2)(input_layer) BN = BatchNormalization(axis=-1, momentum = 0.99, epsilon=0.001, center=True, scale = True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros')(convlayer) activation = layers.PReLU()(BN) return activation
def convBN(input_layer, conv_channels): convlayer = Conv1D(conv_channels, 3, padding = 'same')(input_layer) BN = BatchNormalization(axis=-1, momentum = 0.99, epsilon=0.001, center=True, scale = True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None)(convlayer) activation = layers.PReLU()(BN) return activation
def preactivated_identity_residual_block(inputs, activation='relu', kernel_size=3, strides=(1, 1)): # Set residual and shortcut as inputs for clarity shortcut = inputs residual = inputs f = K.int_shape(inputs)[ 3] # Filters must be fixed to input shape for dimension adding later # Pre-activation using BN and P/ReLU residual = layers.BatchNormalization()(inputs) if activation == 'relu': residual = layers.ReLU()(residual) elif activation == 'prelu': residual = layers.PReLU()(residual) else: raise NotImplementedError # Padding is 'same' to ensure shortcut/residual have same dims residual = Conv2D(f, kernel_size=kernel_size, strides=strides, padding='same')(residual) residual = BatchNormalization()(residual) if activation == 'relu': residual = layers.ReLU()(residual) elif activation == 'prelu': residual = layers.PReLU()(residual) else: raise NotImplementedError # Second convolution layer -> addition. Only pre-activation, no post. residual = Conv2D(f, kernel_size=kernel_size, strides=strides, padding='same')(residual) output = layers.Add()([shortcut, residual]) # Add shortcut and residual return output
def _create_Kao_Pnet(self, weight_path='./models/mtcnn/12net.h5'): ''' ''' input = KL.Input(shape=[None, None, 3]) x = KL.Conv2D(10, (3, 3), strides=1, padding='valid', name='conv1', data_format="channels_last")(input) x = KL.PReLU(shared_axes=[1, 2], name='PReLU1')(x) x = KL.MaxPool2D(pool_size=2, data_format="channels_last")(x) x = KL.Conv2D(16, (3, 3), strides=1, padding='valid', name='conv2', data_format="channels_last")(x) x = KL.PReLU(shared_axes=[1, 2], name='PReLU2')(x) x = KL.Conv2D(32, (3, 3), strides=1, padding='valid', name='conv3', data_format="channels_last")(x) x = KL.PReLU(shared_axes=[1, 2], name='PReLU3')(x) classifier = KL.Conv2D(2, (1, 1), activation='softmax', name='conv4-1', data_format="channels_last")(x) bbox_regress = KL.Conv2D(4, (1, 1), name='conv4-2', data_format="channels_last")(x) model = Model([input], [classifier, bbox_regress]) model.load_weights(weight_path, by_name=True) return model
def activate(self, layer): """ activate layer with given activation function :param layer: the input layer :return: the layer after activation """ if self.activ == 'lrelu': return layers.LeakyReLU()(layer) elif self.activ == 'prelu': return layers.PReLU()(layer) else: return Activation(self.activ)(layer)
def ResBlock(x, filters): res = x x = L.Conv2D(filters=filters, kernel_size=( 3, 3), strides=(1, 1), padding='same')(x) x = L.BatchNormalization()(x) x = L.PReLU(shared_axes=[1, 2])(x) x = L.Conv2D(filters=filters, kernel_size=( 3, 3), strides=(1, 1), padding='same')(x) x = L.BatchNormalization()(x) x = L.Add()([res, x]) return x
def combined_features_model(dense1=1024, dense2=None, dropout=0.25, k_reg=0.0001): k_regularizer = keras.regularizers.l2(k_reg) input_tensor = keras.layers.Input(shape=MODEL_INPUT_SHAPE) x = layers.Dense(dense1, activation=None, kernel_initializer='he_uniform', kernel_regularizer=k_regularizer)(input_tensor) x = layers.PReLU()(x) x = layers.Dropout(dropout)(x) if dense2: x = layers.Dense(dense2, activation=None, kernel_initializer='he_uniform', kernel_regularizer=k_regularizer)(x) x = layers.PReLU()(x) x = layers.Dropout(dropout)(x) predictions = layers.Dense(N_CLASSES, activation='softmax')(x) model = keras.models.Model(inputs=input_tensor, outputs=predictions) return model
def inception_resnet_v2(num_classes, input_size=150): base_model = InceptionResNetV2(include_top=False, weights='imagenet', input_shape=(input_size, input_size, 3), pooling='max') x = base_model.output x = layers.BatchNormalization()(x) x = layers.Dropout(0.5)(x) x = layers.Dense(1024)(x) x = layers.BatchNormalization()(x) x = layers.PReLU(shared_axes=[1])(x) x = layers.Dropout(0.5)(x) x = layers.Dense(512)(x) x = layers.BatchNormalization()(x) features = layers.PReLU(shared_axes=[1])(x) prediction = layers.Dense(num_classes, activation='softmax', name='fc_prediction')(features) model = Model(inputs=base_model.input, outputs=prediction) model.summary() return model, base_model
def ensemble_model(dense1=None, dense2=None, n_layers=4, dropout=0.25, k_reg=0): """Creates NN ensemble model""" k_regularizer = keras.regularizers.l2(k_reg) input_tensor = keras.layers.Input(shape=MODEL_INPUT_SHAPE) if dense1: x = layers.Dense(dense1, activation=None, kernel_initializer='he_uniform', kernel_regularizer=k_regularizer)(input_tensor) x = layers.PReLU()(x) x = layers.Dropout(dropout)(x) x = layers.BatchNormalization()(x) if dense2: for n in range(n_layers - 1): x = layers.Dense(dense2, activation=None, kernel_initializer='he_uniform', kernel_regularizer=k_regularizer)(x) x = layers.PReLU()(x) if not n == n_layers - 2: # Don't want dropout and BN on last layer x = layers.Dropout(dropout)(x) x = layers.BatchNormalization()(x) if dense1: predictions = layers.Dense(N_CLASSES, activation='softmax', kernel_regularizer=k_regularizer)(x) else: predictions = layers.Dense( N_CLASSES, activation='softmax', kernel_regularizer=k_regularizer)(input_tensor) model = keras.models.Model(inputs=input_tensor, outputs=predictions) return model
def _last_layer(output, last_layer): if last_layer == 'softmax': output = layers.Softmax(axis=-1)(output) elif last_layer == 'sigmoid': output = layers.Activation(activation='sigmoid')(output) elif last_layer == 'relu': output = layers.Activation(activation='relu')(output) elif last_layer == 'leaky_relu': output = layers.LeakyReLU()(output) elif last_layer == 'prelu': output = layers.PReLU()(output) return output
def resblock(input, filters, stride=1): input_channels = input.shape[-1] residual = layers.Conv2D(filters / 4, kernel_size=3, strides=1, padding='same')(input) residual = layers.BatchNormalization()(residual) residual = layers.PReLU()(residual) residual = layers.Conv2D(filters / 2, kernel_size=3, strides=stride, padding='same')(residual) residual = layers.BatchNormalization()(residual) residual = layers.PReLU()(residual) residual = layers.Conv2D(filters, kernel_size=3, strides=1, padding='same')(residual) residual = layers.BatchNormalization()(residual) if stride > 1 or input_channels != filters: # Perform identity shortcut convolution for dims not matching shortcut = layers.Conv2D(filters, strides=stride, kernel_size=1)(input) shortcut = layers.BatchNormalization()(shortcut) else: shortcut = input output = layers.Add()([shortcut, residual]) output = layers.PReLU()(output) return output
def _create_Kao_Rnet(self, weight_path='./models/mtcnn/24net.h5'): ''' ''' input = KL.Input(shape=[24, 24, 3]) # change this shape to [None,None,3] to enable arbitraty shape input x = KL.Conv2D(28, (3, 3), strides=1, padding='valid', name='conv1', data_format="channels_last")(input) x = KL.PReLU(shared_axes=[1, 2], name='prelu1')(x) x = KL.MaxPool2D(pool_size=3, strides=2, padding='same', data_format="channels_last")(x) x = KL.Conv2D(48, (3, 3), strides=1, padding='valid', name='conv2', data_format="channels_last")(x) x = KL.PReLU(shared_axes=[1, 2], name='prelu2')(x) x = KL.MaxPool2D(pool_size=3, strides=2, data_format="channels_last")(x) x = KL.Conv2D(64, (2, 2), strides=1, padding='valid', name='conv3', data_format="channels_last")(x) x = KL.PReLU(shared_axes=[1, 2], name='prelu3')(x) x = KL.Permute((3, 2, 1))(x) x = KL.Flatten()(x) x = KL.Dense(128, name='conv4')(x) x = KL.PReLU(name='prelu4')(x) classifier = KL.Dense(2, activation='softmax', name='conv5-1')(x) bbox_regress = KL.Dense(4, name='conv5-2')(x) model = Model([input], [classifier, bbox_regress]) model.load_weights(weight_path, by_name=True) return model
def conv_bn_activation_block(inputs, filters, activation='relu', kernel_size=5, padding='valid', strides=(1, 1)): conv_layer = Conv2D(filters, kernel_size=kernel_size, strides=strides, padding=padding)(inputs) bn_layer = BatchNormalization()(conv_layer) if activation == 'relu': output = layers.ReLU()(bn_layer) elif activation == 'prelu': output = layers.PReLU()(bn_layer) else: return print("Unspecified activation.") return output
def create_smNet(input_dims=(32, 32, 1), output_dims=1, min_z=-500, max_z=500): input = layers.Input(shape=input_dims) x = layers.Conv2D(filters=64, kernel_size=7)(input) x = layers.BatchNormalization()(x) x = layers.PReLU()(x) x = layers.Conv2D(filters=128, kernel_size=5)(x) x = layers.BatchNormalization()(x) x = layers.PReLU()(x) x = resblock(x, 128, 1) x = resblock(x, 128, 1) x = resblock(x, 128, 1) x = resblock(x, 256, 1) x = resblock(x, 256, 1) x = resblock(x, 256, 1) x = resblock(x, 256, 1) x = layers.Conv2D(128, kernel_size=1, strides=1)(x) x = layers.BatchNormalization()(x) x = layers.PReLU()(x) x = layers.Conv2D(64, kernel_size=1, strides=1)(x) x = layers.BatchNormalization()(x) x = layers.PReLU()(x) x = layers.Conv2D(1, kernel_size=1, strides=1)(x) x = layers.BatchNormalization()(x) x = layers.PReLU()(x) # FCs x = layers.Flatten()(x) x = layers.Dense(10)(x) x = layers.PReLU()(x) output = layers.Dense(output_dims)(x) output = HardTanh(min_z=min_z, max_z=max_z)(output) model = keras.Model(input, output) return model
################################################################################ ################################################################################ ################################################################################ # Try replacing GRU, or SimpleRNN. RNN = layers.LSTM HIDDEN_SIZE = 128 BATCH_SIZE = 128 LAYERS = 5 print('Build model...') model = Sequential() # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE. # Note: In a situation where your input sequences have a variable length, # use input_shape=(None, num_feature). model.add(RNN(HIDDEN_SIZE, input_shape=(INPUT_LEN, len(chars)), activation=layers.PReLU(), recurrent_activation='sigmoid',\ dropout=0.25, recurrent_dropout=0.125)) model.add(BatchNormalization(center=True, scale=True)) # As the decoder RNN's input, repeatedly provide with the last output of # RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum # length of output, e.g., when DIGITS=3, max output is 999+999=1998. model.add(layers.RepeatVector(OUTPUT_LEN)) model.add(BatchNormalization(center=True, scale=True)) # The decoder RNN could be multiple layers stacked or a single layer. for _ in range(LAYERS): # By setting return_sequences to True, return not only the last output but # all the outputs so far in the form of (num_samples, timesteps, # output_dim). This is necessary as TimeDistributed in the below expects # the first dimension to be the timesteps. model.add(RNN(HIDDEN_SIZE, return_sequences=True, activation=layers.PReLU(), recurrent_activation='sigmoid',\ dropout=0.25, recurrent_dropout = 0.125))
################################################################################ ################################################################################ # Try replacing GRU, or SimpleRNN. RNN = layers.LSTM HIDDEN_SIZE = 128 BATCH_SIZE = 128 LAYERS = 1 print('Build model...') model = Sequential() # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE. # Note: In a situation where your input sequences have a variable length, # use input_shape=(None, num_feature). model.add(RNN(HIDDEN_SIZE, input_shape=(INPUT_LEN, len(chars)),\ activation=layers.PReLU(), recurrent_activation='sigmoid',\ dropout=0.25, recurrent_dropout=0.125)) model.add(BatchNormalization(center=True, scale=True)) # As the decoder RNN's input, repeatedly provide with the last output of # RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum # length of output, e.g., when DIGITS=3, max output is 999+999=1998. model.add(layers.RepeatVector(OUTPUT_LEN)) model.add(BatchNormalization(center=True, scale=True)) # The decoder RNN could be multiple layers stacked or a single layer. for _ in range(LAYERS): # By setting return_sequences to True, return not only the last output but # all the outputs so far in the form of (num_samples, timesteps, # output_dim). This is necessary as TimeDistributed in the below expects # the first dimension to be the timesteps. model.add(RNN(HIDDEN_SIZE, return_sequences=True,\ activation=layers.PReLU(), recurrent_activation='sigmoid',\
def UNet_like2(input_tensor=None): img_input = input_tensor ### Conv1 conv1 = layers.Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal', name='conv1_1')(img_input) conv1 = layers.BatchNormalization(axis=3, name='conv1_1bn')(conv1) conv1 = layers.PReLU(shared_axes=[1, 2], name='prelu1_1')(conv1) conv1 = layers.Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal', name='conv1_2')(conv1) conv1 = layers.BatchNormalization(axis=3, name='conv1_2bn')(conv1) conv1 = layers.PReLU(shared_axes=[1, 2], name='prelu1_2')(conv1) pool1 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(conv1) ### Conv 2 conv2 = layers.Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', name='conv2_1')(pool1) conv2 = layers.BatchNormalization(axis=3, name='conv2_1bn')(conv2) conv2 = layers.PReLU(shared_axes=[1, 2], name='prelu2_1')(conv2) conv2 = layers.Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', name='conv2_2')(conv2) conv2 = layers.BatchNormalization(axis=3, name='conv2_2bn')(conv2) conv2 = layers.PReLU(shared_axes=[1, 2], name='prelu2_2')(conv2) pool2 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool2')(conv2) ### Conv 3 conv3 = layers.Conv2D(256, (3, 3), padding='same', kernel_initializer='he_normal', name='conv3_1')(pool2) conv3 = layers.BatchNormalization(axis=3, name='conv3_1bn')(conv3) conv3 = layers.PReLU(shared_axes=[1, 2], name='prelu3_1')(conv3) conv3 = layers.Conv2D(256, (3, 3), padding='same', kernel_initializer='he_normal', name='conv3_2')(conv3) conv3 = layers.BatchNormalization(axis=3, name='conv3_2bn')(conv3) conv3 = layers.PReLU(shared_axes=[1, 2], name='prelu3_2')(conv3) pool3 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3')(conv3) ### Conv 4 conv4 = layers.Conv2D(512, (3, 3), padding='same', kernel_initializer='he_normal', name='conv4_1')(pool3) conv4 = layers.BatchNormalization(axis=3, name='conv4_1bn')(conv4) conv4 = layers.PReLU(shared_axes=[1, 2], name='prelu4_1')(conv4) conv4 = layers.Conv2D(512, (3, 3), padding='same', kernel_initializer='he_normal', name='conv4_2')(conv4) conv4 = layers.BatchNormalization(axis=3, name='conv4_2bn')(conv4) conv4 = layers.PReLU(shared_axes=[1, 2], name='prelu4_2')(conv4) drop4 = layers.Dropout(0.5)(conv4) ### pool4 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4')(drop4) ### Conv 5 conv5 = layers.Conv2D(1024, (3, 3), padding='same', kernel_initializer='he_normal', name='conv5_1')(pool4) conv5 = layers.BatchNormalization(axis=3, name='conv5_1bn')(conv5) conv5 = layers.PReLU(shared_axes=[1, 2], name='prelu5_1')(conv5) conv5 = layers.Conv2D(1024, (3, 3), padding='same', kernel_initializer='he_normal', name='conv5_2')(conv5) conv5 = layers.BatchNormalization(axis=3, name='conv5_2bn')(conv5) conv5 = layers.PReLU(shared_axes=[1, 2], name='prelu5_2')(conv5) drop5 = layers.Dropout(0.5)(conv5) ### ### upconv + conv 6 upconv6 = layers.Conv2D(512, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv6_1')( layers.UpSampling2D(size=(2, 2))(drop5)) upconv6 = layers.PReLU(shared_axes=[1, 2], name='prelu6_1')(upconv6) merge6 = layers.concatenate([drop4, upconv6], axis=3) conv6 = layers.Conv2D(512, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv6_2')(merge6) conv6 = layers.PReLU(shared_axes=[1, 2], name='prelu6_2')(conv6) conv6 = layers.Conv2D(512, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv6_3')(conv6) conv6 = layers.PReLU(shared_axes=[1, 2], name='prelu6_3')(conv6) ### Upconv + Conv 7 upconv7 = layers.Conv2D(256, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv7_1')( layers.UpSampling2D(size=(2, 2))(conv6)) upconv7 = layers.PReLU(shared_axes=[1, 2], name='prelu7_1')(upconv7) merge7 = layers.concatenate([conv3, upconv7], axis=3) conv7 = layers.Conv2D(256, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv7_2')(merge7) conv7 = layers.PReLU(shared_axes=[1, 2], name='prelu7_2')(conv7) conv7 = layers.Conv2D(256, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv7_3')(conv7) conv7 = layers.PReLU(shared_axes=[1, 2], name='prelu7_3')(conv7) ### Upconv + Conv 8 upconv8 = layers.Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv8_1')( layers.UpSampling2D(size=(2, 2))(conv7)) upconv8 = layers.PReLU(shared_axes=[1, 2], name='prelu8_1')(upconv8) merge8 = layers.concatenate([conv2, upconv8], axis=3) conv8 = layers.Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv8_2')(merge8) conv8 = layers.PReLU(shared_axes=[1, 2], name='prelu8_2')(conv8) conv8 = layers.Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv8_3')(conv8) conv8 = layers.PReLU(shared_axes=[1, 2], name='prelu8_3')(conv8) ### Upconv + Conv 9 upconv9 = layers.Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv9_1')( layers.UpSampling2D(size=(2, 2))(conv8)) upconv9 = layers.PReLU(shared_axes=[1, 2], name='prelu9_1')(upconv9) merge9 = layers.concatenate([conv1, upconv9], axis=3) conv9 = layers.Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv9_2')(merge9) conv9 = layers.PReLU(shared_axes=[1, 2], name='prelu9_2')(conv9) conv9 = layers.Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv9_3')(conv9) conv9 = layers.PReLU(shared_axes=[1, 2], name='prelu9_3')(conv9) conv9 = layers.Conv2D(2, (3, 3), padding='same', kernel_initializer='he_normal', name='upconv9_4')(conv9) conv9 = layers.PReLU(shared_axes=[1, 2], name='prelu9_4')(conv9) ### Conv 10 conv10 = layers.Conv2D(1, (1, 1), activation='sigmoid', name='conv10')(conv9) # Create model model = models.Model(img_input, conv10, name='U-Net') return model
def WaveNetMinimal(): """ Repurposed for my use from https://arxiv.org/abs/1609.03499 """ dilation_depth = 8 # nb_stacks = 1 nb_output_bins = 4 nb_filters = 64 use_bias = False def residual_block(x): original_x = x # TODO: initalization, regularization? # Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet. tanh_out = CausalAtrousConvolution1D(nb_filters, 2, atrous_rate=2**i, border_mode='valid', causal=True, bias=use_bias, name='dilated_conv_%d_tanh_s%d' % (2**i, s), activation='tanh')(x) x = layers.Dropout(0.2)(x) sigm_out = CausalAtrousConvolution1D(nb_filters, 2, atrous_rate=2**i, border_mode='valid', causal=True, bias=use_bias, name='dilated_conv_%d_sigm_s%d' % (2**i, s), activation='sigmoid')(x) x = layers.Merge(mode='mul', name='gated_activation_%d_s%d' % (i, s))([tanh_out, sigm_out]) res_x = layers.Convolution1D(nb_filters, 1, border_mode='same', bias=use_bias)(x) res_x = layers.Merge(mode='sum')([original_x, res_x]) return res_x _log.info('Building model...') input = Input(shape=(1000, nb_output_bins), name='input_part') out = input out = CausalAtrousConvolution1D(nb_filters, 2, atrous_rate=1, border_mode='valid', causal=True, name='initial_causal_conv')(out) for s in range(nb_stacks): for i in range(0, dilation_depth + 1): out = residual_block(out) out = layers.PReLU()(out) out = layers.Convolution1D(nb_filter=64, filter_length=3, border_mode='same', init='he_normal')(out) out = layers.Dropout(0.2)(out) out = layers.Activation('relu')(out) out = layers.Convolution1D(nb_filter=64, filter_length=3, border_mode='same', init='he_normal')(out) out = layers.Dropout(0.2)(out) out = layers.Activation('relu')(out) out = layers.Flatten()(out) predictions = layers.Dense(919, name='fc1')(out) predictions = layers.Activation('sigmoid', name="output_sigmoid")(predictions) model = Model(input, predictions) _log.info('Compiling model...') model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) return model
def run_script(): # Tunable parameters TRAINING_SIZE = 5000 TEST_SIZE = 1000 INPUT_LEN = 10 # The maximum number of digits in the input integers DECIMALS = 2 # the number of decimals in the scientific notation # This number is fixed OUTPUT_LEN = 6 + DECIMALS chars = '0123456789e+. ' ctable = CharacterTable(chars) print('Generating data...') questions, expected = generate_data_set(TRAINING_SIZE+TEST_SIZE, INPUT_LEN, DECIMALS) ################################################################################ ################################################################################ ################################################################################ print('Vectorization...') x = np.zeros((len(questions), INPUT_LEN, len(chars)), dtype=np.bool) y = np.zeros((len(questions), OUTPUT_LEN, len(chars)), dtype=np.bool) # Encode all inputs and outputs (i.e. turn strings into matrices) for i, sentence in enumerate(questions): x[i] = ctable.encode(sentence, INPUT_LEN) for i, sentence in enumerate(expected): y[i] = ctable.encode(sentence, OUTPUT_LEN) # Shuffle (x, y) in unison as the later parts of x will almost all be larger # digits. """ indices = np.arange(len(y)) np.random.shuffle(indices) x = x[indices] y = y[indices] """ # Split the data over the training set (90% training and 10% validation) and the test set split1 = int(0.9*TRAINING_SIZE) split2 = TRAINING_SIZE (x_train, x_val, x_test) = x[:split1], x[split1:split2], x[split2:] (y_train, y_val, y_test) = y[:split1], y[split1:split2], y[split2:] print('Training Data:') print(x_train.shape) print(y_train.shape) print('Validation Data:') print(x_val.shape) print(y_val.shape) print('Test Data:') print(x_val.shape) print(y_val.shape) ################################################################################ ################################################################################ ################################################################################ # Try replacing GRU, or SimpleRNN. RNN = layers.LSTM HIDDEN_SIZE = 128 BATCH_SIZE = 128 LAYERS = 1 print('Build model...') model = Sequential() # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE. # Note: In a situation where your input sequences have a variable length, # use input_shape=(None, num_feature). model.add(RNN(HIDDEN_SIZE, input_shape=(INPUT_LEN, len(chars)),\ activation=layers.PReLU(), recurrent_activation='sigmoid',\ dropout=0.25, recurrent_dropout=0.125)) model.add(BatchNormalization(center=True, scale=True)) # As the decoder RNN's input, repeatedly provide with the last output of # RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum # length of output, e.g., when DIGITS=3, max output is 999+999=1998. model.add(layers.RepeatVector(OUTPUT_LEN)) model.add(BatchNormalization(center=True, scale=True)) # The decoder RNN could be multiple layers stacked or a single layer. for _ in range(LAYERS): # By setting return_sequences to True, return not only the last output but # all the outputs so far in the form of (num_samples, timesteps, # output_dim). This is necessary as TimeDistributed in the below expects # the first dimension to be the timesteps. model.add(RNN(HIDDEN_SIZE, return_sequences=True,\ activation=layers.PReLU(), recurrent_activation='sigmoid',\ dropout=0.25, recurrent_dropout=0.125)) model.add(BatchNormalization(center=True, scale=True)) # Apply a dense layer to the every temporal slice of an input. For each of step # of the output sequence, decide which character should be chosen. model.add(layers.TimeDistributed(layers.Dense(len(chars), activation='softmax'))) model.compile(loss='categorical_crossentropy', optimizer='Nadam', metrics=['accuracy']) model.summary() #training_accuracies = [] #training_losses = [] val_all = [] val_all_but_one = [] # Train the model each generation and show predictions against the validation # dataset. for iteration in range(1, 150): print() print('-' * 50) print('Iteration', iteration) hist = model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=1, validation_data=(x_val, y_val)) # Select 10 samples from the validation set at random so we can visualize # errors. #print(hist.history) #training_accuracies.append([hist.history['acc'][0], hist.history['val_acc'][0]]) #training_losses.append([hist.history['loss'][0], hist.history['val_loss'][0]]) for i in range(10): ind = np.random.randint(0, len(x_val)) rowx, rowy = x_val[np.array([ind])], y_val[np.array([ind])] preds = model.predict_classes(rowx, verbose=0) q = ctable.decode(rowx[0]) correct = ctable.decode(rowy[0]) guess = ctable.decode(preds[0], calc_argmax=False) print('Q', q, end=' ') print('T', correct, end=' ') if correct == guess: print('OK', end=' ') else: print('..', end=' ') print(guess) full, one_off = 0, 0 predict = model.predict_classes(x_val, verbose=0) for i in range(len(x_val)): correct = ctable.decode(y_val[i]) guess = ctable.decode(predict[i], calc_argmax=False) if correct == guess: full += 1 elif match(correct, guess): one_off += 1 if (iteration > 50): q = ctable.decode(x_val[i]) print('Q', q, end=' ') print('T', correct, end=' ') print('..', end=' ') print(guess) print('{}% of validation examples are completely correct'.format(100. *float(full)/len(x_val))) print('{}% of validation examples are one off'.format(100.*float(one_off)/len(x_val))) val_all.append(100.*float(full)/len(x_val)) val_all_but_one.append(100.*float(one_off)/len(x_val)) ################################################################################ ################################################################################ ################################################################################ scores = model.evaluate(x_test, y_test, verbose=1) print('-----------------------------------------') print('Test accuracy: ', scores[1]) #np.save('training_accuracies', np.array(training_accuracies)) #np.save('training_losses', np.array(training_losses)) np.save('results/learn_rule/val_all_'+str(INPUT_LEN), np.array(val_all)) np.save('results/learn_rule/val_all_but_one_'+str(INPUT_LEN), np.array(val_all_but_one)) full, one_off = 0, 0 predict = model.predict_classes(x_test, verbose=0) for i in range(len(x_test)): correct = ctable.decode(y_test[i]) guess = ctable.decode(predict[i], calc_argmax=False) if correct == guess: full += 1 elif match(correct, guess): one_off += 1 print('{}% of test examples are completely correct'.format(100. *float(full)/len(x_test))) print('{}% of test examples are one off'.format(100.*float(one_off)/len(x_test))) print(100.*float(full)/len(x_test), 100.*float(one_off)/len(x_test))
def create_model(desired_sample_rate, dilation_depth, nb_stacks): # desired_sample_rate = 4410 nb_output_bins = 4 # nb_filters = 256 nb_filters = 64 # dilation_depth = 9 # # nb_stacks = 1 use_bias = False res_l2 = 0 final_l2 = 0 fragment_length = 488 + compute_receptive_field_( desired_sample_rate, dilation_depth, nb_stacks)[0] fragment_stride = 488 use_skip_connections = True learn_all_outputs = True def residual_block(x): original_x = x # TODO: initalization, regularization? # Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet. tanh_out = CausalAtrousConvolution1D(nb_filters, 2, atrous_rate=2**i, border_mode='valid', causal=True, bias=use_bias, name='dilated_conv_%d_tanh_s%d' % (2**i, s), activation='tanh', W_regularizer=l2(res_l2))(x) x = layers.Dropout(0.2)(x) sigm_out = CausalAtrousConvolution1D(nb_filters, 2, atrous_rate=2**i, border_mode='valid', causal=True, bias=use_bias, name='dilated_conv_%d_sigm_s%d' % (2**i, s), activation='sigmoid', W_regularizer=l2(res_l2))(x) x = layers.Merge(mode='mul', name='gated_activation_%d_s%d' % (i, s))([tanh_out, sigm_out]) res_x = layers.Convolution1D(nb_filters, 1, border_mode='same', bias=use_bias, W_regularizer=l2(res_l2))(x) skip_x = layers.Convolution1D(nb_filters, 1, border_mode='same', bias=use_bias, W_regularizer=l2(res_l2))(x) res_x = layers.Merge(mode='sum')([original_x, res_x]) return res_x, skip_x input = Input(shape=(fragment_length, nb_output_bins), name='input_part') out = input skip_connections = [] out = CausalAtrousConvolution1D(nb_filters, 2, atrous_rate=1, border_mode='valid', causal=True, name='initial_causal_conv')(out) for s in range(nb_stacks): for i in range(0, dilation_depth + 1): out, skip_out = residual_block(out) skip_connections.append(skip_out) if use_skip_connections: out = layers.Merge(mode='sum')(skip_connections) out = layers.PReLU()(out) # out = layers.Convolution1D(nb_filter=256, filter_length=1, border_mode='same', # W_regularizer=l2(final_l2))(out) out = layers.Convolution1D(nb_filter=nb_output_bins, filter_length=3, border_mode='same')(out) out = layers.Dropout(0.5)(out) out = layers.PReLU()(out) out = layers.Convolution1D(nb_filter=nb_output_bins, filter_length=3, border_mode='same')(out) if not learn_all_outputs: raise DeprecationWarning( 'Learning on just all outputs is wasteful, now learning only inside receptive field.' ) out = layers.Lambda( lambda x: x[:, -1, :], output_shape=(out._keras_shape[-1], ))( out) # Based on gif in deepmind blog: take last output? # out = layers.Activation('softmax', name="output_softmax")(out) out = layers.PReLU()(out) # out = layers.Activation('sigmoid', name="output_sigmoid")(out) out = layers.Flatten()(out) predictions = layers.Dense(919, activation='sigmoid', name='fc1')(out) model = Model(input, predictions) # x = model.output # x = layers.Flatten()(x) # # x = layers.Dense(output_dim=1024)(x) # # x = layers.PReLU()(x) # # x = layers.Dropout(0.5)(x) # # x = layers.Dense(output_dim=919)(x) # # x = layers.Activation('sigmoid')(x) # model = Model(input=model.input, output=predictions) receptive_field, receptive_field_ms = compute_receptive_field_( desired_sample_rate, dilation_depth, nb_stacks) _log.info('Receptive Field: %d (%dms)' % (receptive_field, int(receptive_field_ms))) return model
def coa_res(model_weights = None): EMBED_HIDDEN_SIZE = 300 #shared weight layers shared_LSTM = LSTM(EMBED_HIDDEN_SIZE, return_sequences=True) ########################### sentence = layers.Input(shape=(max_para,dimension), dtype='float32') encoded_sentence =shared_LSTM(sentence) question = layers.Input(shape=(max_q,dimension), dtype='float32') encoded_question = shared_LSTM(question) #Encoder merge_1 = layers.dot([encoded_sentence, encoded_question], axes = 2 ) A_Q = layers.Activation("softmax")(merge_1) merge_2 = layers.dot([encoded_question, encoded_sentence], axes = 2 ) A_D = layers.Activation("softmax")(merge_2) C_Q = layers.dot([A_Q, encoded_sentence], axes = 1 ) C_Q = layers.concatenate([encoded_question, C_Q], axis=2) C_D = layers.dot([A_D, C_Q], axes=1) C_ = layers.concatenate([encoded_sentence, C_D], axis=2) U = Bidirectional(LSTM(EMBED_HIDDEN_SIZE, return_sequences=True))(C_) U = Dropout(0.5)(U) #Decoder start = convBN(U, 100) start = RU(start, 100, 1) start = Dropout(0.5)(start) start = convBN_pool(start, 64) start = RU(start, 64, 1) start = convBN_pool(start, 64) start = RU(start, 64, 2) start = convBN_pool(start, 128) start = RU(start, 128, 1) start =layers.PReLU()(start) start = RU(start, 128, 2) start = Dropout(0.5)(start) start = convBN_pool(start, 256) start = RU(start, 256, 2) start = Dropout(0.5)(start) start = convBN_pool(start, 128) start = RU(start, 128, 1) start = layers.PReLU()(start) start = Dropout(0.5)(start) start = convBN(start, 64) start = RU(start, 64, 1) start = layers.PReLU()(start) start = Flatten()(start) start = Dropout(0.5)(start) start = Dense(max_para, activation='softmax', name='output_1')(start) end = GRU(100, return_sequences=True)(U) end = convBN(end, 100) end = RU(end, 100, 1) end = Dropout(0.5)(end) end = convBN_pool(end, 64) end = RU(end, 64, 1) end = convBN_pool(end, 64) end = RU(end, 64, 2) end = convBN_pool(end, 128) end = RU(end, 128, 1) end = layers.PReLU()(end) end = RU(end, 128, 2) end = Dropout(0.5)(end) end = convBN_pool(end, 256) end = RU(end, 256, 2) end = Dropout(0.5)(end) end = convBN_pool(end, 128) end = RU(end, 128, 1) end = layers.PReLU()(end) end = Dropout(0.5)(end) end = convBN(end, 64) end = RU(end, 64, 1) end = layers.PReLU()(end) end = Flatten()(end) end = Dropout(0.5)(end) end = Dense(max_para, activation='softmax', name='output_2')(end) model = Model([sentence, question],[start, end]) if model_weights != None: model.load_weights(model_weights) model.compile(optimizer='adam', loss={'output_1': 'categorical_crossentropy', 'output_2': 'categorical_crossentropy'}, metrics=['accuracy']) return model