def createConvModel(self, train_shape, output_count): # lenet_5_model = Sequential([ # layers.Conv1D(6, kernel_size=(5), strides=1, activation='relu', input_shape=(train_shape[0], train_shape[1]), # padding='same'), # C1 # layers.MaxPool1D(), # S2 # layers.Conv1D(16, kernel_size=(5), strides=1, activation='relu', padding='valid'), # C3 # #layers.MaxPool1D(), # S4 # layers.GlobalMaxPool1D(), # Flatten # layers.Dense(120, activation='relu'), # C5 # layers.Dense(84, activation=tf.nn.leaky_relu), # F6 # layers.Dense(output_count) # Output layer # ]) # lenet_5_model.compile(loss='mse', # optimizer='adam', # metrics=['mse', 'mae']) input_tensor = Input(shape=(train_shape[0], train_shape[1])) x = layers.Conv1D(6, kernel_size=(3), activation='relu', strides=1)(input_tensor) x = layers.AveragePooling1D()(x) x = layers.Conv1D(16, kernel_size=(3), padding='valid', activation='relu', strides=1)(x) x = layers.AveragePooling1D()(x) x = layers.Flatten()(x) x = layers.Dense(120, activation='relu')(x) x = layers.Dense(84, activation='relu')(x) output_tensor = layers.Dense(output_count)(x) lenet_5_model = tf.keras.Model(input_tensor, output_tensor) lenet_5_model.compile(loss='mse', optimizer='adam', metrics=['mse', 'mae']) return lenet_5_model
def _pooling_function(self, inputs, pool_size, strides, padding, data_format): input_real, input_imag = complex_to_real_imag(inputs) real_outputs = KL.AveragePooling1D(pool_size, strides, padding)(input_real) imag_outputs = KL.AveragePooling1D(pool_size, strides, padding)(input_imag) outputs = real_imag_to_complex(real_outputs, imag_outputs) return outputs
def DilateConcatConvNet(): inputs = layers.Input(shape=(1024,4)) # Returns a placeholder tensor x = layers.Conv1D(filters=16, kernel_size=7,activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(inputs) x = layers.Conv1D(filters=32, kernel_size=5,activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(x) x = layers.Conv1D(filters=64, kernel_size=7, activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(x) x = layers.AveragePooling1D(pool_size=2,strides=2)(x) x = layers.Dropout(0.2)(x) dil1 = layers.Conv1D(filters=64, kernel_size=37, dilation_rate=2, activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(x) dil2 = layers.Conv1D(filters=64, kernel_size=37, dilation_rate=4, activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(x) dil3 = layers.Conv1D(filters=64, kernel_size=37, dilation_rate=6, activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(x) x = layers.Cropping1D((108,108))(x) dil1 = layers.Cropping1D((72,72))(dil1) dil2 = layers.Cropping1D((36,36))(dil2) x = layers.Reshape((288, 1, 64))(x) dil1 = layers.Reshape((288, 1, 64))(dil1) dil2 = layers.Reshape((288, 1, 64))(dil2) dil3 = layers.Reshape((288, 1, 64))(dil3) concat = layers.concatenate([x, dil1, dil2, dil3], axis=2) c1 = layers.Conv2D(filters=96,kernel_size=[5,4],strides=(1,4),activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(concat) c1 = layers.AveragePooling2D(pool_size=(2,1),strides=(2,1))(c1) c1 = layers.Reshape((142,96))(c1) c2 = layers.Conv1D(filters=128,kernel_size=7,activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(c1) c2 = layers.AveragePooling1D(pool_size=2,strides=2)(c2) c3 = layers.Conv1D(filters=160,kernel_size=5,activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(c2) c3 = layers.AveragePooling1D(pool_size=2,strides=2)(c3) c4 = layers.Conv1D(filters=192,kernel_size=7,activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(c3) c4 = layers.AveragePooling1D(pool_size=2,strides=2)(c4) c5 = layers.Conv1D(filters=224,kernel_size=5,activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(c4) c5 = layers.AveragePooling1D(pool_size=2,strides=2)(c5) f = layers.Flatten()(c5) fc1 = layers.Dense(1024)(f) fc2 = layers.Dense(2048)(fc1) model = tf.keras.Model(inputs=inputs, outputs=fc2) return model
def model(train_data, filters=1, kernel_size=9, motif_init=None, lr=0.001): seq_length = train_data[0]["seq"].shape[1] pwm_list = train_data[2] if motif_init is not None: # Motif init is a dictionary with fields: "stddev" kinit = ci.PSSMKernelInitializer(pwm_list, stddev=motif_init.get("stddev", 0.05), # if not specified, use 0.05 add_noise_before_Pwm2Pssm=True) binit = "zeros" else: kinit = "glorot_uniform" binit = "zeros" # sequence in_dna = cl.InputDNA(seq_length=seq_length, name="seq") x = cl.ConvDNA(filters=filters, kernel_size=kernel_size, activation="relu", kernel_initializer=kinit, bias_initializer=binit, name="conv1")(in_dna) x = kl.AveragePooling1D(pool_size=4)(x) x = kl.Flatten()(x) x = kl.Dense(units=1)(x) m = Model(in_dna, x) m.compile(Adam(lr=lr), loss="binary_crossentropy", metrics=["acc"]) return m
def LSTMCNN_embedding(input_shape, n_filters=64, filter_size=5, pool_length=4, \ dropout=False, dropW=0.2, dropU=0.2, lstm_dims=128, embedding=None, \ hiddens=[], notactivate=False, bidirectional=False): model = Sequential() # model.add( layers.InputLayer(input_shape = (input_shape,)) ) model.add(embedding) if dropout: model.add(layers.Dropout(dropout)) model.add( layers.Conv1D(n_filters, filter_size, border_mode='valid', subsample_length=1)) if not notactivate: model.add(layers.Activation('relu')) model.add(layers.AveragePooling1D(pool_length)) if bidirectional: model.add( layers.Bidirectional( \ layers.LSTM(lstm_dims, dropout_W=dropW, dropout_U=dropU) ) ) else: model.add(layers.LSTM(lstm_dims, dropout_W=dropW, dropout_U=dropU)) for hidden in hiddens: model.add(layers.Dense(hidden)) if dropout: model.add(layers.Dropout(dropout)) model.add(layers.Activation('relu')) return model
def createConvModel(self, train_shape, output_count): input_tensor = Input(shape=(train_shape[0], train_shape[1])) x = layers.Conv1D(6, kernel_size=(3), activation='relu', strides=1)(input_tensor) x = layers.AveragePooling1D()(x) x = layers.Conv1D(16, kernel_size=(3), padding='valid', activation='relu', strides=1)(x) x = layers.AveragePooling1D()(x) x = layers.Flatten()(x) x = layers.Dense(120, activation='relu')(x) x = layers.Dense(84, activation='relu')(x) output_tensor = layers.Dense(output_count)(x) lenet_5_model = tf.keras.Model(input_tensor, output_tensor) lenet_5_model.compile(loss='mse', optimizer='adam', metrics=['mse', 'mae']) return lenet_5_model
def DilateSumNet(): inputs = layers.Input(shape=(1024,4)) # Returns a placeholder tensor x = layers.Conv1D(filters=32, kernel_size=7,activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(inputs) x = layers.Conv1D(filters=64, kernel_size=5,activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(x) x = layers.Conv1D(filters=128, kernel_size=7, activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(x) conv = layers.AveragePooling1D(pool_size=2, strides=2, padding='same')(x) dil1 = layers.Conv1D(filters=128, kernel_size=37, dilation_rate=2, activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(conv) dil1Avg = layers.AveragePooling1D(pool_size=2, strides=2, padding='same')(dil1) dil1Avg = layers.Cropping1D(36)(dil1Avg) dil2 = layers.Conv1D(filters=128, kernel_size=37, dilation_rate=4, activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(conv) dil2Avg = layers.AveragePooling1D(pool_size=2, strides=2, padding='same')(dil2) dil2Avg = layers.Cropping1D(18)(dil2Avg) dil3 = layers.Conv1D(filters=128, kernel_size=37, dilation_rate=6, activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(conv) dil3Avg = layers.AveragePooling1D(pool_size=2, strides=2, padding='same')(dil3) add = layers.add([dil1Avg, dil2Avg, dil3Avg]) add = layers.Reshape((144,1,128))(add) up1 = layers.Conv2DTranspose(filters=128, kernel_size=(5,1), strides=(2,1))(add) up1conv = layers.Conv2D(filters=64, kernel_size=(3,1),activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(up1) up1conv = layers.Conv2D(filters=64, kernel_size=(3,1),activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(up1conv) up2 = layers.Conv2DTranspose(filters=64, kernel_size=(7,1), strides=(2,1))(up1conv) up2conv = layers.Conv2D(filters=32, kernel_size=(3,1),activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(up2) up2conv = layers.Conv2D(filters=32, kernel_size=(3,1),activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(up2conv) up3 = layers.Conv2DTranspose(filters=32, kernel_size=(5,1), strides=(2,1))(up2conv) up3conv = layers.Conv2D(filters=16, kernel_size=(3,1),activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(up3) up3conv = layers.Conv2D(filters=16, kernel_size=(3,1),activation=tf.nn.relu, padding='valid', kernel_initializer='he_normal')(up3conv) pre_predictions = layers.Reshape((1149,16))(up3conv) pre_predictions = layers.Cropping1D((62,63))(pre_predictions) predictions = layers.LocallyConnected1D(kernel_size=1,filters=2)(pre_predictions) predictions = layers.Reshape((2048,))(predictions) model = tf.keras.Model(inputs=inputs, outputs=predictions) return model
def get_proportions_model(image_shape, bag_size, filter1=5, kernel1=9, filters2=10, kernel2=5): """Creates a model for classifying labels by learning from label proportions. Keyword arguments: image_shape: 2 tuplet, the dimensions of each image. bag_size: Integer, The number of images in each bag. filter1: Integer, The dimensionality of the output space of the first layer (default 5). kernel1: A 2 tuple specifying the height and width of the 2D convolutional window of the first layer (default 9). filters2: Integer, the dimensionaloty of the output space of the second layer (default 10). kernel2: A 2 tuple specifying the height and width of the 2D convolutional window of the second layer (default 5). """ # Input > 2 conv layers > single output # Takes an image from the dataset and spits out the probability of the image containing the measured label image_input = layers.Input(shape=image_shape) conv1 = layers.Convolution2D(filter1, kernel1, activation='relu', input_shape=image_shape)(image_input) conv2 = layers.Convolution2D(filters2, kernel2, activation='relu')(conv1) flat = layers.Flatten()(conv2) image_output = layers.Dense(1, activation='sigmoid')(flat) conv_model = models.Model(inputs=image_input, outputs=image_output) conv_model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy']) # input > TimeDistributed > average of the results of each element of the TimeSitributed layer > output, the average as a single number # Takes in a bag of images, processes them through the same neural network (the one we created above), and then averages the outputs for every image. Both the average and the singular outputs will be used in the training. set_input = layers.Input(shape=(bag_size, ) + image_shape) set_processing = layers.TimeDistributed(conv_model, name='inter')(set_input) set_output = layers.AveragePooling1D(bag_size)(set_processing) set_flat_output = layers.Flatten()(set_output) set_model = models.Model(inputs=set_input, outputs=set_flat_output) # Arbitrary optimizer, I don't know if I should use a different one. set_model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy']) return set_model
def create_model(num_filters, kernel_size, vocab_size, embedding_dim, maxlen): model = Sequential() model.add(layers.Embedding(vocab_size, embedding_dim, input_length=maxlen)) model.add(layers.SpatialDropout1D(0.25)) model.add(layers.Conv1D(num_filters, kernel_size, padding='same', activation='relu')) model.add(layers.AveragePooling1D()) model.add(layers.GlobalMaxPool1D()) model.add(layers.Dense(10, activation='relu')) model.add(layers.Dropout(0.25)) model.add(layers.Dense(1, activation='sigmoid')) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) return model
def get_pool(pooling): pooling = deepcopy(pooling) pool_type = pooling["pool_type"] del pooling["pool_type"] if pool_type == "max": return [kl.MaxPooling1D(pool_size=pooling["pool_size"]), kl.Flatten()] elif pool_type == "mean": return [ kl.AveragePooling1D(pool_size=pooling["pool_size"]), kl.Flatten() ] elif pool_type == "weight": return [cl.SmoothPositionWeight(**pooling), cl.GlobalSumPooling1D()] else: raise ValueError("")
def cnn(): optimizer = keras.optimizers.RMSprop(lr=0.001, clipvalue=1.0) input_layer_one = keras.Input(shape=(None, 12750)) m = layers.Conv1D(32, (5, ), activation=backend.sigmoid)(input_layer_one) # 124, 5 m = layers.Conv1D(32, (3, ), activation=backend.sigmoid)(m) m = layers.AveragePooling1D()(m) m = layers.Dense(52, activation=backend.sigmoid)(m) m = layers.Dropout(0.5)(m) m = layers.Dense(26, activation=backend.sigmoid)(m) m = layers.Dense(1)(m) model = keras.models.Model(inputs=input_layer_one, outputs=m) model.compile(optimizer=optimizer, loss='categorical_crossentropy') return model
def create_sequence_model(base_model, sequence_length, input_shape, output_size, use_lstm): assert base_model == "voxnet" or base_model == "pointnet" if base_model == "voxnet": base_model = create_voxnet_model_homepage(input_shape, output_size) elif base_model == "pointnet": base_model = create_point_net(input_shape, output_size) model = models.Sequential() model.add(layers.TimeDistributed(base_model, input_shape=(sequence_length,) + input_shape)) if use_lstm == True: model.add(layers.LSTM(8, activation="relu")) else: model.add(layers.AveragePooling1D(sequence_length)) model.add(layers.Flatten()) model.add(layers.Dense(output_size)) return model
def build_model(X_train, L1, L2, optim='rmsprop', model_type='ff'): from keras import models from keras import layers if model_type == 'ff': if L2 != None: model = models.Sequential() model.add( layers.Dense(L1, activation='relu', input_shape=(X_train.shape[1], ))) # train_X model.add(layers.Dense(L2, activation='relu')) model.add(layers.Dense(8)) model.compile(optimizer=optim, loss='mse', metrics=['mae']) elif L2 == None: model = models.Sequential() model.add( layers.Dense(L1, activation='relu', input_shape=(X_train.shape[1], ))) # train_X model.add(layers.Dense(8)) model.compile(optimizer=optim, loss='mse', metrics=['mae']) elif model_type == '1dconv': model = models.Sequential() model.add( layers.Conv1D(20, 10, activation='relu', input_shape=(X_train.shape[1], 1))) model.add(layers.AveragePooling1D()) model.add(layers.Conv1D(12, 7, activation='relu')) model.add(layers.MaxPooling1D()) model.add(layers.Conv1D(6, 5, activation='relu')) model.add(layers.GlobalAveragePooling1D()) model.add(layers.Dropout(0.5)) model.add(layers.Dense(8, activation='linear')) model.compile(optimizer='rmsprop', loss='mse', metrics=['mae']) return model
def _build_averagepooling1d(layer_description): return layers.AveragePooling1D()
def build_birnn_feature_coattention_cnn_model(voca_dim, time_steps, num_features, feature_dim, output_dim, model_dim, mlp_dim, num_filters, filter_sizes, item_embedding=None, rnn_depth=1, mlp_depth=1, drop_out=0.5, rnn_drop_out=0., rnn_state_drop_out=0., cnn_drop_out=0.5, pooling='max', trainable_embedding=False, gpu=False, return_customized_layers=False): """ Create A Bidirectional Attention Model. :param voca_dim: vocabulary dimension size. :param time_steps: the length of input :param output_dim: the output dimension size :param model_dim: rrn dimension size :param mlp_dim: the dimension size of fully connected layer :param item_embedding: integer, numpy 2D array, or None (default=None) If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor. If item_embedding is a matrix, this matrix will be used as the embedding matrix. If item_embedding is None, then connect input tensor to RNN layer directly. :param rnn_depth: rnn depth :param mlp_depth: the depth of fully connected layers :param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism :param drop_out: dropout rate of fully connected layers :param rnn_drop_out: dropout rate of rnn layers :param rnn_state_drop_out: dropout rate of rnn state tensor :param trainable_embedding: boolean :param gpu: boolean, default=False If True, CuDNNLSTM is used instead of LSTM for RNN layer. :param return_customized_layers: boolean, default=False If True, return model and customized object dictionary, otherwise return model only :return: keras model """ if model_dim % 2 == 1: model_dim += 1 if item_embedding is not None: inputs = models.Input(shape=(time_steps, ), dtype='int32', name='input0') x1 = inputs # item embedding if isinstance(item_embedding, np.ndarray): assert voca_dim == item_embedding.shape[0] x1 = layers.Embedding(voca_dim, item_embedding.shape[1], input_length=time_steps, weights=[ item_embedding, ], trainable=trainable_embedding, mask_zero=False, name='embedding_layer0')(x1) elif utils.is_integer(item_embedding): x1 = layers.Embedding(voca_dim, item_embedding, input_length=time_steps, trainable=trainable_embedding, mask_zero=False, name='embedding_layer0')(x1) else: raise ValueError( "item_embedding must be either integer or numpy matrix") else: inputs = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='input0') x1 = inputs inputs1 = models.Input(shape=(num_features, feature_dim), dtype='float32', name='input1') x2 = layers.Dense(feature_dim, name="feature_map_layer", activation="relu")(inputs1) if gpu: # rnn encoding for i in range(rnn_depth): x1 = layers.Bidirectional(layers.CuDNNLSTM(int(model_dim / 2), return_sequences=True), name='bi_lstm_layer' + str(i))(x1) x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x1) x1 = layers.Dropout(rnn_drop_out, name="rnn_dropout_layer" + str(i))(x1) else: # rnn encoding for i in range(rnn_depth): x1 = layers.Bidirectional(layers.LSTM( int(model_dim / 2), return_sequences=True, dropout=rnn_drop_out, recurrent_dropout=rnn_state_drop_out), name='bi_lstm_layer' + str(i))(x1) x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x1) # attention attens = clayers.CoAttentionWeight(name="coattention_weights_layer")( [x1, x2]) attens1 = clayers.FeatureNormalization( name="normalized_coattention_weights_layer1", axis=1)(attens) attens2 = clayers.FeatureNormalization( name="normalized_coattention_weights_layer2", axis=2)(attens) # compare focus1 = layers.Dot((1, 1), name="focus_layer1")([attens1, x1]) focus2 = layers.Dot((2, 1), name="focus_layer2")([attens2, x2]) pair1 = layers.Concatenate(axis=-1, name="pair_layer1")([x1, focus2]) pair2 = layers.Concatenate(axis=-1, name="pair_layer2")([x2, focus1]) x1 = layers.TimeDistributed(layers.Dense(model_dim, activation="relu"), name="compare_layer1")(pair1) x2 = layers.TimeDistributed(layers.Dense(model_dim, activation="relu"), name="compare_layer2")(pair2) # Multi-Channel CNN for x1 pooled_outputs = [] for i in range(len(filter_sizes)): conv = layers.Conv1D(num_filters, kernel_size=filter_sizes[i], padding='valid', activation='relu')(x1) if pooling == 'max': conv = layers.MaxPooling1D(pool_size=time_steps - filter_sizes[i] + 1, strides=1, padding='valid')(conv) else: conv = layers.AveragePooling1D(pool_size=time_steps - filter_sizes[i] + 1, strides=1, padding='valid')(conv) pooled_outputs.append(conv) x1 = layers.Concatenate(name='concated_layer')(pooled_outputs) x1 = layers.Flatten()(x1) x1 = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x1) x1 = layers.BatchNormalization(name="batch_norm_layer")(x1) # Average Pool for x2 x2 = layers.GlobalAveragePooling1D(name="average_pool_layer")(x2) x = layers.Concatenate(axis=1, name="concat_deep_feature_layer")([x1, x2]) # MLP Layers for i in range(mlp_depth - 1): x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x) x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x) outputs = layers.Dense(output_dim, activation="softmax", name="softmax_layer0")(x) model = models.Model(inputs, outputs) if return_customized_layers: return model, { 'CoAttentionWeight': clayers.CoAttentionWeight, "FeatureNormalization": clayers.FeatureNormalization } return model
def InceptionResNetV2(include_top=True, weights=None, input_tensor=None, input_shape=None, pooling=None, classes=100, **kwargs): """Instantiates the Inception-ResNet v2 architecture. Note that the data format convention used by the model is the one specified in your Keras config at `~/.keras/keras.json`. # Arguments include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization) or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as input for the model. input_shape: optional shape tuple if input_tensor is not specified. and width should be no smaller than 75. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 3D tensor output of the last convolutional block. - `'avg'` means that global average pooling will be applied to the output of the last convolutional block, and thus the output of the model will be a 2D tensor. - `'max'` means that global max pooling will be applied. classes: optional number of classes to classify inputs into, only to be specified if `include_top` is `True`, and if no `weights` argument is specified. # Returns A Keras `Model` instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if weights is not None and not os.path.exists(weights): raise ValueError( 'The `weights` argument should be either `None` (random initialization) or the path to the weights file to be loaded.' ) if input_tensor is None: inputs = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): inputs = layers.Input(tensor=input_tensor, shape=input_shape) else: inputs = input_tensor # Stem block: 35 x 192 x = conv1d_bn(inputs, 32, 3, strides=2, padding='valid') x = conv1d_bn(x, 32, 3, padding='valid') x = conv1d_bn(x, 64, 3) x = layers.MaxPooling1D(3, strides=2)(x) x = conv1d_bn(x, 80, 1, padding='valid') x = conv1d_bn(x, 192, 3, padding='valid') x = layers.MaxPooling1D(3, strides=2)(x) # Mixed 5b (Inception-A block): 35 x 320 branch_0 = conv1d_bn(x, 96, 1) branch_1 = conv1d_bn(x, 48, 1) branch_1 = conv1d_bn(branch_1, 64, 5) branch_2 = conv1d_bn(x, 64, 1) branch_2 = conv1d_bn(branch_2, 96, 3) branch_2 = conv1d_bn(branch_2, 96, 3) branch_pool = layers.AveragePooling1D(3, strides=1, padding='same')(x) branch_pool = conv1d_bn(branch_pool, 64, 1) branches = [branch_0, branch_1, branch_2, branch_pool] channel_axis = 1 if backend.image_data_format() == 'channels_first' else 2 x = layers.Concatenate(axis=channel_axis, name='mixed_5b')(branches) # 10x block35 (Inception-ResNet-A block): 35 x 320 for block_idx in range(1, 11): x = inception_resnet_block(x, scale=0.17, block_type='block35', block_idx=block_idx) # Mixed 6a (Reduction-A block): 17 x 1088 branch_0 = conv1d_bn(x, 384, 3, strides=2, padding='valid') branch_1 = conv1d_bn(x, 256, 1) branch_1 = conv1d_bn(branch_1, 256, 3) branch_1 = conv1d_bn(branch_1, 384, 3, strides=2, padding='valid') branch_pool = layers.MaxPooling1D(3, strides=2, padding='valid')(x) branches = [branch_0, branch_1, branch_pool] x = layers.Concatenate(axis=channel_axis, name='mixed_6a')(branches) # 20x block17 (Inception-ResNet-B block): 17 x 1088 for block_idx in range(1, 21): x = inception_resnet_block(x, scale=0.1, block_type='block17', block_idx=block_idx) # Mixed 7a (Reduction-B block): 8 x 2080 branch_0 = conv1d_bn(x, 256, 1) branch_0 = conv1d_bn(branch_0, 384, 3, strides=2, padding='valid') branch_1 = conv1d_bn(x, 256, 1) branch_1 = conv1d_bn(branch_1, 288, 3, strides=2, padding='valid') branch_2 = conv1d_bn(x, 256, 1) branch_2 = conv1d_bn(branch_2, 288, 3) branch_2 = conv1d_bn(branch_2, 320, 3, strides=2, padding='valid') branch_pool = layers.MaxPooling1D(3, strides=2, padding='valid')(x) branches = [branch_0, branch_1, branch_2, branch_pool] x = layers.Concatenate(axis=channel_axis, name='mixed_7a')(branches) # 10x block8 (Inception-ResNet-C block): 8 x 2080 for block_idx in range(1, 10): x = inception_resnet_block(x, scale=0.2, block_type='block8', block_idx=block_idx) x = inception_resnet_block(x, scale=1., activation=None, block_type='block8', block_idx=10) # Final convolution block: 8 x 1536 x = conv1d_bn(x, 1536, 1, name='conv_7b') if include_top: # Classification block x = layers.GlobalAveragePooling1D(name='avg_pool')(x) x = layers.Dense(classes, activation='softmax', name='predictions')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling1D()(x) elif pooling == 'max': x = layers.GlobalMaxPooling1D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = utils.get_source_inputs(input_tensor) else: inputs = inputs # Create model. model = models.Model(inputs, x, name='1d_inception_resnet_v2') # Load weights. if weights is not None: model.load_weights(weights) return model
#This model tries to implement the one specified in #Schrider and colleagues' paper (page 234, 2018) ### #Building dependencies from keras import layers, regularizers, Input import matplotlib.pyplot as plt import numpy as np lambda_l2 = 0.0001 pop = 2500 ht_input = Input(shape=(pop * 2, ), dtype='float32', name='ht') x = layers.Conv1D(filters = 256,kernel_size = 2, activation = 'relu', kernel_regularizer= regularizers.l2(lambda_l2),input_shape=(30, 1))\ (ht_input) x = layers.AveragePooling1D(pool_size=2)(x) for i in range(0, 4): x = layers.Conv1D(filters=256, kernel_size=2, activation='relu', kernel_regularizer=regularizers.l2(lambda_l2))(x) x = layers.AveragePooling1D(pool_size=2)(x) x = layers.Flatten()(x) x = layers.Dense(units=256, activation='relu')(x) pos_input = Input(shape=(3500, 5008), dtype='float32', name='pos') y = layers.Dense(64, activation='relu', kernel_regularizer=regularizers.l2(lambda_l2))(pos_input) for i in range(0, 4): y = layers.Dense(64,
def build_model(batchSize, numTimeSteps=98, numTargets=12, numFeats=10): DropoutProb = 0.2 # Dropout probability [%] FIRST_LAYER_NEURONS = 16 NEXT_LAYERS_NEURONS = 48 L2_lambda = 0.0008 # Training parameters adamLearningRate = 0.001 adamBeta1 = 0.9 adamBeta2 = 0.999 adamEpsilon = 1e-08 adamDecay = 0.0 # Make Network i = Input(batch_shape=(batchSize, numTimeSteps, numFeats)) o = TCN(nb_filters=FIRST_LAYER_NEURONS, kernel_size=3, nb_stacks=1, dilations=[1], padding='causal', use_skip_connections=False, dropout_rate=DropoutProb, activation='linear', return_sequences=True, use_batch_norm=True, name='tcn_0')(i) o = TCN(nb_filters=NEXT_LAYERS_NEURONS, kernel_size=3, nb_stacks=1, dilations=[2, 4, 8], padding='causal', use_skip_connections=True, dropout_rate=DropoutProb, activation='relu', return_sequences=True, use_batch_norm=True, name='tcn_1')(o) o = layers.AveragePooling1D(pool_size=(4))(o) o = Flatten()(o) o = Dense(numTargets)(o) o = Activation('softmax', name='softmax')((o)) model = Model(inputs=[i], outputs=[o]) # Define optimizer adam = Adam(lr=adamLearningRate, beta_1=adamBeta1, beta_2=adamBeta2, epsilon=adamEpsilon, decay=adamDecay) # Compile model model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) print(model.count_params()) print(model.summary()) #K.clear_session() return model
def build_model(self, inception=True, res=True, strided=False, maxpool=True, avgpool=False, batchnorm=True): self.i = 0 pad = 'same' padp = 'same' c_act = self.config['c_act'] r_act = self.config['r_act'] rk_act = self.config['rk_act'] r = kr.regularizers.l2(self.config['reg']) c = self.input stride_size = self.config['strides'] if strided else 1 if inception: c0 = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) c1 = layers.Conv1D(self.config['filters'], kernel_size=8, strides=stride_size, padding=pad, activation=c_act)(c) c2 = layers.Conv1D(self.config['filters'], kernel_size=32, strides=stride_size, padding=pad, activation=c_act)(c) c = layers.concatenate([c0, c1, c2]) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) c0 = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) c1 = layers.Conv1D(self.config['filters'], kernel_size=8, strides=stride_size, padding=pad, activation=c_act)(c) c2 = layers.Conv1D(self.config['filters'], kernel_size=32, strides=stride_size, padding=pad, activation=c_act)(c) c = layers.concatenate([c0, c1, c2]) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) c0 = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) c1 = layers.Conv1D(self.config['filters'], kernel_size=8, strides=stride_size, padding=pad, activation=c_act)(c) c2 = layers.Conv1D(self.config['filters'], kernel_size=32, strides=stride_size, padding=pad, activation=c_act)(c) c = layers.concatenate([c0, c1, c2]) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) else: # No inception Modules c = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(self.input) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) c = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) c = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) if res: # Residual RNN g1 = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(c) g2 = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g1) g_concat1 = layers.concatenate([g1, g2]) g3 = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g_concat1) g_concat2 = layers.concatenate([g1, g2, g3]) g = layers.GRU(self.config['state_size'], return_sequences=False, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g_concat2) else: # No Residual RNN g = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(c) g = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g) g = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g) g = layers.GRU(self.config['state_size'], return_sequences=False, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g) d = layers.Dense(self.config['output_size'])(g) out = layers.Softmax()(d) self.model = Model(self.input, out) print("{} initialized.".format(self.model.name))
def upscale(method: str, old_model_name: str, avg_pool_unaffected=True): old_model = utils.load_model('Models/{}.yaml'.format(old_model_name), 'Models/{}.h5'.format(old_model_name)) new_model = models.Sequential() first_layer = True for layer in old_model.layers: if type(layer) is keras.layers.convolutional.Conv1D: biases = layer.get_weights()[1] old_kernels = utils.get_kernels(layer.get_weights()[0]) nodes = layer.kernel.shape[2].value if method == 'nearest_neighbor': new_kernels = nearest_neighbor(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(4 * 24000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'linear': new_kernels = linear(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(4 * 24000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'distance_weighting': new_kernels = distance_weighting(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(4 * 24000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'same': new_weights = layer.get_weights() if first_layer: new_layer = layers.Conv1D( nodes, kernel_size=layer.kernel.shape[0].value, activation=layer.activation, input_shape=(4 * 24000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D( nodes, kernel_size=layer.kernel.shape[0].value, activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'dilate': new_kernels = dilate_kernels(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(4 * 24000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'nearest_directly': new_kernels = nearest_directly(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(4 * 24000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'linear_directly': new_kernels = linear_directly(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(4 * 24000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'inverse_directly': new_kernels = inverse_directly(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(4 * 24000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif type(layer) is keras.layers.pooling.MaxPooling1D: pool_size = layer.pool_size[0] new_model.add(layers.MaxPooling1D(pool_size=pool_size)) elif type(layer) is keras.layers.pooling.AveragePooling1D: if avg_pool_unaffected is True: pool_size = layer.pool_size[0] new_model.add(layers.AveragePooling1D(pool_size=pool_size)) else: if method == 'dilate': new_kernels = scale_avg_pooling(nodes, [1 / 2, 0, 1 / 2, 0]) elif method == 'nearest_directly': new_kernels = scale_avg_pooling( nodes, [1 / 2, 1 / 2, 1 / 2, 1 / 2]) elif method == 'linear_directly': new_kernels = scale_avg_pooling( nodes, [1 / 2, 1 / 2, 1 / 2, 1 / 4]) elif method == 'inverse_directly': new_kernels = scale_avg_pooling( nodes, [1 / 2, 1 / 2, 1 / 2, 1 / 2]) dummy_bias = np.zeros(nodes) new_weights = [utils.get_weights(new_kernels), dummy_bias] new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[-1], activation='linear', padding='same', strides=2, weights=new_weights) new_model.add(new_layer) elif type(layer) is keras.layers.Flatten: f_dim = layer.input_shape new_model.add(layers.Flatten()) # if method != 'same': # new_model.add(layers.Flatten()) elif type(layer) is keras.layers.Dense: original_shape = layer.get_weights()[0].shape output_dim = layer.get_weights()[1].shape[0] shape = (f_dim[1], f_dim[2], output_dim) weights, biases = layer.get_weights() old_conv_weights = weights.reshape(shape) old_kernels = utils.get_kernels(old_conv_weights) if method == 'nearest_neighbor': new_kernels = nearest_neighbor(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] * 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) elif method == 'linear': new_kernels = linear(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape(original_shape[0] * 2, output_dim), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) elif method == 'distance_weighting': new_kernels = distance_weighting(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] * 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) elif method == 'same': new_kernels = np.concatenate((old_kernels, old_kernels), axis=2) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] * 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) # output_dim = layer.get_weights()[1].shape[0] # # shape = (f_dim[1], f_dim[2], output_dim) # new_weights = weights.reshape(shape) # new_layer = layers.Conv1D(output_dim, # f_dim[1], # strides=1, # activation=layer.activation, # padding='valid', # weights=[new_weights, biases]) # # new_model.add(new_layer) # # new_model.add(layers.Lambda(lambda x: K.batch_flatten(x))) elif method == 'dilate': new_kernels = dilate_kernels(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] * 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) elif method == 'nearest_directly': new_kernels = nearest_directly(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] * 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) elif method == 'linear_directly': new_kernels = linear_directly(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] * 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) elif method == 'inverse_directly': new_kernels = inverse_directly(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] * 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) return new_model
def downscale(method: str, old_model_name: str, avg_pool_unaffected=False): old_model = utils.load_model('Models/{}.yaml'.format(old_model_name), 'Models/{}.h5'.format(old_model_name)) new_model = models.Sequential() first_layer = True for layer in old_model.layers: if type(layer) is keras.layers.convolutional.Conv1D: biases = layer.get_weights()[1] old_kernels = utils.get_kernels(layer.get_weights()[0]) nodes = layer.kernel.shape[2].value if method == 'nearest_neighbor': new_kernels = nearest_neighbor(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(48000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'linear': new_kernels = linear(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(48000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'distance_weighting': new_kernels = distance_weighting(old_kernels) new_weights = [utils.get_weights(new_kernels), biases] if first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, input_shape=(48000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D(nodes, kernel_size=new_kernels.shape[2], activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif method == 'same': new_weights = layer.get_weights() if first_layer: new_layer = layers.Conv1D( nodes, kernel_size=layer.kernel.shape[0].value, activation=layer.activation, input_shape=(48000, 1), padding='same', weights=new_weights) first_layer = False elif not first_layer: new_layer = layers.Conv1D( nodes, kernel_size=layer.kernel.shape[0].value, activation=layer.activation, padding='same', weights=new_weights) new_model.add(new_layer) elif type(layer) is keras.layers.pooling.MaxPooling1D: pool_size = layer.pool_size[0] new_model.add(layers.MaxPooling1D(pool_size=pool_size)) elif type(layer) is keras.layers.pooling.AveragePooling1D: nodes = layer.get_output_at(0).shape[-1].value pool_size = layer.pool_size[0] if method == 'nearest_neighbor': new_model.add(layers.AveragePooling1D(pool_size=pool_size)) elif method == 'linear': if avg_pool_unaffected is True: new_model.add(layers.AveragePooling1D(pool_size=pool_size)) else: new_kernels = down_scale_avg_pooling( nodes, [3 / 2, -1 / 4, -1 / 4]) dummy_bias = np.zeros(nodes) new_weights = [utils.get_weights(new_kernels), dummy_bias] new_layer = layers.Conv1D( nodes, kernel_size=new_kernels.shape[-1], activation='linear', padding='same', strides=2, weights=new_weights) new_model.add(new_layer) elif method == 'distance_weighting': if avg_pool_unaffected is True: new_model.add(layers.AveragePooling1D(pool_size=pool_size)) else: new_kernels = down_scale_avg_pooling( nodes, [-1 / 4, 1 / 2, 3 / 4]) dummy_bias = np.zeros(nodes) new_weights = [utils.get_weights(new_kernels), dummy_bias] new_layer = layers.Conv1D( nodes, kernel_size=new_kernels.shape[-1], activation='linear', padding='same', strides=2, weights=new_weights) new_model.add(new_layer) elif method == 'same': new_model.add(layers.AveragePooling1D(pool_size=pool_size)) elif type(layer) is keras.layers.Flatten: new_model.add(layers.Flatten()) f_dim = layer.input_shape elif type(layer) is keras.layers.Dense: original_shape = layer.get_weights()[0].shape output_dim = layer.get_weights()[1].shape[0] shape = (f_dim[1], f_dim[2], output_dim) weights, biases = layer.get_weights() old_conv_weights = weights.reshape(shape) old_kernels = utils.get_kernels(old_conv_weights) if method == 'nearest_neighbor': new_kernels = nearest_neighbor(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] // 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) elif method == 'linear': new_kernels = linear(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape(original_shape[0] // 2, output_dim), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) elif method == 'distance_weighting': new_kernels = distance_weighting(old_kernels) new_kernels = pad_zeros(new_kernels, old_kernels.shape[-1]) new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] // 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) elif method == 'same': new_kernels = np.split(old_kernels, 2, axis=2)[0] new_conv_weights = utils.get_weights(new_kernels) new_dense_weights = [ new_conv_weights.reshape( (original_shape[0] // 2, output_dim)), biases ] new_model.add( layers.Dense(output_dim, activation=layer.activation, weights=new_dense_weights)) return new_model
def build_birnn_cnn_model(voca_dim, time_steps, output_dim, rnn_dim, mlp_dim, num_filters, filter_sizes, item_embedding=None, rnn_depth=1, mlp_depth=1, drop_out=0.5, rnn_drop_out=0.5, rnn_state_drop_out=0.5, cnn_drop_out=0.5, pooling='max', trainable_embedding=False, gpu=False, return_customized_layers=False): """ Create A Bidirectional CNN Model. :param voca_dim: vocabulary dimension size. :param time_steps: the length of input :param output_dim: the output dimension size :param rnn_dim: rrn dimension size :param num_filters: the number of filters :param filter_sizes: list of integers The kernel size. :param mlp_dim: the dimension size of fully connected layer :param item_embedding: integer, numpy 2D array, or None (default=None) If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor. If item_embedding is a matrix, this matrix will be used as the embedding matrix. If item_embedding is None, then connect input tensor to RNN layer directly. :param rnn_depth: rnn depth :param mlp_depth: the depth of fully connected layers :param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism :param drop_out: dropout rate of fully connected layers :param rnn_drop_out: dropout rate of rnn layers :param rnn_state_drop_out: dropout rate of rnn state tensor :param cnn_drop_out: dropout rate of between cnn layer and fully connected layers :param pooling: str, either 'max' or 'average' Pooling method. :param trainable_embedding: boolean :param gpu: boolean, default=False If True, CuDNNLSTM is used instead of LSTM for RNN layer. :param return_customized_layers: boolean, default=False If True, return model and customized object dictionary, otherwise return model only :return: keras model """ if item_embedding is not None: inputs = models.Input(shape=(time_steps, ), dtype='int32', name='input0') x = inputs # item embedding if isinstance(item_embedding, np.ndarray): assert voca_dim == item_embedding.shape[0] x = layers.Embedding(voca_dim, item_embedding.shape[1], input_length=time_steps, weights=[ item_embedding, ], trainable=trainable_embedding, mask_zero=False, name='embedding_layer0')(x) elif utils.is_integer(item_embedding): x = layers.Embedding(voca_dim, item_embedding, input_length=time_steps, trainable=trainable_embedding, mask_zero=False, name='embedding_layer0')(x) else: raise ValueError( "item_embedding must be either integer or numpy matrix") else: inputs = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='input0') x = inputs if gpu: # rnn encoding for i in range(rnn_depth): x = layers.Bidirectional(layers.CuDNNLSTM(rnn_dim, return_sequences=True), name='bi_lstm_layer' + str(i))(x) x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x) x = layers.Dropout(rnn_drop_out, name="rnn_dropout_layer" + str(i))(x) else: # rnn encoding for i in range(rnn_depth): x = layers.Bidirectional(layers.LSTM( rnn_dim, return_sequences=True, dropout=rnn_drop_out, recurrent_dropout=rnn_state_drop_out), name='bi_lstm_layer' + str(i))(x) x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x) pooled_outputs = [] for i in range(len(filter_sizes)): conv = layers.Conv1D(num_filters, kernel_size=filter_sizes[i], padding='valid', activation='relu')(x) if pooling == 'max': conv = layers.MaxPooling1D(pool_size=time_steps - filter_sizes[i] + 1, strides=1, padding='valid')(conv) else: conv = layers.AveragePooling1D(pool_size=time_steps - filter_sizes[i] + 1, strides=1, padding='valid')(conv) pooled_outputs.append(conv) x = layers.Concatenate(name='concated_layer')(pooled_outputs) x = layers.Flatten()(x) x = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x) x = layers.BatchNormalization(name="batch_norm_layer")(x) # MLP Layers for i in range(mlp_depth - 1): x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x) x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x) outputs = layers.Dense(output_dim, activation="softmax", name="softmax_layer0")(x) model = models.Model(inputs, outputs) if return_customized_layers: return model, dict() return model