def get_gnet(n_ch, patch_height, patch_width): inputs = Input((n_ch, patch_height, patch_width)) conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(inputs) conv1 = Dropout(0.2)(conv1) conv1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv1) up1 = UpSampling2D(size=(2, 2))(conv1) # conv2 = Convolution2D(16, 3, 3, activation='relu', border_mode='same')(up1) conv2 = Dropout(0.2)(conv2) conv2 = Convolution2D(16, 3, 3, activation='relu', border_mode='same')(conv2) pool1 = MaxPooling2D(pool_size=(2, 2))(conv2) # conv3 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(pool1) conv3 = Dropout(0.2)(conv3) conv3 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv3) pool2 = MaxPooling2D(pool_size=(2, 2))(conv3) # conv4 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(pool2) conv4 = Dropout(0.2)(conv4) conv4 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv4) pool3 = MaxPooling2D(pool_size=(2, 2))(conv4) # conv5 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(pool3) conv5 = Dropout(0.2)(conv5) conv5 = Convolution2D(128, 3, 3, activation='relu', border_mode='same')(conv5) # up2 = merge([UpSampling2D(size=(2, 2))(conv5), conv4], mode='concat', concat_axis=1) conv6 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(up2) conv6 = Dropout(0.2)(conv6) conv6 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')(conv6) # up3 = merge([UpSampling2D(size=(2, 2))(conv6), conv3], mode='concat', concat_axis=1) conv7 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(up3) conv7 = Dropout(0.2)(conv7) conv7 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv7) # up4 = merge([UpSampling2D(size=(2, 2))(conv7), conv2], mode='concat', concat_axis=1) conv8 = Convolution2D(16, 3, 3, activation='relu', border_mode='same')(up4) conv8 = Dropout(0.2)(conv8) conv8 = Convolution2D(16, 3, 3, activation='relu', border_mode='same')(conv8) # pool4 = MaxPooling2D(pool_size=(2, 2))(conv8) conv9 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(pool4) conv9 = Dropout(0.2)(conv9) conv9 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')(conv9) # conv10 = Convolution2D(2, 1, 1, activation='relu', border_mode='same')(conv9) conv10 = Reshape((2, patch_height * patch_width))(conv10) conv10 = Permute((2, 1))(conv10) ############ conv10 = Activation('softmax')(conv10) model = Model(input=inputs, output=conv10) # sgd = SGD(lr=0.01, decay=1e-6, momentum=0.3, nesterov=False) model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy']) return model
def HRNetResidual(input_shape=(128, 128, 3), num_keypoints=20): # stem inputs = Input(shape=input_shape, name='image') x1 = stem(inputs, 64) x1 = Conv2D(64 * 4, 1, padding='same', use_bias=False)(x1) x1 = BatchNormalization()(x1) for block in range(4): x1 = bottleneck(x1) # stage I x1 = Conv2D(32, 3, padding='same', use_bias=False)(x1) x1 = BatchNormalization()(x1) x1 = Activation('relu')(x1) x2 = transition_block(x1, 2) # stage II for block in range(4): x1 = residual_block(x1, 32) x2 = residual_block(x2, 64) x1, x2 = fuse([x1, x2]) x3 = transition_block(x2, 2) # stage III for module in range(4): for block in range(4): x1 = residual_block(x1, 32) x2 = residual_block(x2, 64) x3 = residual_block(x3, 128) x1, x2, x3 = fuse([x1, x2, x3]) x4 = transition_block(x3, 2) # stage IV for module in range(3): for block in range(4): x1 = residual_block(x1, 32) x2 = residual_block(x2, 64) x3 = residual_block(x3, 128) x4 = residual_block(x4, 256) x1, x2, x3, x4 = fuse([x1, x2, x3, x4]) # head x2 = UpSampling2D(size=(2, 2))(x2) x3 = UpSampling2D(size=(4, 4))(x3) x4 = UpSampling2D(size=(8, 8))(x4) x = Concatenate()([x1, x2, x3, x4]) x = Conv2D(480, 1)(x) x = BatchNormalization(epsilon=1.001e-5)(x) x = Activation('relu')(x) x = Conv2D(num_keypoints, 1)(x) # extra x = BatchNormalization(epsilon=1.001e-5)(x) x = Activation('relu')(x) x = UpSampling2D(size=(4, 4), interpolation='bilinear')(x) x = Permute([3, 1, 2])(x) x = Reshape([num_keypoints, input_shape[0] * input_shape[1]])(x) x = Activation('softmax')(x) x = Reshape([num_keypoints, input_shape[0], input_shape[1]])(x) outputs = ExpectedValue2D(name='keypoints')(x) model = Model(inputs, outputs, name='hrnet-residual') return model
def forward(self, x): x = Permute((2, 1, 3))(x) x = self.pad(x) return Permute((2, 1, 3))(x)
def __init__(self, sr=44100): """ Build the CNN model and load the weights Parameters ---------- model_capacity : 'tiny', 'small', 'medium', 'large', or 'full' String specifying the model capacity, which determines the model's capacity multiplier to 4 (tiny), 8 (small), 16 (medium), 24 (large), or 32 (full). 'full' uses the model size specified in the paper, and the others use a reduced number of filters in each convolutional layer, resulting in a smaller model that is faster to evaluate at the cost of slightly reduced pitch estimation accuracy. Returns ------- model : tensorflow.keras.models.Model The pre-trained keras model loaded in memory """ from tensorflow.keras.layers import Input, Reshape, Conv2D, BatchNormalization from tensorflow.keras.layers import MaxPool2D, Dropout, Permute, Flatten, Dense from tensorflow.keras.models import Model models = { 'tiny': None, 'small': None, 'medium': None, 'large': None, 'full': None } # the model is trained on 16kHz audio #model_srate = 16000 self.sr = sr #if models[model_capacity] is None: capacity_multiplier = 32 #{ # 'tiny': 4, 'small': 8, 'medium': 16, 'large': 24, 'full': 32 # }[model_capacity] layers = [1, 2, 3, 4, 5, 6] filters = [n * capacity_multiplier for n in [32, 4, 4, 4, 8, 16]] widths = [512, 64, 64, 64, 64, 64] strides = [(4, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)] x = Input(shape=(1024,), name='input', dtype='float32') y = Reshape(target_shape=(1024, 1, 1), name='input-reshape')(x) for l, f, w, s in zip(layers, filters, widths, strides): y = Conv2D(f, (w, 1), strides=s, padding='same', activation='relu', name="conv%d" % l)(y) y = BatchNormalization(name="conv%d-BN" % l)(y) y = MaxPool2D(pool_size=(2, 1), strides=None, padding='valid', name="conv%d-maxpool" % l)(y) y = Dropout(0.25, name="conv%d-dropout" % l)(y) y = Permute((2, 1, 3), name="transpose")(y) y = Flatten(name="flatten")(y) y = Dense(360, activation='sigmoid', name="classifier")(y) self.model = Model(inputs=x, outputs=y) filename = "model-full.h5" self.model.load_weights(filename) self.model.compile('adam', 'binary_crossentropy')
def build_model(self): """Helper method for creating the model""" vocab = set() for story, q, answer in self.train_stories + self.test_stories: vocab |= set(story + q + [answer]) vocab = sorted(vocab) # Reserve 0 for masking via pad_sequences vocab_size = len(vocab) + 1 story_maxlen = max( len(x) for x, _, _ in self.train_stories + self.test_stories) query_maxlen = max( len(x) for _, x, _ in self.train_stories + self.test_stories) word_idx = {c: i + 1 for i, c in enumerate(vocab)} self.inputs_train, self.queries_train, self.answers_train = vectorize_stories( word_idx, story_maxlen, query_maxlen, self.train_stories) self.inputs_test, self.queries_test, self.answers_test = vectorize_stories( word_idx, story_maxlen, query_maxlen, self.test_stories) # placeholders input_sequence = Input((story_maxlen, )) question = Input((query_maxlen, )) # encoders # embed the input sequence into a sequence of vectors input_encoder_m = Sequential() input_encoder_m.add(Embedding(input_dim=vocab_size, output_dim=64)) input_encoder_m.add(Dropout(self.config.get("dropout", 0.3))) # output: (samples, story_maxlen, embedding_dim) # embed the input into a sequence of vectors of size query_maxlen input_encoder_c = Sequential() input_encoder_c.add( Embedding(input_dim=vocab_size, output_dim=query_maxlen)) input_encoder_c.add(Dropout(self.config.get("dropout", 0.3))) # output: (samples, story_maxlen, query_maxlen) # embed the question into a sequence of vectors question_encoder = Sequential() question_encoder.add( Embedding(input_dim=vocab_size, output_dim=64, input_length=query_maxlen)) question_encoder.add(Dropout(self.config.get("dropout", 0.3))) # output: (samples, query_maxlen, embedding_dim) # encode input sequence and questions (which are indices) # to sequences of dense vectors input_encoded_m = input_encoder_m(input_sequence) input_encoded_c = input_encoder_c(input_sequence) question_encoded = question_encoder(question) # compute a "match" between the first input vector sequence # and the question vector sequence # shape: `(samples, story_maxlen, query_maxlen)` match = dot([input_encoded_m, question_encoded], axes=(2, 2)) match = Activation("softmax")(match) # add the match matrix with the second input vector sequence response = add([match, input_encoded_c ]) # (samples, story_maxlen, query_maxlen) response = Permute( (2, 1))(response) # (samples, query_maxlen, story_maxlen) # concatenate the match matrix with the question vector sequence answer = concatenate([response, question_encoded]) # the original paper uses a matrix multiplication. # we choose to use a RNN instead. answer = LSTM(32)(answer) # (samples, 32) # one regularization layer -- more would probably be needed. answer = Dropout(self.config.get("dropout", 0.3))(answer) answer = Dense(vocab_size)(answer) # (samples, vocab_size) # we output a probability distribution over the vocabulary answer = Activation("softmax")(answer) # build the final model model = Model([input_sequence, question], answer) return model
def STSnet(n_clss, ch_rows=4, ch_cols=5, samples=128, kernLength=None, F1=4, D=2, F2=8, norm_rate=0.25, dropoutRate=0.25, dropoutType='Dropout'): """Keras Implementation of STSnet. # Arguments: n_clss: int, number of classes to clasify (default 4) ch_rows: number of rows of 2D spatial distribution (default 4) ch_cols: number of columns of 2D spatial distribution (default 5) samples: number of epoch samples form each trial (default 128) dropoutRate: dropout value (default 0.25) kernLength: kernel length value in Conv2D layer(default None) F1: number of filters in Conv2D layer (default 4) D: deep multiplier to DepthwiseConv2D (default 2) F2: number of filters to each kernel from SeparableConv2D (default 8) norm_rate: maximum norm for the incoming dense weights(default 0.25) dropoutType: 'SpatialDropout2D' or 'Dropout' (defautl 'Dropout') """ if dropoutType == 'SpatialDropout2D': dropoutType = SpatialDropout2D elif dropoutType == 'Dropout': dropoutType = Dropout else: raise ValueError('dropoutType must be one of SpatialDropout2D ' 'or Dropout, passed as a string.') if kernLength is None: kernLength = np.int(samples/2) print('kernel_length', kernLength) # ------------------------------------------------------------------------- # DEPTWISE 2D MODEL DW_in = Input(shape=(F1, ch_rows, ch_cols)) DW_out = DepthwiseConv2D((ch_rows, ch_cols), use_bias=False, depth_multiplier=D, depthwise_constraint=max_norm(1.))(DW_in) DW = Model(inputs=DW_in, outputs=DW_out) # ------------------------------------------------------------------------- input1 = Input(shape=(ch_rows * ch_cols, samples)) # ------------------------------------------------------------------------- reshape1 = Reshape(target_shape=(1, ch_rows * ch_cols, samples))(input1) block1 = Conv2D(F1, (1, kernLength), padding='same', input_shape=(1, ch_rows*ch_cols, samples), use_bias=False)(reshape1) block1 = BatchNormalization(axis=1)(block1) reshape2 = Reshape(target_shape=(F1, ch_rows, ch_cols, samples))(block1) perm_dims1 = 4, 1, 2, 3 perm1 = Permute(perm_dims1)(reshape2) block1 = TimeDistributed(DW)(perm1) perm_dims2 = 2, 3, 4, 1 perm2 = Permute(perm_dims2)(block1) reshape3 = Reshape(target_shape=(F1*D, 1, samples))(perm2) block1 = BatchNormalization(axis=1)(reshape3) block1 = Activation('elu')(block1) block1 = AveragePooling2D((1, np.int(kernLength/16)))(block1) block1 = dropoutType(dropoutRate)(block1) block2 = SeparableConv2D(F2, (1, 16), use_bias=False, padding='same')(block1) block2 = BatchNormalization(axis=1)(block2) block2 = Activation('elu')(block2) block2 = AveragePooling2D((1, np.int(kernLength/8)))(block2) block2 = dropoutType(dropoutRate)(block2) flatten = Flatten(name='flatten')(block2) dense = Dense(n_clss, name='dense', kernel_constraint=max_norm(norm_rate))(flatten) softmax = Activation('softmax', name='softmax')(dense) return Model(inputs=input1, outputs=softmax)
def _crnn_layers(self, input_shape, output_shape): channel_axis = 3 melgram_input = Input(shape=input_shape, dtype="float32") # Input block padding = self.network_input_width - input_shape[1] left_pad = int(padding / 2) if padding % 2: right_pad = left_pad + 1 else: right_pad = left_pad input_padding = ((0, 0), (left_pad, right_pad)) hidden = ZeroPadding2D(padding=input_padding)(melgram_input) # Conv block 1 hidden = Conv2D(64, (3, 3), padding=self.padding, name='conv1')(hidden) hidden = BatchNormalization(axis=channel_axis, name='bn1')(hidden) hidden = ELU()(hidden) hidden = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(hidden) hidden = Dropout(0.1, name='dropout1')(hidden) # Conv block 2 hidden = Conv2D(128, (3, 3), padding=self.padding, name='conv2')(hidden) hidden = BatchNormalization(axis=channel_axis, name='bn2')(hidden) hidden = ELU()(hidden) hidden = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(hidden) hidden = Dropout(0.1, name='dropout2')(hidden) # Conv block 3 hidden = Conv2D(128, (3, 3), padding=self.padding, name='conv3')(hidden) hidden = BatchNormalization(axis=channel_axis, name='bn3')(hidden) hidden = ELU()(hidden) hidden = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(hidden) hidden = Dropout(0.1, name='dropout3')(hidden) # Conv block 4 hidden = Conv2D(128, (3, 3), padding=self.padding, name='conv4')(hidden) hidden = BatchNormalization(axis=channel_axis, name='bn4')(hidden) hidden = ELU()(hidden) hidden = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(hidden) hidden = Dropout(0.1, name='dropout4')(hidden) # reshaping hidden = Reshape((15, 128))(hidden) # GRU block 1, 2, output embed_size = 32 hidden = GRU(embed_size, return_sequences=True, name='gru1')(hidden) hidden = GRU(embed_size, return_sequences=self.attention, name='gru2')(hidden) if self.attention: attention = Dense(1)(hidden) attention = Flatten()(attention) attention_act = Activation("softmax")(attention) attention = RepeatVector(embed_size)(attention_act) attention = Permute((2, 1))(attention) merged = Multiply()([hidden, attention]) hidden = Lambda(lambda xin: K.sum(xin, axis=1))(merged) if self.output_dropout: hidden = Dropout(self.output_dropout)(hidden) output = Dense(output_shape, activation='sigmoid', name='crnn_output')(hidden) return melgram_input, output
def bottleneck_encoder(self, tensor, nfilters, downsampling=False, dilated=False, asymmetric=False, normal=False, drate=0.1, name=''): """ Encoder :param tensor: input tensor :param nfilters: Number of filters :param downsampling: Downsample the feature map :param dilated: determines if ther should be dilated convultion :param asymmetric: Determines if there should be asymmetric convolution :param normal: enables 3x3 convolution on feature map :param drate: rate of dilation :param name: the name for the weight variable. :return: encoder output """ y = tensor skip = tensor stride = 1 ksize = 1 # Filters operating on downsampled images have a bigger receptive field and hence gathers more context. if downsampling: stride = 2 ksize = 2 skip = MaxPooling2D(pool_size=(2, 2), name=f'max_pool_{name}')(skip) skip = Permute((1, 3, 2), name=f'permute_1_{name}')(skip) # (B, H, W, C) -> (B, H, C, W) ch_pad = nfilters - tf.compat.v2.keras.backend.int_shape(tensor)[-1] skip = ZeroPadding2D(padding=((0, 0), (0, ch_pad)), name=f'zeropadding_{name}')(skip) skip = Permute((1, 3, 2), name=f'permute_2_{name}')(skip) # (B, H, C, W) -> (B, H, W, C) y = Conv2D(filters=nfilters // 4, kernel_size=(ksize, ksize), kernel_initializer='he_normal', strides=(stride, stride), padding='same', use_bias=False, name=f'1x1_conv_{name}')(y) y = BatchNormalization(momentum=0.1, name=f'bn_1x1_{name}')(y) y = PReLU(shared_axes=[1, 2], name=f'prelu_1x1_{name}')(y) if normal: # deconv with 3x3 filter y = Conv2D(filters=nfilters // 4, kernel_size=(3, 3), kernel_initializer='he_normal', padding='same', name=f'3x3_conv_{name}')(y) elif asymmetric: # decompose 5x5 convolution to two asymmetric layers as 5x1 and 1x5 y = Conv2D(filters=nfilters // 4, kernel_size=(5, 1), kernel_initializer='he_normal', padding='same', use_bias=False, name=f'5x1_conv_{name}')(y) y = Conv2D(filters=nfilters // 4, kernel_size=(1, 5), kernel_initializer='he_normal', padding='same', name=f'1x5_conv_{name}')(y) elif dilated: y = Conv2D(filters=nfilters // 4, kernel_size=(3, 3), kernel_initializer='he_normal', dilation_rate=(dilated, dilated), padding='same', name=f'dilated_conv_{name}')(y) y = BatchNormalization(momentum=0.1, name=f'bn_main_{name}')(y) y = PReLU(shared_axes=[1, 2], name=f'prelu_{name}')(y) y = Conv2D(filters=nfilters, kernel_size=(1, 1), kernel_initializer='he_normal', use_bias=False, name=f'final_1x1_{name}')(y) y = BatchNormalization(momentum=0.1, name=f'bn_final_{name}')(y) y = SpatialDropout2D(rate=drate, name=f'spatial_dropout_final_{name}')(y) y = Add(name=f'add_{name}')([y, skip]) y = PReLU(shared_axes=[1, 2], name=f'prelu_out_{name}')(y) return y
def HRNetResidual(input_shape=(128, 128, 3), num_keypoints=20): """Instantiates HRNET Residual model # Arguments input_shape: List of three elements e.g. ''(H, W, 3)'' num_keypoints: Int. # Returns Tensorflow-Keras model. # References -[High-Resolution Representations for Labeling Pixels and Regions](https://arxiv.org/pdf/1904.04514.pdf) """ # stem inputs = Input(shape=input_shape, name='image') x1 = stem(inputs, 64) x1 = Conv2D(64 * 4, 1, padding='same', use_bias=False)(x1) x1 = BatchNormalization()(x1) for block in range(4): x1 = bottleneck(x1) # stage I x1 = Conv2D(32, 3, padding='same', use_bias=False)(x1) x1 = BatchNormalization()(x1) x1 = Activation('relu')(x1) x2 = transition_block(x1, 2) # stage II for block in range(4): x1 = residual_block(x1, 32) x2 = residual_block(x2, 64) x1, x2 = fuse([x1, x2]) x3 = transition_block(x2, 2) # stage III for module in range(4): for block in range(4): x1 = residual_block(x1, 32) x2 = residual_block(x2, 64) x3 = residual_block(x3, 128) x1, x2, x3 = fuse([x1, x2, x3]) x4 = transition_block(x3, 2) # stage IV for module in range(3): for block in range(4): x1 = residual_block(x1, 32) x2 = residual_block(x2, 64) x3 = residual_block(x3, 128) x4 = residual_block(x4, 256) x1, x2, x3, x4 = fuse([x1, x2, x3, x4]) # head x2 = UpSampling2D(size=(2, 2))(x2) x3 = UpSampling2D(size=(4, 4))(x3) x4 = UpSampling2D(size=(8, 8))(x4) x = Concatenate()([x1, x2, x3, x4]) x = Conv2D(480, 1)(x) x = BatchNormalization(epsilon=1.001e-5)(x) x = Activation('relu')(x) x = Conv2D(num_keypoints, 1)(x) # extra x = BatchNormalization(epsilon=1.001e-5)(x) x = Activation('relu')(x) x = UpSampling2D(size=(4, 4), interpolation='bilinear')(x) x = Permute([3, 1, 2])(x) x = Reshape([num_keypoints, input_shape[0] * input_shape[1]])(x) x = Activation('softmax')(x) x = Reshape([num_keypoints, input_shape[0], input_shape[1]])(x) outputs = ExpectedValue2D(name='keypoints')(x) model = Model(inputs, outputs, name='hrnet-residual') return model
def interest_evolution( concat_behavior, deep_input_item, user_behavior_length, gru_type="GRU", use_neg=False, neg_concat_behavior=None, embedding_size=8, att_hidden_size=(64, 16), att_activation='sigmoid', att_weight_normalization=False, ): if gru_type not in ["GRU", "AIGRU", "AGRU", "AUGRU"]: raise ValueError("gru_type error ") aux_loss_1 = None rnn_outputs = DynamicGRU(embedding_size * 2, return_sequence=True, name="gru1")( [concat_behavior, user_behavior_length]) if gru_type == "AUGRU" and use_neg: aux_loss_1 = auxiliary_loss(rnn_outputs[:, :-1, :], concat_behavior[:, 1:, :], neg_concat_behavior[:, 1:, :], tf.subtract(user_behavior_length, 1), stag="gru") # [:, 1:] if gru_type == "GRU": rnn_outputs2 = DynamicGRU(embedding_size * 2, return_sequence=True, name="gru2")( [rnn_outputs, user_behavior_length]) # attention_score = AttentionSequencePoolingLayer(hidden_size=att_hidden_size, activation=att_activation, weight_normalization=att_weight_normalization, return_score=True)([ # deep_input_item, rnn_outputs2, user_behavior_length]) # outputs = Lambda(lambda x: tf.matmul(x[0], x[1]))( # [attention_score, rnn_outputs2]) # hist = outputs hist = AttentionSequencePoolingLayer( att_hidden_units=att_hidden_size, att_activation=att_activation, weight_normalization=att_weight_normalization, return_score=False)( [deep_input_item, rnn_outputs2, user_behavior_length]) else: # AIGRU AGRU AUGRU scores = AttentionSequencePoolingLayer( att_hidden_units=att_hidden_size, att_activation=att_activation, weight_normalization=att_weight_normalization, return_score=True)( [deep_input_item, rnn_outputs, user_behavior_length]) if gru_type == "AIGRU": hist = multiply([rnn_outputs, Permute([2, 1])(scores)]) final_state2 = DynamicGRU( embedding_size * 2, gru_type="GRU", return_sequence=False, name='gru2')([hist, user_behavior_length]) else: # AGRU AUGRU final_state2 = DynamicGRU(embedding_size * 2, gru_type=gru_type, return_sequence=False, name='gru2')([ rnn_outputs, user_behavior_length, Permute([2, 1])(scores) ]) hist = final_state2 return hist, aux_loss_1
def build_net(self, dev): with tf.variable_scope('a3c') and tf.device(dev): screenInput = Input( shape=(U.screen_channel(), self.ssize, self.ssize), name='screenInput', ) permutedScreenInput = Permute((2,3,1))(screenInput) conv1 = Conv2D(16, kernel_size=5, strides=(1,1), padding='same',name='conv1')(permutedScreenInput) conv2 = Conv2D(32, kernel_size=3, strides=(1,1), padding='same',name='conv2')(conv1) infoInput = Input( shape=(self.isize,), name='infoInput', ) customInput = Input( shape=(self.custom_input_size,), name='customInput', ) nonSpatialInput = Concatenate(name='nonSpatialInputConcat')([infoInput, customInput]) broadcasted = Lambda(self.broadcast,name='broadcasting')(nonSpatialInput) combinedSpatialNonSpatial = Concatenate(name='combinedConcat')([broadcasted, conv2]) conv3 = Conv2D(1, kernel_size=1, strides=(1,1), padding='same',name='conv3')(combinedSpatialNonSpatial) flatConv3 = Flatten(name='flatConv3')(conv3) lstmInput = Lambda(self.expand_dims, name='lstmInput')(flatConv3) self.NUM_LSTM = 100 hStateInput = Input( shape=(self.NUM_LSTM,), name='hStateInput' ) cStateInput = Input( shape=(self.NUM_LSTM,), name='cStateInput' ) lstm, hStates, cStates = LSTM(self.NUM_LSTM, return_state=True)(lstmInput, initial_state=[hStateInput, cStateInput]) fc1 = Dense(256, activation='relu',name='dense1')(lstm) fc2 = Dense(1, activation='linear',name='fc2')(fc1) value = Lambda(self.Squeeze,name='value')(fc2) policy = Dense(self.isize, activation='softmax',name='policy')(fc1) broadcastLstm = Lambda(self.broadcast, name='breadcastLstm')(lstm) spatialLstm = Concatenate(name='spatialLstm')([conv3, broadcastLstm]) conv4 = Conv2D(1,kernel_size=1, strides=(1,1), padding='same',name='conv4')(spatialLstm) flatConv4 = Flatten(name='flattenedConv3')(conv4) spatialPolicy = Softmax(name='spatialPolicy')(flatConv4) conv5 = Conv2D(1, kernel_size=1, strides=(1,1), padding='same',name='conv5')(spatialLstm) flatConv5 = Flatten(name='flattenedConv5')(conv5) bestRoach = Softmax(name='bestRoach')(flatConv5) self.model = Model( inputs=[screenInput, infoInput, customInput, hStateInput, cStateInput], outputs=[value, policy, spatialPolicy, hStates, cStates, bestRoach] ) self.model._make_predict_function()
# encode input sequence and questions (which are indices) # to sequences of dense vectors input_encoded_m = input_encoder_m(input_sequence) input_encoded_c = input_encoder_c(input_sequence) question_encoded = question_encoder(question) # compute a 'match' between the first input vector sequence # and the question vector sequence # shape: `(samples, story_maxlen, query_maxlen)` match = dot([input_encoded_m, question_encoded], axes=(2, 2)) match = Activation('softmax')(match) # add the match matrix with the second input vector sequence response = add([match, input_encoded_c]) # (samples, story_maxlen, query_maxlen) response = Permute((2, 1))(response) # (samples, query_maxlen, story_maxlen) # concatenate the match matrix with the question vector sequence answer = concatenate([response, question_encoded]) # the original paper uses a matrix multiplication for this reduction step. # we choose to use a RNN instead. answer = LSTM(32)(answer) # (samples, 32) # one regularization layer -- more would probably be needed. answer = Dropout(0.3)(answer) answer = Dense(vocab_size)(answer) # (samples, vocab_size) # we output a probability distribution over the vocabulary answer = Activation('softmax')(answer) # build the final model
def cnn_line_lstm_ctc(input_shape, output_shape, **kwargs): image_height, image_width = input_shape output_length, num_classes = output_shape image_input = Input(shape=input_shape, name='image') y_true = Input(shape=(output_length, ), name='y_true') input_length = Input(shape=(1, ), name='input_length') label_length = Input(shape=(1, ), name='label_length') gpu_present = len(device_lib.list_local_devices()) > 1 lstm_fn = CuDNNLSTM if gpu_present else LSTM ##### Your code below (Lab 3) image_reshaped = Reshape((image_height, image_width, 1))(image_input) # (image_height, image_width, 1) convnet_outputs = image_reshaped # convnet_outputs = Dropout(0.5)(convnet_outputs) convnet_outputs = Conv2D(16, 3, padding='SAME')(convnet_outputs) convnet_outputs = BatchNormalization()(convnet_outputs) convnet_outputs = LeakyReLU()(convnet_outputs) convnet_outputs = MaxPooling2D(2, 2)(convnet_outputs) # convnet_outputs = Dropout(0.5)(convnet_outputs) convnet_outputs = Conv2D(32, 3, padding='SAME')(convnet_outputs) convnet_outputs = BatchNormalization()(convnet_outputs) convnet_outputs = LeakyReLU()(convnet_outputs) convnet_outputs = MaxPooling2D(2, 2)(convnet_outputs) convnet_outputs = Dropout(0.2)(convnet_outputs) convnet_outputs = Conv2D(48, 3, padding='SAME')(convnet_outputs) convnet_outputs = BatchNormalization()(convnet_outputs) convnet_outputs = LeakyReLU()(convnet_outputs) convnet_outputs = MaxPooling2D(2, 2)(convnet_outputs) convnet_outputs = Dropout(0.2)(convnet_outputs) convnet_outputs = Conv2D(64, 3, padding='SAME')(convnet_outputs) convnet_outputs = BatchNormalization()(convnet_outputs) convnet_outputs = LeakyReLU()(convnet_outputs) convnet_outputs = Dropout(0.2)(convnet_outputs) convnet_outputs = Conv2D(80, 3, padding='SAME')(convnet_outputs) convnet_outputs = BatchNormalization()(convnet_outputs) convnet_outputs = LeakyReLU()(convnet_outputs) num_windows = 119 convnet_outputs = Permute([2, 1, 3])(convnet_outputs) convnet_outputs = Reshape([num_windows, 240])(convnet_outputs) # (num_windows, 128) for i in range(5): convnet_outputs = Dropout(0.5)(convnet_outputs) lstm_output = Bidirectional(lstm_fn( 256, return_sequences=True))(convnet_outputs) lstm_output = Dropout(0.5)(lstm_output) softmax_output = Dense(num_classes, activation='softmax', name='softmax_output')(lstm_output) # (num_windows, num_classes) ##### Your code above (Lab 3) input_length_processed = Lambda( lambda x, num_windows=None: x * num_windows, arguments={'num_windows': num_windows})(input_length) ctc_loss_output = Lambda( lambda x: K.ctc_batch_cost(x[0], x[1], x[2], x[3]), name='ctc_loss')( [y_true, softmax_output, input_length_processed, label_length]) ctc_decoded_output = Lambda( lambda x: ctc_decode(x[0], x[1], output_length), name='ctc_decoded')([softmax_output, input_length_processed]) model = KerasModel( inputs=[image_input, y_true, input_length, label_length], outputs=[ctc_loss_output, ctc_decoded_output]) return model
def get_unet(n_ch, patch_height, patch_width): inputs = Input(shape=(n_ch, patch_height, patch_width)) conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', data_format='channels_first')(inputs) conv1 = Dropout(0.2)(conv1) conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', data_format='channels_first')(conv1) pool1 = MaxPooling2D((2, 2))(conv1) # conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', data_format='channels_first')(pool1) conv2 = Dropout(0.2)(conv2) conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', data_format='channels_first')(conv2) pool2 = MaxPooling2D((2, 2))(conv2) # conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', data_format='channels_first')(pool2) conv3 = Dropout(0.2)(conv3) conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', data_format='channels_first')(conv3) up1 = UpSampling2D(size=(2, 2))(conv3) up1 = concatenate([conv2, up1], axis=1) conv4 = Conv2D(64, (3, 3), activation='relu', padding='same', data_format='channels_first')(up1) conv4 = Dropout(0.2)(conv4) conv4 = Conv2D(64, (3, 3), activation='relu', padding='same', data_format='channels_first')(conv4) # up2 = UpSampling2D(size=(2, 2))(conv4) up2 = concatenate([conv1, up2], axis=1) conv5 = Conv2D(32, (3, 3), activation='relu', padding='same', data_format='channels_first')(up2) conv5 = Dropout(0.2)(conv5) conv5 = Conv2D(32, (3, 3), activation='relu', padding='same', data_format='channels_first')(conv5) # conv6 = Conv2D(2, (1, 1), activation='relu', padding='same', data_format='channels_first')(conv5) conv6 = Reshape((2, patch_height * patch_width))(conv6) conv6 = Permute((2, 1))(conv6) ############ conv7 = Activation('softmax')(conv6) model = Model(inputs=inputs, outputs=conv7) # sgd = SGD(lr=0.01, decay=1e-6, momentum=0.3, nesterov=False) model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy']) return model
def create_network(n_commands, n_channel, n_value1, n_value2, n_durations, embed_size=100, rnn_units=256, use_attention=False): """ create the structure of the neural network """ commands_in = Input(shape=(None, ), name="commands_in") channel_in = Input(shape=(None, ), name="channels_in") value1_in = Input(shape=(None, ), name="values1_in") value2_in = Input(shape=(None, ), name="values2_in") durations_in = Input(shape=(None, ), name="durations_in") x1 = Embedding(n_commands, embed_size)(commands_in) x2 = Embedding(n_channel, embed_size)(channel_in) x3 = Embedding(n_value1, embed_size)(value1_in) x4 = Embedding(n_value2, embed_size)(value2_in) x5 = Embedding(n_durations, embed_size)(durations_in) x = Concatenate()([x1, x2, x3, x4, x5]) x = LSTM(rnn_units, return_sequences=True)(x) # x = Dropout(0.2)(x) if use_attention: x = LSTM(rnn_units, return_sequences=True)(x) # x = Dropout(0.2)(x) e = Dense(1, activation='tanh')(x) e = Reshape([-1])(e) alpha = Activation('softmax')(e) alpha_repeated = Permute([2, 1])(RepeatVector(rnn_units)(alpha)) c = Multiply()([x, alpha_repeated]) c = Lambda(lambda xin: K.sum(xin, axis=1), output_shape=(rnn_units, ))(c) else: c = LSTM(rnn_units)(x) # c = Dropout(0.2)(c) commands_out = Dense(n_commands, activation='softmax', name='commands_out')(c) channel_out = Dense(n_channel, activation='softmax', name='channels_out')(c) value1_out = Dense(n_value1, activation='softmax', name='values1_out')(c) value2_out = Dense(n_value2, activation='softmax', name='values2_out')(c) durations_out = Dense(n_durations, activation='softmax', name='durations_out')(c) model = Model( [commands_in, channel_in, value1_in, value2_in, durations_in], [commands_out, channel_out, value1_out, value2_out, durations_out]) if use_attention: att_model = Model( [commands_in, channel_in, value1_in, value2_in, durations_in], alpha) else: att_model = None opti = RMSprop(lr=0.001) model.compile(loss=[ 'categorical_crossentropy', 'categorical_crossentropy', 'categorical_crossentropy', 'categorical_crossentropy', 'categorical_crossentropy' ], optimizer=opti) return model, att_model
def EEGNet_old(nb_classes, Chans=64, Samples=128, regRate=0.0001, dropoutRate=0.25, kernels=[(2, 32), (8, 4)], strides=(2, 4)): """ Keras Implementation of EEGNet_v1 (https://arxiv.org/abs/1611.08024v2) This model is the original EEGNet model proposed on arxiv https://arxiv.org/abs/1611.08024v2 with a few modifications: we use striding instead of max-pooling as this helped slightly in classification performance while also providing a computational speed-up. Note that we no longer recommend the use of this architecture, as the new version of EEGNet performs much better overall and has nicer properties. Inputs: nb_classes : total number of final categories Chans, Samples : number of EEG channels and samples, respectively regRate : regularization rate for L1 and L2 regularizations dropoutRate : dropout fraction kernels : the 2nd and 3rd layer kernel dimensions (default is the [2, 32] x [8, 4] configuration) strides : the stride size (note that this replaces the max-pool used in the original paper) """ # start the model input_main = Input((Chans, Samples)) layer1 = Conv2D(16, (Chans, 1), input_shape=(Chans, Samples, 1), kernel_regularizer=l1_l2(l1=regRate, l2=regRate))(input_main) layer1 = BatchNormalization()(layer1) layer1 = Activation('elu')(layer1) layer1 = Dropout(dropoutRate)(layer1) permute_dims = 2, 1, 3 permute1 = Permute(permute_dims)(layer1) layer2 = Conv2D(4, kernels[0], padding='same', kernel_regularizer=l1_l2(l1=0.0, l2=regRate), strides=strides)(permute1) layer2 = BatchNormalization()(layer2) layer2 = Activation('elu')(layer2) layer2 = Dropout(dropoutRate)(layer2) layer3 = Conv2D(4, kernels[1], padding='same', kernel_regularizer=l1_l2(l1=0.0, l2=regRate), strides=strides)(layer2) layer3 = BatchNormalization()(layer3) layer3 = Activation('elu')(layer3) layer3 = Dropout(dropoutRate)(layer3) flatten = Flatten(name='flatten')(layer3) dense = Dense(nb_classes, name='dense')(flatten) softmax = Activation('softmax', name='softmax')(dense) return Model(inputs=input_main, outputs=softmax)
def est_net(lr, Nt, net, mode): # Second part, channel estimator x0 = Input(shape=(R, 2)) if net == 'CNN': # change the order x1 = Permute((2, 1))(x0) x1 = Flatten()(x1) x1 = Reshape((R, 2))(x1) x1 = Conv1D(filters=96, kernel_size=7, padding='same')(x1) x1 = BatchNormalization()(x1) x1 = Activation('relu')(x1) if mode == 'naive': x1 = naive_attention(x1) if mode == 'feature': x1 = feature_attention(x1, x0) x2 = Conv1D(filters=96, kernel_size=5, padding='same')(x1) x2 = BatchNormalization()(x2) x2 = Activation('relu')(x2) if mode == 'naive': x2 = naive_attention(x2) if mode == 'feature': x2 = feature_attention(x2, x0) x3 = Flatten()(x2) prediction = Dense(2 * Nt, activation='linear')(x3) if net == 'CNN2': x1 = Conv1D(filters=96, kernel_size=7, padding='same')(x0) x1 = BatchNormalization()(x1) x1 = Activation('relu')(x1) x2 = Conv1D(filters=96, kernel_size=7, padding='same')(x1) x2 = BatchNormalization()(x2) x2 = Activation('relu')(x2) x2 = Conv1D(filters=96, kernel_size=5, padding='same')(x2) x2 = BatchNormalization()(x2) x2 = Activation('relu')(x2) x2 = Conv1D(filters=96, kernel_size=5, padding='same')(x2) x2 = BatchNormalization()(x2) x2 = Activation('relu')(x2) x2 = Conv1D(filters=96, kernel_size=3, padding='same')(x2) x2 = BatchNormalization()(x2) x2 = Activation('relu')(x2) x3 = Flatten()(x2) prediction = Dense(2 * Nt, activation='linear')(x3) if net == 'CNN3': x1 = Flatten()(x0) #x1 = Dense(4*Nt,activation='relu')(x1) #x1 = BatchNormalization()(x1) x1 = Dense(2 * Nt, activation='linear')(x1) x1 = Reshape((Nt, 2))(x1) x1 = Conv1D(filters=96, kernel_size=7, padding='same')(x1) x1 = BatchNormalization()(x1) x1 = Activation('relu')(x1) if mode == 'naive': x1 = naive_attention(x1) if mode == 'feature': x1 = feature_attention(x1, x0) x2 = Conv1D(filters=96, kernel_size=5, padding='same')(x1) x2 = BatchNormalization()(x2) x2 = Activation('relu')(x2) if mode == 'naive': x2 = naive_attention(x2) if mode == 'feature': x2 = feature_attention(x2, x0) x3 = Flatten()(x2) #prediction = Conv1D(filters=2,kernel_size=3,padding='same')(x2) #prediction = Flatten()(prediction) prediction = Dense(2 * Nt, activation='linear')(x3) if net == 'res_CNN': x = Conv1D(filters=96, kernel_size=7, padding='same')(x0) x = BatchNormalization()(x) x_init = Activation('relu')(x) if mode == 'naive': x_init = naive_attention(x_init) if mode == 'feature': x_init = feature_attention(x_init, x0) for i in range(3): # number of residual blocks x = Conv1D(filters=96, kernel_size=5, padding='same')(x_init) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv1D(filters=96, kernel_size=3, padding='same')(x) x = BatchNormalization()(x) if mode == 'naive': x = naive_attention(x) if mode == 'feature': x = feature_attention(x, x0) x_init = Add()([x, x_init]) x_init = Activation('relu')(x_init) x2 = Conv1D(filters=96, kernel_size=3, padding='same')(x_init) x2 = BatchNormalization()(x2) x2 = Activation('relu')(x2) if mode == 'naive': x2 = naive_attention(x2) if mode == 'feature': x2 = feature_attention(x2, x0) x3 = Flatten()(x2) prediction = Dense(2 * Nt, activation='linear')(x3) if net == 'FNN': x1 = Flatten()(x0) # 256,512,256 x1 = Dense(256, activation='relu')(x1) x1 = BatchNormalization()(x1) x1 = Dense(512, activation='relu')(x1) x1 = BatchNormalization()(x1) x1 = Dense(256, activation='relu')(x1) x1 = BatchNormalization()(x1) x3 = Flatten()(x1) prediction = Dense(2 * Nt, activation='linear')(x3) if net == 'FNN3': x1 = Flatten()(x0) x1 = Dense(96 * 96 // 2, activation='relu')(x1) x1 = BatchNormalization()(x1) x3 = Flatten()(x1) prediction = Dense(2 * Nt, activation='linear')(x3) if net == 'FNN2': x1 = Flatten()(x0) #x1 = Dense(128,activation='relu')(x1) #x1 = BatchNormalization()(x1) x1 = Dense(16 * 192, activation='relu')(x1) x1 = BatchNormalization()(x1) # grouping x1 = Reshape((16, 192))(x1) # attention x1 = naive_attention(x1) #x1 = feature_attention(x1,x0) #x1 = Flatten()(x1) #x1 = Dense(512,activation='relu')(x1) #x1 = BatchNormalization()(x1) x3 = Flatten()(x1) prediction = Dense(2 * Nt, activation='linear')(x3) model = Model(inputs=x0, outputs=prediction) model.compile(loss='mse', optimizer=Adam(lr=lr)) model.summary() return model
def attRNN(): sr = 8000 inputs = Input((8000, 1), name='input') x = Reshape((1, -1))(inputs) m = Melspectrogram(n_dft=1024, n_hop=128, input_shape=(1, 8000), padding='same', sr=sr, n_mels=80, fmin=40.0, fmax=sr / 2, power_melgram=1.0, return_decibel_melgram=True, trainable_fb=False, trainable_kernel=False, name='mel_stft') m.trainable = False x = m(x) x = Normalization2D(int_axis=0, name='mel_stft_norm')(x) # note that Melspectrogram puts the sequence in shape (batch_size, melDim, timeSteps, 1) # we would rather have it the other way around for LSTMs x = Permute((2, 1, 3))(x) x = Conv2D(10, (5, 1), activation='relu', padding='same')(x) x = BatchNormalization()(x) x = Conv2D(1, (5, 1), activation='relu', padding='same')(x) x = BatchNormalization()(x) # x = Reshape((125, 80)) (x) # keras.backend.squeeze(x, axis) x = Lambda(lambda q: K.squeeze(q, -1), name='squeeze_last_dim')(x) x = Bidirectional(LSTM(64, return_sequences=True))( x) # [b_s, seq_len, vec_dim] x = Bidirectional(LSTM(64, return_sequences=True))( x) # [b_s, seq_len, vec_dim] xFirst = Lambda(lambda q: q[:, -1])(x) # [b_s, vec_dim] query = Dense(128)(xFirst) # dot product attention attScores = Dot(axes=[1, 2])([query, x]) attScores = Softmax(name='attSoftmax')(attScores) # [b_s, seq_len] # rescale sequence attVector = Dot(axes=[1, 1])([attScores, x]) # [b_s, vec_dim] x = Dense(64, activation='relu')(attVector) x = Dense(32)(x) output = Dense(9, activation='softmax', name='output')(x) model = Model(inputs=[inputs], outputs=[output]) model.compile(optimizer='adam', loss=['sparse_categorical_crossentropy'], metrics=['sparse_categorical_accuracy']) return model
def _buildDQN(self): """ Build a network consistent with each type of inputs """ layers = [] outs_conv = [] inputs = [] for i, dim in enumerate(self._input_dimensions): # - observation[i] is a FRAME if len(dim) == 3 or len(dim) == 4: if (len(dim) == 4): input = Input(shape=(dim[-4], dim[-3], dim[-2], dim[-1])) inputs.append(input) input = Reshape((dim[-4] * dim[-3], dim[-2], dim[-1]), input_shape=(dim[-4], dim[-3], dim[-2], dim[-1]))(input) x = Permute( (2, 3, 1), input_shape=(dim[-4] * dim[-3], dim[-2], dim[-1]))( input) #data_format='channels_last' else: input = Input(shape=(dim[-3], dim[-2], dim[-1])) inputs.append(input) x = Permute((2, 3, 1), input_shape=(dim[-3], dim[-2], dim[-1]))( input) #data_format='channels_last' x = Conv2D(8, (4, 4), activation='relu', padding='valid')(x) #Conv on the frames x = Conv2D(16, (3, 3), activation='relu', padding='valid')(x) #Conv on the frames x = MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid')(x) x = Conv2D(16, (3, 3), activation='relu', padding='valid')(x) #Conv on the frames out = Flatten()(x) # - observation[i] is a VECTOR elif len(dim) == 2: if dim[0] > 3: input = Input(shape=(dim[0], dim[1])) inputs.append(input) reshaped = Reshape((dim[0], dim[1], 1), input_shape=(dim[0], dim[1]))(input) x = Conv2D(16, (2, 1), activation='relu', padding='valid')(reshaped) #Conv on the history x = Conv2D(16, (2, 1), activation='relu', padding='valid')( x) #Conv on the history & features out = Flatten()(x) else: input = Input(shape=(dim[0], dim[1])) inputs.append(input) out = Flatten()(input) # - observation[i] is a SCALAR - else: if dim[0] > 3: # this returns a tensor input = Input(shape=(dim[0], )) inputs.append(input) reshaped = Reshape((1, dim[0], 1), input_shape=(dim[0], ))(input) x = Conv2D(8, (1, 2), activation='relu', padding='valid')(reshaped) #Conv on the history x = Conv2D(8, (1, 2), activation='relu', padding='valid')(x) #Conv on the history out = Flatten()(x) else: input = Input(shape=(dim[0], )) inputs.append(input) out = input outs_conv.append(out) if (self._action_as_input == True): if (isinstance(self._n_actions, int)): print( "Error, env.nActions() must be a continuous set when using actions as inputs in the NN" ) else: input = Input(shape=(len(self._n_actions), )) inputs.append(input) outs_conv.append(input) if len(outs_conv) > 1: x = concatenate(outs_conv) else: x = outs_conv[0] # we stack a deep fully-connected network on top x = Dense(50, activation='relu')(x) x = Dense(20, activation='relu')(x) if (self._action_as_input == False): if (isinstance(self._n_actions, int)): out = Dense(self._n_actions)(x) else: out = Dense(len(self._n_actions))(x) else: out = Dense(1)(x) model = Model(inputs=inputs, outputs=out) layers = model.layers # Grab all the parameters together. params = [ param for layer in layers for param in layer.trainable_weights ] if (self._action_as_input == True): return model, params, inputs else: return model, params
def build_and_load_model_tf(model_capacity: str): """ Build the CNN model and load the weights, using the TensorFlow backend Parameters ---------- model_capacity : 'tiny', 'small', 'medium', 'large', or 'full' String specifying the model capacity, which determines the model's capacity multiplier to 4 (tiny), 8 (small), 16 (medium), 24 (large), or 32 (full). 'full' uses the model size specified in the paper, and the others use a reduced number of filters in each convolutional layer, resulting in a smaller model that is faster to evaluate at the cost of slightly reduced pitch estimation accuracy. Returns ------- model : tensorflow.keras.models.Model The pre-trained model loaded in memory """ from tensorflow.keras.layers import (Input, Reshape, Conv2D, BatchNormalization) from tensorflow.keras.layers import (MaxPool2D, Dropout, Permute, Flatten, Dense) from tensorflow.keras.models import Model if models['tf'][model_capacity] is None: capacity_multiplier = { 'tiny': 4, 'small': 8, 'medium': 16, 'large': 24, 'full': 32 }[model_capacity] layers = [1, 2, 3, 4, 5, 6] filters = [n * capacity_multiplier for n in [32, 4, 4, 4, 8, 16]] widths = [512, 64, 64, 64, 64, 64] strides = [(4, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)] x = Input(shape=(1024, ), name='input', dtype='float32') y = Reshape(target_shape=(1024, 1, 1), name='input-reshape')(x) for l, f, w, s in zip(layers, filters, widths, strides): y = Conv2D(f, (w, 1), strides=s, padding='same', activation='relu', name="conv%d" % l)(y) y = BatchNormalization(name="conv%d-BN" % l)(y) y = MaxPool2D(pool_size=(2, 1), strides=None, padding='valid', name="conv%d-maxpool" % l)(y) y = Dropout(0.25, name="conv%d-dropout" % l)(y) y = Permute((2, 1, 3), name="transpose")(y) y = Flatten(name="flatten")(y) y = Dense(360, activation='sigmoid', name="classifier")(y) model = Model(inputs=x, outputs=y) package_dir = os.path.dirname(os.path.realpath(__file__)) filename = "model-{}.h5".format(model_capacity) model.load_weights(os.path.join(package_dir, filename)) model.compile('adam', 'binary_crossentropy') models['tf'][model_capacity] = model
def attention_3d_block(inputs): a = Permute((2, 1))(inputs) a = Dense(3, activation='softmax')(a) a_probs = Permute((2, 1))(a) output_attention_mul = Multiply()([inputs, a_probs]) return output_attention_mul
def buildModel( num_input_channels=24, num_timesteps=20, num_filters=20, num_LSTM=20, num_Dense=200, drop_rate=0.2, reg=None, ): kernel_size = (3, 3) num_classes = 2 ac = "relu" opt = Adam(lr=0.001, decay=0, beta_1=0.9, beta_2=0.999, epsilon=1e-08) inp = Input( (num_timesteps, num_input_channels, 8, 8) ) # timesteps, channels, rows, columns permuted = Permute((1, 3, 4, 2))(inp) # expects channels_last x = TimeDistributed( Conv2D( num_filters, kernel_size, activation=ac, kernel_regularizer=reg, # data_format="channels_first", ) )(permuted) x = BatchNormalization(axis=-1)(x) x = TimeDistributed( Conv2D( 2 * num_filters, kernel_size, activation=ac, kernel_regularizer=reg, # data_format="channels_first", ) )(x) x = BatchNormalization(axis=-1)(x) x = TimeDistributed( Conv2D( 4 * num_filters, kernel_size, activation=ac, kernel_regularizer=reg, # data_format="channels_first", ) )(x) x = BatchNormalization(axis=-1)(x) x = TimeDistributed( Conv2D( 8 * num_filters, kernel_size, activation=ac, kernel_regularizer=reg, padding="same", # data_format="channels_first", ) )(x) x = BatchNormalization(axis=-1)(x) x = TimeDistributed(Flatten())(x) x = LSTM(num_LSTM)(x) x = Dropout(drop_rate)(x) x = Dense(num_Dense)(x) x = Dropout(drop_rate)(x) out = Dense(num_classes, activation="softmax")(x) myModel = Model(inputs=[inp], outputs=[out]) myModel.compile(loss="binary_crossentropy", metrics=["accuracy"], optimizer=opt) LOGGER.info("built model!") LOGGER.info(myModel.summary()) return myModel
def create_model(input_shape, config): weight_decay = 0.001 model = Sequential() model.add( Convolution2D( 16, 7, 7, kernel_regularizer=l2(weight_decay), activation="relu", input_shape=input_shape, )) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add( Convolution2D(32, 5, 5, kernel_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add( Convolution2D(64, 3, 3, kernel_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add( Convolution2D(128, 3, 3, kernel_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 1))) model.add( Convolution2D(256, 3, 3, kernel_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 1))) # model.load_weights("logs/2017-04-08-13-03-44/weights.08.model", by_name=True) # for ref_layer in ref_model.layers: # layer = model.get_layer(ref_layer.name) # if layer: # layer.set_weights(ref_layer.get_weights()) for layer in model.layers: layer.trainable = False # (bs, y, x, c) --> (bs, x, y, c) model.add(Permute((2, 1, 3))) # (bs, x, y, c) --> (bs, x, y * c) bs, x, y, c = model.layers[-1].output_shape model.add(Reshape((x, y * c))) model.add( Bidirectional(LSTM(512, return_sequences=False), merge_mode="concat")) model.add(Dense(config["num_classes"], activation="softmax")) return model
def __init__(self, input_shape, model_folder="models/serialized/"): super().__init__() conv_model = Sequential() conv_model.add(Permute((1, 2, 3), input_shape=input_shape)) for l in Convblock(64, 1, 2): conv_model.add(l) for l in Convblock(128, 2, 2): conv_model.add(l) for l in Convblock(256, 3, 3): conv_model.add(l) for l in Convblock(512, 4, 3): conv_model.add(l) for l in Convblock(512, 5, 3): conv_model.add(l) conv_model.add( Convolution2D(2048, kernel_size=(7, 7), padding="same", activation="relu", name="fc_6")) # Replacing fully connected layers of VGG Net using convolutions -- reduced filters conv_model.add( Convolution2D(2048, kernel_size=(1, 1), padding="same", activation="relu", name="fc7")) conv_model.add( Convolution2D(1, kernel_size=(1, 1), padding="same", activation="relu", name="score_fr")) Conv_size = conv_model.layers[-1].output_shape[2] print("Convolution size: " + str(Conv_size)) # conv_model.add( Conv2DTranspose(1, strides=(2, 2), kernel_size=(4, 4), padding="valid", activation=None, name="score2")) # I = (O-1)*Stride + K Deconv_size = conv_model.layers[-1].output_shape[2] print("Deconvolution size: " + str(Deconv_size)) # 2 if image size is 512*512 Extra = (Deconv_size - 2 * Conv_size) print("Extra size: " + str(Extra)) # Cropping to get correct size conv_model.add(Cropping2D(cropping=((0, Extra), (0, Extra)))) Conv_size = conv_model.layers[-1].output_shape[2] skip_con1 = Convolution2D(1, kernel_size=(1, 1), padding="same", activation=None, name="score_pool4") # Adding skip connection which takes adds the output of Max pooling layer 4 to current layer Summed = add(inputs=[ skip_con1(conv_model.layers[14].output), conv_model.layers[-1].output ]) # Upsampling output of first skip connection x = Conv2DTranspose(21, kernel_size=(4, 4), strides=(2, 2), padding="valid", activation=None, name="score4")(Summed) x = Cropping2D(cropping=((0, 2), (0, 2)))(x) # Conv to be applied to pool3 skip_con2 = Convolution2D(21, kernel_size=(1, 1), padding="same", activation=None, name="score_pool3") # Adding skip connection which takes output og Max pooling layer 3 to current layer Summed = add(inputs=[skip_con2(conv_model.layers[10].output), x]) Up = Conv2DTranspose(1, kernel_size=(16, 16), strides=(8, 8), padding="valid", activation=None, name="upsample")(Summed) # Cropping the extra part obtained due to transpose convolution final = Cropping2D(cropping=((0, 8), (0, 8)))(Up) self.seg_model = Model(conv_model.input, final) self.seg_model.summary() self.model_folder = model_folder self.weights_path = (model_folder + "fcn_weights.best.hdf5").format('seg_model') self.set_callbacks() self.name = "FCN8"
def HRNetDense(input_shape=(128, 128, 3), num_keypoints=20, growth_rate=4): # stem inputs = Input(shape=input_shape) x1 = stem(inputs, 64) x1 = Conv2D(64 * 4, 1, padding='same', use_bias=False)(x1) x1 = BatchNormalization()(x1) for block in range(4): x1 = bottleneck(x1) # stage I x1 = Conv2D(32, 3, padding='same', use_bias=False)(x1) x1 = BatchNormalization()(x1) x1 = Activation('relu')(x1) x2 = transition_block(x1, 2) print('stage 1', x1.shape, x2.shape) # stage II x1 = dense_block(x1, 4, growth_rate) x2 = dense_block(x2, 4, growth_rate) x1, x2 = fuse([x1, x2]) x3 = transition_block(x2, 0.5) print('stage 2', x1.shape, x2.shape, x3.shape) # stage III x1 = dense_block(x1, 4, growth_rate) x2 = dense_block(x2, 4, growth_rate) x3 = dense_block(x3, 4, growth_rate) x1, x2, x3 = fuse([x1, x2, x3]) x4 = transition_block(x3, 0.5) print('stage 3', x1.shape, x2.shape, x3.shape, x4.shape) # stage IV x1 = dense_block(x1, 3, growth_rate) x2 = dense_block(x2, 3, growth_rate) x3 = dense_block(x3, 3, growth_rate) x4 = dense_block(x4, 3, growth_rate) x1, x2, x3, x4 = fuse([x1, x2, x3, x4]) print('stage 4', x1.shape, x2.shape, x3.shape, x4.shape) x2 = UpSampling2D(size=(2, 2))(x2) x3 = UpSampling2D(size=(4, 4))(x3) x4 = UpSampling2D(size=(8, 8))(x4) x = Concatenate()([x1, x2, x3, x4]) # head x = Conv2D(480, 1)(x) x = BatchNormalization(epsilon=1.001e-5)(x) x = Activation('relu')(x) x = Conv2D(num_keypoints, 1)(x) # extra x = BatchNormalization(epsilon=1.001e-5)(x) x = Activation('relu')(x) x = UpSampling2D(size=(4, 4), interpolation='bilinear')(x) x = Permute([3, 1, 2])(x) x = Reshape([num_keypoints, input_shape[0] * input_shape[1]])(x) x = Activation('softmax')(x) x = Reshape([num_keypoints, input_shape[0], input_shape[1]])(x) outputs = ExpectedValue2D(name='expected_uv')(x) model = Model(inputs, outputs, name='hrnet-dense') return model
def build_model(X, nb_classes): pool_size = (3, 3) # size of pooling area for max pooling kernel_size = (5, 5) # convolution kernel size nb_layers = 6 input_shape = (X.shape[1], X.shape[2], X.shape[3]) model = Sequential() # 4 2D Convolution Layers model.add( Conv2D(64, input_shape=input_shape, kernel_size=kernel_size, kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01), padding="same", activation="relu")) model.add(MaxPooling2D(pool_size=pool_size)) model.add(BatchNormalization()) model.add(Dropout(0.25)) # model.add(Conv2D(64,kernel_size=kernel_size,kernel_regularizer=l2(0.01),bias_regularizer=l2(0.01),padding="same",activation="relu")) # model.add(MaxPooling2D(pool_size=pool_size)) # model.add(BatchNormalization()) # model.add(Dropout(0.25)) # model.add(Conv2D(128,kernel_size=kernel_size,kernel_regularizer=l2(0.01),bias_regularizer=l2(0.01),padding="same",activation="relu")) # model.add(MaxPooling2D(pool_size=pool_size)) # model.add(BatchNormalization()) # model.add(Dropout(0.2)) # model.add(Conv2D(128,kernel_size=kernel_size,kernel_regularizer=l2(0.01),bias_regularizer=l2(0.01),padding="same",activation="relu")) # model.add(MaxPooling2D(pool_size=pool_size)) # model.add(BatchNormalization()) # model.add(Dropout(0.25)) model.add(Permute((2, 1, 3))) last_shape = model.layers[-1].output_shape model.add(Reshape((last_shape[1], last_shape[2] * last_shape[3]))) # 2 LSTM layers model.add( LSTM(64, kernel_regularizer=l2(0.01), recurrent_regularizer=l2(0.01), recurrent_dropout=0.5, return_sequences=True, go_backwards=False, activation='tanh')) model.add( LSTM(64, kernel_regularizer=l2(0.01), recurrent_regularizer=l2(0.01), recurrent_dropout=0.5, return_sequences=True, go_backwards=True, activation='tanh')) model.add(Dropout(0.25)) # model.add(Flatten()) model.add(TimeDistributed(Dense(128, activation='relu'))) model.add(Dropout(0.25)) model.add(TimeDistributed(Dense(64, activation='relu'))) model.add(Dropout(0.2)) model.add(Flatten()) #relu throughout the network and softmax at output layer (multiple classes) model.add(Dense(nb_classes, activation='softmax')) return model
def Create_Model(length, vocab_size): # INPUT, EMBEDDING inputs0 = Input(batch_shape=(BATCH_SIZE, len(net_train[0]))) embedding1 = Embedding(vocab_size, 150)(inputs0) drop_out1 = Dropout(0.1, name='dropout1')(embedding1) # 2 PARTs FOR BIDIRECTIONAL LSTM lstm_fwd = LSTM(150, return_sequences=True, name='lstm_fwd')(drop_out1) lstm_bwd = LSTM(150, return_sequences=True, go_backwards=True, name='lstm_bwd')(drop_out1) # CONCAT 2 PARTS FOR BILSTM bilstm = concatenate([lstm_fwd, lstm_bwd], name='bilstm') # , mode='concat' drop_out2 = Dropout(0.1)(bilstm) # "LAST ELEMENT FROM TIME DIM" h_n = Lambda(get_H_n, output_shape=(300, ), name="h_n")(drop_out2) Whn = Dense(300, kernel_regularizer=l2(0.01), name="Wh_n")(h_n) Whn_x_e = RepeatVector(L, name="Wh_n_x_e")(Whn) # "FIRST first-sentence-MAXLEN ELEMENTS FROM TIME DIM" Y = Lambda(get_Y, arguments={"xmaxlen": L}, name="Y", output_shape=(L, 300))(drop_out2) WY = TimeDistributed(Dense(300, kernel_regularizer=l2(0.01)), name="WY")(Y) # CONCAT 2 LAMBA "ATTENTIONS"? merged1 = Add()( [Whn_x_e, WY]) # concatenate([Whn_x_e, WY], name="merged1", mode='sum') M = Activation('tanh', name="M")(merged1) alpha_ = TimeDistributed(Dense(1, activation='linear'), name="alpha_")(M) # FLATTEN AND GATE flat_alpha = Flatten(name="flat_alpha")(alpha_) alpha = Dense(L, activation='softmax', name="alpha")(flat_alpha) # CHANGE SHAPE OF Y Y_trans = Permute((2, 1), name="y_trans")(Y) # of shape (None,300,20) # CONCAT CHANGED Y WITH MAIN PATHWAY OF NN r_ = Lambda(get_R, output_shape=(300, 1))([Y_trans, alpha]) #, output_shape=(300, 1) #r_ = concatenate([Y_trans, alpha], output_shape=(300, 1), name="r_", mode=get_R) r = Reshape((300, ), name="r")(r_) Wr = Dense(300, kernel_regularizer=l2(0.01))(r) # CALL ALL THE WAY BACK TO h_n... god knows why Wh = Dense(300, kernel_regularizer=l2(0.01))(h_n) # MERGE MAIN PATHWAY OF NN WITH CALLBACK LAYER TO h_n merged2 = Add()([Wr, Wh]) # concatenate([Wr, Wh], mode='sum') h_star = Activation('tanh')(merged2) outputs = Dense(3, activation='softmax')(h_star) model = Model(inputs=[inputs0], outputs=outputs) model.compile(Adam(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy']) print(model.summary()) plot_model(model, show_shapes=True, to_file='multichannel2.png') return model
def create_model(input_shape, config, is_training=True): weight_decay = 0.001 model = Sequential() model.add( Convolution2D(64, 3, 3, W_regularizer=l2(weight_decay), activation="relu", input_shape=input_shape)) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add( Convolution2D(128, 3, 3, W_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add( Convolution2D(256, 3, 3, W_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) # model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add( Convolution2D(256, 3, 3, W_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add( Convolution2D(512, 3, 3, W_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) # model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add( Convolution2D(512, 3, 3, W_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add( Convolution2D(512, 3, 3, W_regularizer=l2(weight_decay), activation="relu")) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) # (bs, y, x, c) --> (bs, x, y, c) model.add(Permute((2, 1, 3))) # (bs, x, y, c) --> (bs, x, y * c) bs, x, y, c = model.layers[-1].output_shape model.add(Reshape((x, y * c))) model.add( Bidirectional(LSTM(256, return_sequences=False), merge_mode="concat")) model.add(Dense(config["num_classes"], activation="softmax")) return model
def _build_model(self): max_seq_length = self.sentence_len * self.num_sentences #Embedding Layer embedding_layer = Embedding(self.vocab_size, self.embedding_dim, input_length=max_seq_length, trainable=self.train_embedding, embeddings_regularizer=regularizers.l2( self.embedding_regularizer_l2), name='imdb_embedding') if self.embedding_weights is not None: embedding_layer = Embedding(self.vocab_size, self.embedding_dim, weights=[self.embedding_weights], input_length=max_seq_length, trainable=self.train_embedding, embeddings_regularizer=regularizers.l2( self.embedding_regularizer_l2), name='imdb_embedding') #input layer : sequence of word indices for each sentence sequence_input = Input(shape=(max_seq_length, ), dtype='int32') z = embedding_layer(sequence_input) if self.input_dropout > 0: z = Dropout(self.input_dropout)(z) conv_blocks = [] i = 0 #same convolution filters to be used for all sentences. word_conv_model = Conv1D(filters=self.word_filters, kernel_size=self.word_kernel_size, padding="valid", activation=self.conv_activation, trainable=self.learn_word_conv, name="word_conv", strides=1) for sent in range(self.num_sentences): #get once sentence from the input sentence = Lambda(lambda x: x[:, sent * self.sentence_len:( sent + 1) * self.sentence_len, :])(z) conv = word_conv_model(sentence) conv = KMaxPooling(k=self.sent_k_maxpool)(conv) #transpose pooled values per sentence conv = Reshape([self.word_filters * self.sent_k_maxpool, 1])(conv) conv_blocks.append(conv) #append all sentence convolution feature maps and make sentence embeddings z = Concatenate()( conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0] #transform to (steps, input_dim) z = Permute([2, 1], name='sentence_embeddings')(z) if self.sent_dropout > 0: z = Dropout(self.sent_dropout)(z) sent_conv = Conv1D(filters=self.sent_filters, kernel_size=self.sent_kernel_size, padding="valid", activation=self.conv_activation, trainable=self.learn_sent_conv, name='sentence_conv', strides=1)(z) z = KMaxPooling(k=self.doc_k_maxpool)(sent_conv) z = Flatten(name='document_embedding')(z) if self.hidden_gaussian_noise_sd: z = GaussianNoise(self.hidden_gaussian_noise_sd)(z) elif self.hidden_dropout: z = Dropout(self.hidden_dropout)(z) for i in range(self.num_hidden_layers): layer_name = 'hidden_{}'.format(i) z = Dense(self.hidden_dims, activation=self.hidden_activation, name=layer_name, kernel_regularizer=regularizers.l2( self.hidden_layer_kernel_regularizer))(z) output_activation = 'sigmoid' if self.num_units_final_layer > 1: output_activation = 'softmax' model_output = Dense(self.num_units_final_layer, activation=output_activation, kernel_regularizer=regularizers.l2( self.final_layer_kernel_regularizer), name='final')(z) self._model = Model(sequence_input, model_output)
def get_model(self): inputs_cnn = [] inputs_lstm = [] outputs_cnn = [] lstm = [] for i in range(self.window_size): input_cnn = Input(shape=(self.mesh_rows, self.mesh_columns, self.depth), name="input" + str(i + 1)) input_lstm = Input(shape=(self.number_channels, self.depth), name="input" + str(i + 1 + self.window_size)) inputs_cnn.append(input_cnn) inputs_lstm.append(input_lstm) conv1 = self.tfaugmented_conv2d(input_cnn, self.conv1_filters, self.conv1_kernel_shape, dk=self.depth_k, dv=self.depth_v, Nh=self.num_heads, relative=self.relative) norm1 = BatchNormalization(axis=-1)(conv1) conv2 = Conv2D(self.conv2_filters, self.conv2_kernel_shape, padding=self.padding2, activation=self.conv2_activation)(norm1) conv3 = Conv2D(self.conv3_filters, self.conv3_kernel_shape, padding=self.padding3, activation=self.conv3_activation)(conv2) flat = Flatten()(conv3) dense = Dense(self.dense_nodes, activation=self.dense_activation)(flat) outputs_cnn.append(dense) permut = Permute((2, 1), input_shape=(self.number_channels, 1))(input_lstm) dense = Dense(self.dense_nodes, activation=self.dense_activation, input_shape=(1, self.number_channels))(permut) lstm.append(dense) merge = concatenate(lstm, axis=1) lstm1 = LSTM(self.lstm1_cells, return_sequences=True)(merge) attention_output = self.attention_block(lstm1) attention_output = tf.expand_dims(attention_output, -1) lstm2 = LSTM(self.lstm2_cells, return_sequences=False)(attention_output) dense3 = Dense(self.dense3_nodes, activation=self.dense3_activation)(lstm2) added = Add()([i for i in outputs_cnn]) final = concatenate([dense3, added], axis=-1) output = Dense(4, activation="softmax")(final) model = Model(inputs=inputs_cnn + inputs_lstm, outputs=output) return model