def rcnn(base_layers, rois, num_classes, image_max_dim, head_fn, pool_size=(7, 7), fc_layers_size=1024): # RoiAlign x = RoiAlign(image_max_dim, pool_size=pool_size)([base_layers, rois]) # # 收缩维度 shared_layer = head_fn(x) # 分类 class_logits = TimeDistributed(layers.Dense(num_classes, activation='linear'), name='rcnn_class_logits')(shared_layer) # 回归(类别相关) deltas = TimeDistributed( layers.Dense(4 * num_classes, activation='linear'), name='rcnn_deltas')( shared_layer) # shape (batch_size,roi_num,4*num_classes) # 变为(batch_size,roi_num,num_classes,4) roi_num = backend.int_shape(deltas)[ 1] # print("roi_num:{}".format(roi_num)) deltas = layers.Reshape((roi_num, num_classes, 4))(deltas) return deltas, class_logits
def seq_2_seq_att_LSTM(X_embedding, MAX_LEN, num_words, EMBEDDING_DIM, LSTM_units, LSTM_dropout): # Encoder # Encoder input shape is (batch size, max length) encoder_inputs = Input(shape=(MAX_LEN,)) encoder_embedding = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM, input_length = MAX_LEN, embeddings_initializer=Constant(X_embedding), trainable=False)(encoder_inputs) # LSTM encoder_lstm = LSTM(units=LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout) encoder_outputs, state_h, state_c = encoder_lstm(encoder_embedding) encoder_states = [state_h, state_c] # Decoder decoder_inputs = Input(shape=(None,)) decoder_embedding_layer = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM, trainable=True) decoder_embedding = decoder_embedding_layer(decoder_inputs) decoder_lstm = LSTM(units=LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout) decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states) # Attention attention_weight = dot([decoder_outputs, encoder_outputs], axes=[2, 2], normalize=True) # cosine similarity attention = Activation('softmax')(attention_weight) context = dot([attention, encoder_outputs], axes=[2,1]) decoder_combined_context = concatenate([context, decoder_outputs]) att_output = TimeDistributed(Dense(64, activation="tanh"))(decoder_combined_context) output = TimeDistributed(Dense(num_words, activation="softmax"))(att_output) model = Model(inputs=[encoder_inputs,decoder_inputs], outputs=output) return model
def bd_model(input_shape, output_sequence_length, english_vocab_size, spanish_vocab_size): """ Build and train a bidirectional RNN model on x and y :param input_shape: Tuple of input shape :param output_sequence_length: Length of output sequence :param english_vocab_size: Number of unique English words in the dataset :param french_vocab_size: Number of unique French words in the dataset :return: Keras model built, but not trained """ # TODO: Implement # Hyperparameters #learning_rate = 0.003 # TODO: Build the layers model = Sequential() model.add( Bidirectional(GRU(128, return_sequences=True), input_shape=input_shape[1:])) model.add(TimeDistributed(Dense(512, activation='relu'))) #model.add(Dropout(0.5)) model.add(TimeDistributed(Dense(spanish_vocab_size, activation='softmax'))) # Compile model model.compile(loss='sparse_categorical_crossentropy', optimizer='Adam', metrics=['accuracy']) return model
def create_rnn_model(self): """ """ seq_input = Input(shape=(self.dense_input_len, 1)) seq_output = Input(shape=(self.dense_input_len, 1)) # norm_seq_input = BatchNormalization(name = 'Dense_BN_trainable')(seq_input) rnn_out = Bidirectional( LSTM(self.rnn_units[0], return_sequences=True, activation='relu'))(seq_input) rnn_out = Bidirectional( LSTM(self.rnn_units[1], return_sequences=True, activation='relu'))(rnn_out) seq_pred = TimeDistributed(Dense(self.hidden_dim[0], activation='relu'))(rnn_out) seq_pred = TimeDistributed(Dense(1, activation='relu'))(seq_pred) # seq_pred = Dense(1, activation = 'relu')(rnn_out) seq_pred = Reshape((self.dense_input_len, ))(seq_pred) seq_input_reshape = Reshape((self.dense_input_len, ))(seq_input) model = Model(seq_input, seq_pred) loss = K.mean( mean_squared_error(seq_input_reshape[:, 1:], seq_pred[:, :-1])) model.add_loss(loss) # def _mean_squared_error(y_true, y_pred): # return K.mean(K.square(y_pred - y_true)) model.compile(optimizer='adam', loss=None) #_mean_squared_error) return model
def build_model(self, pos_mode=0, use_mask=False, active_layers=999): v_input = Input(shape=(self.seq_len, self.d_feature), name='v_input') d0 = TimeDistributed(Dense(self.d_model))(v_input) pos_input = Input(shape=(self.seq_len, ), dtype='int32', name='pos_input') if pos_mode == 0: # use fixed pos embedding pos_embedding = Embedding(self.seq_len, self.d_model, trainable=False,\ weights=[GetPosEncodingMatrix(self.seq_len, self.d_model)]) p0 = pos_embedding(pos_input) elif pos_mode == 1: # use trainable pos embedding pos_embedding = Embedding(self.seq_len, self.d_model) p0 = pos_embedding(pos_input) else: # no pos embedding p0 = None if p0 != None: combine_input = Add()([d0, p0]) else: combine_input = d0 # no pos sub_mask = None if use_mask: sub_mask = Lambda(GetSubMask)(pos_input) enc_output = self.encoder(combine_input, mask=sub_mask, active_layers=active_layers) # score time_score_dense1 = TimeDistributed( Dense(self.d_model, activation='tanh'))(enc_output) time_score_dense2 = TimeDistributed(Dense(1))(time_score_dense1) flat = Flatten()(time_score_dense2) score_output = Activation(activation='softmax')(flat) self.model = Model([pos_input, v_input], score_output) return self.model
def demo_create_encoder(latent_dim, cat_dim, window_size, input_dim): input_layer = Input(shape=(window_size, input_dim)) code = TimeDistributed(Dense(64, activation='linear'))(input_layer) code = Bidirectional(LSTM(128, return_sequences=True))(code) code = BatchNormalization()(code) code = ELU()(code) code = Bidirectional(LSTM(64))(code) code = BatchNormalization()(code) code = ELU()(code) cat = Dense(64)(code) cat = BatchNormalization()(cat) cat = PReLU()(cat) cat = Dense(cat_dim, activation='softmax')(cat) latent_repr = Dense(64)(code) latent_repr = BatchNormalization()(latent_repr) latent_repr = PReLU()(latent_repr) latent_repr = Dense(latent_dim, activation='linear')(latent_repr) decode = Concatenate()([latent_repr, cat]) decode = RepeatVector(window_size)(decode) decode = Bidirectional(LSTM(64, return_sequences=True))(decode) decode = ELU()(decode) decode = Bidirectional(LSTM(128, return_sequences=True))(decode) decode = ELU()(decode) decode = TimeDistributed(Dense(64))(decode) decode = ELU()(decode) decode = TimeDistributed(Dense(input_dim, activation='linear'))(decode) error = Subtract()([input_layer, decode]) return Model(input_layer, [decode, latent_repr, cat, error])
def get_test_model(seq_len, stacked_features_size, edges_features_matrix_depth, neighbourhood_size, units): # INPUTS stacked_base_features_input = Input(shape=(seq_len, stacked_features_size), name='stacked_base_features') adjacency_matrix_input = Input(shape=(seq_len, seq_len), name='adjacency_matrix') edges_features_matrix_input = Input(shape=(seq_len, seq_len, edges_features_matrix_depth), name='edges_features_matrix') inputs = (stacked_base_features_input, adjacency_matrix_input, edges_features_matrix_input) # ACTUAL MODEL x = Subgraphing(neighbourhood_size)(inputs) x = RNN(GraphReduceCell(units), return_sequences=True)(x) # OUTPUTS reactivity_pred = TimeDistributed(Dense(1), name='reactivity')(x) deg_Mg_pH10_pred = TimeDistributed(Dense(1), name='deg_Mg_pH10')(x) deg_Mg_50C_pred = TimeDistributed(Dense(1), name='deg_Mg_50C')(x) scored_outputs = [reactivity_pred, deg_Mg_pH10_pred, deg_Mg_50C_pred] stacked_outputs = Concatenate(axis=2, name='stacked_outputs')(scored_outputs) # MODEL DEFINING model = Model(inputs=inputs, outputs={'stacked_scored_labels': stacked_outputs}, name='graph_reduce_model') return model
def model_v2(top_layer_units): model = Sequential() model.add( TimeDistributed(Conv1D(filters=16, kernel_size=4, padding='same', activation=tf.nn.relu, data_format='channels_last'), input_shape=(NUM_MFCC, NUM_FRAMES, 1))) model.add( TimeDistributed( Conv1D(filters=8, kernel_size=2, padding='same', activation=tf.nn.relu))) model.add(TimeDistributed(MaxPooling1D(pool_size=2))) model.add(TimeDistributed(Flatten())) model.add(LSTM(50, return_sequences=True)) model.add(Dropout(0.3)) model.add(Flatten()) model.add(Dense(units=512, activation=tf.nn.tanh)) model.add(Dense(units=256, activation=tf.nn.tanh)) model.add( Dense(units=top_layer_units, activation=tf.nn.softmax, name='top_layer')) return model
def BasicBlock(inputs, filters, strides=1, downsample=None): residual = inputs out = TimeDistributed( Conv2D(filters=filters, kernel_size=3, strides=strides, padding='same', kernel_initializer='he_uniform'))(inputs) out = BatchNormalization()(out) out = Activation('relu')(out) out = TimeDistributed( Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', kernel_initializer='he_uniform'))(out) out = BatchNormalization()(out) if downsample is not None: residual = downsample(inputs) out = add([out, residual]) out = Activation('relu')(out) return out
def __init__(self, n_head, d_model, d_k, d_v, dropout, mode=0, use_norm=True): self.mode = mode self.n_head = n_head self.d_k = d_k self.d_v = d_v self.dropout = dropout if mode == 0: self.qs_layer = Dense(n_head * d_k, use_bias=False) self.ks_layer = Dense(n_head * d_k, use_bias=False) self.vs_layer = Dense(n_head * d_v, use_bias=False) elif mode == 1: self.qs_layers = [] self.ks_layers = [] self.vs_layers = [] for _ in range(n_head): self.qs_layers.append( TimeDistributed(Dense(d_k, use_bias=False))) self.ks_layers.append( TimeDistributed(Dense(d_k, use_bias=False))) self.vs_layers.append( TimeDistributed(Dense(d_v, use_bias=False))) self.attention = ScaledDotProductAttention(d_model) self.layer_norm = LayerNormalization() if use_norm else None self.w_o = TimeDistributed(Dense(d_model))
def build(self): n_steps, n_length = 4, 32 self.train_X = self.train_X.reshape( (self.train_X.shape[0], n_steps, n_length, self.n_features)) # self.n_features = 6 self.test_X = self.test_X.reshape( (self.test_X.shape[0], n_steps, n_length, self.n_features)) model = Sequential() model.add(TimeDistributed(Conv1D(filters=64, kernel_size=3, activation='relu'), input_shape=(None, n_length, self.n_features))) model.add(TimeDistributed( Conv1D(filters=16, kernel_size=3, activation='relu'))) model.add(TimeDistributed(Dropout(0.5))) model.add(TimeDistributed(MaxPooling1D(pool_size=2))) model.add(TimeDistributed(Flatten())) model.add(LSTM(100)) model.add(Dropout(0.5)) model.add(Dense(100, activation='relu')) model.add(Dense(self.n_outputs, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # print(model.summary()) # input() super().build(model)
def create_decoder(input_timesteps, latent_dim): decoder_in = K.layers.Input(shape=( input_timesteps, latent_dim, ), name='latent_sample_in') action_data = Input(shape=(input_timesteps, 1), name='action_data') observations = TimeDistributed(Dense(units=100, activation='relu'), name='obs1')(decoder_in) observations = TimeDistributed(Dense(units=100, activation='relu'), name='obs2')(observations) observations = TimeDistributed(Dense(units=100, activation='relu'), name='obs3')(observations) observations = TimeDistributed(Dense(units=3 * 2), name='obs_out')(observations) rewards = TimeDistributed(Dense(units=100, activation='relu'), name='rew1')(decoder_in) rewards = TimeDistributed(Dense(units=100, activation='relu'), name='rew2')(rewards) rewards = TimeDistributed(Dense(units=100, activation='relu'), name='rew3')(rewards) rewards = TimeDistributed(Dense(units=1 * 2), name='rew_out')(rewards) state_action = Concatenate()([decoder_in, action_data]) next_state = TimeDistributed(Dense(units=100, activation='relu'), name='ns1')(state_action) #next_state = TimeDistributed(Dense(units=100, activation='relu'), name='ns2')(next_state) #next_state = TimeDistributed(Dense(units=100, activation='relu'), name='ns3')(next_state) next_state = TimeDistributed(Dense(units=latent_dim * 2), name='next_state')(next_state) return Model(inputs=[decoder_in, action_data], outputs=[observations, rewards, next_state])
def default_final_detection_model(pyramid_feature_size=256, final_detection_feature_size=256, roi_size=(14, 14), name='final_detection_submodel'): """Creates a final detection model for 3D RetinaMask models. Args: pyramid_feature_size (int): Number of features for the input to the final detection model. final_detection_feature_size (int): Number of filters used in the 2D convolution layers. roi_size (tuple): Size of the region of interest, serves as the x and y dimensions of the input to the final detection model. name (str): Name of the model. Returns: tensorflow.keras.Model: a FinalDetection submodel for 3D RetinaMask. """ options = { 'kernel_size': 3, 'strides': 1, 'padding': 'same', 'kernel_initializer': normal(mean=0.0, stddev=0.01, seed=None), 'bias_initializer': 'zeros', 'activation': 'relu' } if K.image_data_format() == 'channels_first': input_shape = (None, pyramid_feature_size, roi_size[0], roi_size[1]) else: input_shape = (None, roi_size[0], roi_size[1], pyramid_feature_size) inputs = Input(shape=input_shape) outputs = inputs for i in range(2): outputs = TimeDistributed( Conv2D(filters=final_detection_feature_size, **options), name='final_detection_submodel_conv1_block{}'.format(i))(outputs) outputs = TimeDistributed( Conv2D(filters=final_detection_feature_size, **options), name='final_detection_submodel_conv2_block{}'.format(i))(outputs) outputs = TimeDistributed( MaxPool2D(), name='final_detection_submodel_pool1_block{}'.format(i))(outputs) outputs = TimeDistributed( Conv2D(filters=final_detection_feature_size, kernel_size=3, padding='valid', kernel_initializer=normal(mean=0.0, stddev=0.01, seed=None), bias_initializer='zeros', activation='relu'))(outputs) outputs = TimeDistributed( Conv2D(filters=1, kernel_size=1, activation='sigmoid'))(outputs) outputs = Lambda(lambda x: tf.squeeze(x, axis=[2, 3]))(outputs) return Model(inputs=inputs, outputs=outputs, name=name)
def create_model(self): hidden_size = 256 enc_timesteps = self.max_encoder_seq_length #timesteps = self.max_encoder_seq_length #perhaps making timesteps size of max sequence length would work?????"" dec_timesteps = self.max_decoder_seq_length print(f"embedding size: {self.glove_model.embedding_size}") # encoder_inputs = Input(shape=(None, self.glove_model.embedding_size), name='encoder_inputs') # decoder_inputs = Input(shape=(None, self.num_decoder_tokens), name='decoder_inputs') encoder_inputs = Input(shape=(enc_timesteps, self.glove_model.embedding_size), name='encoder_inputs') decoder_inputs = Input(shape=(dec_timesteps, self.num_decoder_tokens), name='decoder_inputs') # Encoder GRU encoder_gru = Bidirectional(GRU(hidden_size, return_sequences=True, return_state=True, name='encoder_gru'), name='bidirectional_encoder') encoder_out, encoder_fwd_state, encoder_back_state = encoder_gru(encoder_inputs) # Set up the decoder GRU, using `encoder_states` as initial state. decoder_gru = GRU(hidden_size*2, return_sequences=True, return_state=True, name='decoder_gru') decoder_out, decoder_state = decoder_gru( decoder_inputs, initial_state=Concatenate(axis=-1)([encoder_fwd_state, encoder_back_state]) ) # Attention layer attn_layer = AttentionLayer(name='attention_layer') attn_out, attn_states = attn_layer([encoder_out, decoder_out]) # Concat attention input and decoder GRU output decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_out, attn_out]) # Dense layer dense = Dense(self.num_decoder_tokens, activation='softmax', name='softmax_layer') dense_time = TimeDistributed(dense, name='time_distributed_layer') decoder_pred = dense_time(decoder_concat_input) # Full model self.model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_pred) self.model.compile(optimizer=tf.train.RMSPropOptimizer(learning_rate=0.01), loss='categorical_crossentropy') self.model.summary() """ Inference model """ batch_size = 1 """ Encoder (Inference) model """ encoder_inf_inputs = Input(batch_shape=(batch_size, enc_timesteps, self.glove_model.embedding_size), name='encoder_inf_inputs') encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state = encoder_gru(encoder_inf_inputs) self.encoder_model = Model(inputs=encoder_inf_inputs, outputs=[encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state]) """ Decoder (Inference) model """ decoder_inf_inputs = Input(batch_shape=(batch_size, 1, self.num_decoder_tokens), name='decoder_word_inputs') encoder_inf_states = Input(batch_shape=(batch_size, dec_timesteps, 2*hidden_size), name='encoder_inf_states') decoder_init_state = Input(batch_shape=(batch_size, 2*hidden_size), name='decoder_init') decoder_inf_out, decoder_inf_state = decoder_gru( decoder_inf_inputs, initial_state=decoder_init_state) attn_inf_out, attn_inf_states = attn_layer([encoder_inf_states, decoder_inf_out]) decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_inf_out, attn_inf_out]) decoder_inf_pred = TimeDistributed(dense)(decoder_inf_concat) self.decoder_model = Model(inputs=[encoder_inf_states, decoder_init_state, decoder_inf_inputs], outputs=[decoder_inf_pred, attn_inf_states, decoder_inf_state])
def define_nmt(hidden_size, batch_size, en_timesteps, en_vsize, fr_timesteps, fr_vsize): """ Defining a NMT model """ # Define an input sequence and process it. if batch_size: encoder_inputs = Input(batch_shape=(batch_size, en_timesteps, en_vsize), name='encoder_inputs') decoder_inputs = Input(batch_shape=(batch_size, fr_timesteps - 1, fr_vsize), name='decoder_inputs') else: encoder_inputs = Input(shape=(en_timesteps, en_vsize), name='encoder_inputs') decoder_inputs = Input(shape=(fr_timesteps - 1, fr_vsize), name='decoder_inputs') # Encoder GRU encoder_gru = Bidirectional(GRU(hidden_size, return_sequences=True, return_state=True, name='encoder_gru'), name='bidirectional_encoder') encoder_out, encoder_fwd_state, encoder_back_state = encoder_gru(encoder_inputs) # Set up the decoder GRU, using `encoder_states` as initial state. decoder_gru = Bidirectional(GRU(hidden_size, return_sequences=True, return_state=True, name='decoder_gru'), name='bidirectional_decoder') decoder_out, decoder_fwd_state, decoder_back_state = decoder_gru(decoder_inputs, initial_state=[encoder_fwd_state, encoder_back_state]) # Attention layer attn_layer = AttentionLayer(name='attention_layer') attn_out, attn_states = attn_layer([encoder_out, decoder_out]) # Concat attention input and decoder GRU output decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_out, attn_out]) # Dense layer dense = Dense(fr_vsize, activation='softmax', name='softmax_layer') dense_time = TimeDistributed(dense, name='time_distributed_layer') decoder_pred = dense_time(decoder_concat_input) # Full model full_model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_pred) full_model.compile(optimizer='adam', loss='categorical_crossentropy') full_model.summary() """ Inference model """ batch_size = 1 """ Encoder (Inference) model """ encoder_inf_inputs = Input(batch_shape=(batch_size, en_timesteps, en_vsize), name='encoder_inf_inputs') encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state = encoder_gru(encoder_inf_inputs) encoder_model = Model(inputs=encoder_inf_inputs, outputs=[encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state]) """ Decoder (Inference) model """ decoder_inf_inputs = Input(batch_shape=(batch_size, 1, fr_vsize), name='decoder_word_inputs') encoder_inf_states = Input(batch_shape=(batch_size, en_timesteps, 2*hidden_size), name='encoder_inf_states') decoder_init_fwd_state = Input(batch_shape=(batch_size, hidden_size), name='decoder_fwd_init') decoder_init_back_state = Input(batch_shape=(batch_size, hidden_size), name='decoder_back_init') decoder_inf_out, decoder_inf_fwd_state, decoder_inf_back_state = decoder_gru(decoder_inf_inputs, initial_state=[decoder_init_fwd_state, decoder_init_back_state]) attn_inf_out, attn_inf_states = attn_layer([encoder_inf_states, decoder_inf_out]) decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_inf_out, attn_inf_out]) decoder_inf_pred = TimeDistributed(dense)(decoder_inf_concat) decoder_model = Model(inputs=[encoder_inf_states, decoder_init_fwd_state, decoder_init_back_state, decoder_inf_inputs], outputs=[decoder_inf_pred, attn_inf_states, decoder_inf_fwd_state, decoder_inf_back_state]) return full_model, encoder_model, decoder_model
def getAdaptationModel(modelPath, adaptationVersion, features, seqLen): fineTuneModel = load_model(modelPath) # Test optimizer's state: #print(fineTuneModel.optimizer.get_config()) #print(dir(fineTuneModel.optimizer)) #print(fineTuneModel.optimizer.lr) fineTuneModel.get_layer('td').trainable = True if adaptationVersion == 2: fineTuneModel.get_layer('td').activation = 'relu' fineTuneModel.get_layer('rnn').trainable = False if fineTuneModel.get_layer('rnn_2nd_layer') != None: fineTuneModel.get_layer('rnn_2nd_layer').trainable = False fineTuneModel.get_layer('nn').trainable = False fineTuneModel.get_layer('nn_dropout').trainable = False fineTuneModel.get_layer('output_softmax').trainable = False if adaptationVersion == 3: fineTuneModel.get_layer('td').activation = 'relu' fineTuneModel.name = "existingModel" newModel = Sequential() newModel.add(TimeDistributed(Dense(features, kernel_initializer='identity', bias_initializer='zeros', activation='relu'), input_shape=(seqLen, features), name='td0', trainable=True)) newModel.add(fineTuneModel) fineTuneModel = newModel if adaptationVersion == 4: # initializer does not work with this initializer cause it is not square fineTuneModel.get_layer('td').activation = 'relu' fineTuneModel.name = "existingModel" newModel = Sequential() newModel.add(TimeDistributed(Dense(10*features, kernel_initializer='identity', bias_initializer='zeros', activation='relu'), input_shape=(seqLen, features), name='td0', trainable=True)) newModel.add(fineTuneModel) fineTuneModel = newModel if onTpu: multiFineTuneModel.compile(loss="categorical_crossentropy", optimizer=tf.train.AdamOptimizer(lr=0.001), metrics=["accuracy"]) multiFineTuneModel = toTpuModel(fineTuneModel) else: multiFineTuneModel = toMultiGpuModel(fineTuneModel) multiFineTuneModel.compile(loss="categorical_crossentropy", optimizer=optimizers.Adam(lr=0.001), metrics=["accuracy"]) # Test optimizer's state: #print(fineTuneModel.optimizer.get_config()) #print(dir(fineTuneModel.optimizer)) #print(fineTuneModel.optimizer.lr) return fineTuneModel, multiFineTuneModel
def build_model(self): """ Function to build the seq2seq model used. :return: Encoder model, decoder model (used for predicting) and full model (used for training). """ # Define model inputs for the encoder/decoder stack x_enc = Input(shape=(self.seq_len_in, self.input_feature_amount), name="x_enc") x_dec = Input(shape=(self.seq_len_out, self.output_feature_amount), name="x_dec") # Add noise x_dec_t = GaussianNoise(0.2)(x_dec) # Define the encoder GRU, which only has to return a state encoder_gru = GRU(self.state_size, return_sequences=True, return_state=True, name="encoder_gru") encoder_out, encoder_state = encoder_gru(x_enc) # Decoder GRU decoder_gru = GRU(self.state_size, return_state=True, return_sequences=True, name="decoder_gru") # Use these definitions to calculate the outputs of out encoder/decoder stack dec_intermediates, decoder_state = decoder_gru(x_dec_t, initial_state=encoder_state) # Define the attention layer attn_layer = AttentionLayer(name="attention_layer") attn_out, attn_states = attn_layer([encoder_out, dec_intermediates]) # Concatenate decoder and attn out decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([dec_intermediates, attn_out]) # Define the dense layer dense = Dense(self.output_feature_amount, activation='linear', name='output_layer') dense_time = TimeDistributed(dense, name='time_distributed_layer') decoder_pred = dense_time(decoder_concat_input) # Define the encoder/decoder stack model encdecmodel = tsModel(inputs=[x_enc, x_dec], outputs=decoder_pred) # Define the separate encoder model for inferencing encoder_inf_inputs = Input(shape=(self.seq_len_in, self.input_feature_amount), name="encoder_inf_inputs") encoder_inf_out, encoder_inf_state = encoder_gru(encoder_inf_inputs) encoder_model = tsModel(inputs=encoder_inf_inputs, outputs=[encoder_inf_out, encoder_inf_state]) # Define the separate encoder model for inferencing decoder_inf_inputs = Input(shape=(1, self.output_feature_amount), name="decoder_inputs") encoder_inf_states = Input(shape=(self.seq_len_in, self.state_size), name="encoder_inf_states") decoder_init_state = Input(shape=(self.state_size,), name="decoder_init") decoder_inf_out, decoder_inf_state = decoder_gru(decoder_inf_inputs, initial_state=decoder_init_state) attn_inf_out, attn_inf_states = attn_layer([encoder_inf_states, decoder_inf_out]) decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_inf_out, attn_inf_out]) decoder_inf_pred = TimeDistributed(dense)(decoder_inf_concat) decoder_model = tsModel(inputs=[encoder_inf_states, decoder_init_state, decoder_inf_inputs], outputs=[decoder_inf_pred, attn_inf_states, decoder_inf_state]) return encoder_model, decoder_model, encdecmodel
def encoder_identity_block(net, f, filters, stage, block, dropout_rate=DROPOUT_RATE): # Defining name basis conv_name_base = 'enc_res' + str(stage) + block + '_branch' bn_name_base = 'enc_bn' + str(stage) + block + '_branch' # Retrieve Filters f1, f2, f3 = filters # Save the input value net_shortcut = net # First component of main path net = TimeDistributed( ConvSN2D(filters=f1, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2a', kernel_initializer=glorot_uniform(seed=0)))(net) net = BatchNormalization(axis=-1, name=bn_name_base + '2a')(net) net = SwishLayer()(net) # Second component of main path net = TimeDistributed(Dropout(dropout_rate))(net) net = TimeDistributed( ConvSN2D(filters=f2, kernel_size=(f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b', kernel_initializer=glorot_uniform(seed=0)))(net) net = BatchNormalization(axis=-1, name=bn_name_base + '2b')(net) net = SwishLayer()(net) # Third component of main path net = TimeDistributed(Dropout(dropout_rate))(net) net = TimeDistributed( ConvSN2D(filters=f3, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c', kernel_initializer=glorot_uniform(seed=0)))(net) net = BatchNormalization(axis=-1, name=bn_name_base + '2c')(net) # Final step: Add shortcut value to main path, and pass it through a RELU activation net = Add()([net, net_shortcut]) net = SwishLayer()(net) return net
def make_decoder(self): reduced_inputs = layers.Input(shape=(self.time_steps, self.img_size, self.img_size, self.nb_channels_out), name='reduced_inputs') x = TimeDistributed(layers.Conv2D(self.interm1, (3, 3), activation=LeakyReLU(0.2), padding='same'))( reduced_inputs) x = TimeDistributed(BatchNormalization())(x) x = TimeDistributed(layers.Conv2D(self.interm2, (3, 3), activation=LeakyReLU(0.2), padding='same'))(x) x = TimeDistributed(BatchNormalization())(x) out = TimeDistributed(layers.Conv2D(self.nb_channels_in, (3, 3), activation=LeakyReLU(0.2), padding='same'), name='predicted_inputs')(x) return Model(inputs=reduced_inputs, outputs=out, name='decoder')
def __build_anchors(anchor_parameters, features, frames_per_batch=1): """Builds anchors for the shape of the features from FPN. Args: anchor_parameters (AnchorParameters): Parameters that determine how anchors are generated. features (list): The FPN features. frames_per_batch (int): Size of z axis in generated batches. If equal to 1, assumes 2D data. Returns: tensor: The anchors for the FPN features. The shape is: ``` (batch_size, num_anchors, 4) ``` """ if len(features) == 1: if frames_per_batch > 1: anchors = TimeDistributed( Anchors(size=anchor_parameters.sizes[0], stride=anchor_parameters.strides[0], ratios=anchor_parameters.ratios, scales=anchor_parameters.scales, name='anchors'))(features[0]) else: anchors = Anchors(size=anchor_parameters.sizes[0], stride=anchor_parameters.strides[0], ratios=anchor_parameters.ratios, scales=anchor_parameters.scales, name='anchors')(features[0]) return anchors else: if frames_per_batch > 1: anchors = [ TimeDistributed( Anchors(size=anchor_parameters.sizes[i], stride=anchor_parameters.strides[i], ratios=anchor_parameters.ratios, scales=anchor_parameters.scales, name='anchors_{}'.format(i)))(f) for i, f in enumerate(features) ] else: anchors = [ Anchors(size=anchor_parameters.sizes[i], stride=anchor_parameters.strides[i], ratios=anchor_parameters.ratios, scales=anchor_parameters.scales, name='anchors_{}'.format(i))(f) for i, f in enumerate(features) ] return Concatenate(axis=-2, name='anchors')(anchors)
def __init__(self, w3, dropout_rate): # w3 are the window sizes of the convolutions, hyperparameters super().__init__() self.logger = logging.getLogger(__name__) self.conv1 = TimeDistributed( Conv1D(1, w3, activation=None, padding='causal', name='atn_weight_conv1')) self.dropout = Dropout(dropout_rate) self.softmax = TimeDistributed(Softmax(name='atn_weight_softmax'))
def __init__(self, params=None, is_training=False): super(Attention, self).__init__() self.is_training = is_training self.max_len = params['max_len'] self.emb_dim = params['emb_dim'] self.attn_weights = TimeDistributed(Dense(self.emb_dim, activation='softmax'), name='attn_weights') self.attn_out = TimeDistributed(Dense(self.emb_dim, activation=None), name='attn_out')
def layers(self): input_layer = Input(self.real_input_shape, self.batch_size) # 64x64x3 # Zero-Padding net = TimeDistributed(ZeroPadding2D((3, 3)))(input_layer) # 70x70x3 # Stage 0 net = TimeDistributed(Conv2D(64, (7, 7), strides=(2, 2), name='conv0', kernel_initializer=glorot_uniform(seed=0)))(net) net = BatchNormalization(axis=-1, name='bn_conv0')(net) net = Activation('relu')(net) # 32x32x64 # Stage 1 net = encoder_convolutional_block(net, f=3, filters=[64, 64, 128], stage=1, block='a', s=1) # net = encoder_identity_block(net, 3, [64, 64, 128], stage=1, block='b') # 32x32x64 self.mean_32x32x64 = encoder_residual_block(net, depth=64) self.stddev_32x32x64 = encoder_residual_block(net, depth=64) # Stage 2 net = encoder_convolutional_block(net, f=3, filters=[64, 64, 128], stage=2, block='a', s=2) # net = encoder_identity_block(net, 3, [64, 64, 128], stage=2, block='b') # 16x16x128 self.mean_16x16x128 = encoder_residual_block(net, depth=128) self.stddev_16x16x128 = encoder_residual_block(net, depth=128) # Stage 3 net = encoder_convolutional_block(net, f=3, filters=[128, 128, 256], stage=3, block='a', s=2) # net = encoder_identity_block(net, 3, [128, 128, 256], stage=3, block='b') # 8x8x256 self.mean_8x8x256 = encoder_residual_block(net, depth=256) self.stddev_8x8x256 = encoder_residual_block(net, depth=256) # Stage 4 net = encoder_convolutional_block(net, f=3, filters=[256, 256, 512], stage=4, block='a', s=2) # net = encoder_identity_block(net, 3, [256, 256, 512], stage=4, block='b') # 4x4x512 self.mean_4x4x512 = encoder_residual_block(net, depth=512) self.stddev_4x4x512 = encoder_residual_block(net, depth=512) return input_layer, [self.mean_4x4x512, self.stddev_4x4x512, self.mean_8x8x256, self.stddev_8x8x256, self.mean_16x16x128, self.stddev_16x16x128, self.mean_32x32x64, self.stddev_32x32x64]
def LocalMultiTaskLayer(x, input_type): classes_root = 35 if input_type.startswith( 'spelling') else 12 # the twelve notes without enharmonic duplicates classes_quality = 12 # ['M', 'm', 'd', 'a', 'M7', 'm7', 'D7', 'd7', 'h7', 'Gr+6', 'It+6', 'Fr+6'] classes_inversion = 4 # root position, 1st, 2nd, and 3rd inversion (the last only for seventh chords) o_qlt = TimeDistributed(Dense(classes_quality, activation='softmax'), name='quality')(x) o_inv = TimeDistributed(Dense(classes_inversion, activation='softmax'), name='inversion')(x) o_roo = TimeDistributed(Dense(classes_root, activation='softmax'), name='root')(x) return [o_qlt, o_inv, o_roo]
def ProgressionMultiTaskLayer(x, input_type): classes_key = 30 if input_type.startswith( 'spelling') else 24 # Major keys: 0-11, Minor keys: 12-23 classes_degree = 21 # 7 degrees * 3: regular, diminished, augmented o_key = TimeDistributed(Dense(classes_key, activation='softmax'), name='key')(x) z = Concatenate()([x, o_key]) o_dg1 = TimeDistributed(Dense(classes_degree, activation='softmax'), name='degree_1')(z) o_dg2 = TimeDistributed(Dense(classes_degree, activation='softmax'), name='degree_2')(z) return [o_key, o_dg1, o_dg2]
def variant_default(cls, x_train, y_train, x_val, y_val, params): model = Sequential() model.add( TimeDistributed( Flatten(input_shape=(x_train.shape[1], x_train.shape[2] * x_train.shape[3])))) model.add( TimeDistributed( Flatten(input_shape=(x_train.shape[1], x_train.shape[2] * x_train.shape[3])))) for i in range(params["lstm_layers"] - 1): model.add( LSTM( params["hidden_size"], return_sequences=True, activation=params["activation"], )) model.add(LSTM(params["hidden_size"], activation=params["activation"])) if params["dropout"] > 0: model.add(Dropout(params["dropout"])) model.add(Dense(10, activation=params["output_activation"])) model.compile( optimizer=tf.keras.optimizers.SGD(lr=params["lr"], momentum=params["momentum"]), loss="sparse_categorical_crossentropy", metrics=["accuracy"], ) history = model.fit( x_train, y_train, validation_data=(x_val, y_val), epochs=params["epochs"], batch_size=params["batch_size"], verbose=1, callbacks=[ tf.keras.callbacks.TensorBoard( "./logs/" + "lstm_default/" + "-".join("=".join((str(k), str(v))) for k, v in params.items()) + "-ts={}".format(str(time.time()))) ], ) return history, model
def seq_2_seq_biLSTM_att(X_embedding, MAX_LEN, num_words, EMBEDDING_DIM, LSTM_units, LSTM_dropout): # Encoder # [?, 100] encoder_inputs = Input(shape=(MAX_LEN,)) # [?, 100, 300] encoder_embedding = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM, input_length = MAX_LEN, embeddings_initializer=Constant(X_embedding), trainable=False)(encoder_inputs) # LSTM encoder_lstm = Bidirectional(LSTM(units=LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout)) # [?, 100, 300] encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder_lstm(encoder_embedding) # [?, 300] state_h = concatenate([forward_h, backward_h]) state_c = concatenate([forward_c, backward_c]) encoder_states = [state_h, state_c] # Decoder # [?, 30] decoder_inputs = Input(shape=(None,)) decoder_embedding_layer = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM, trainable=True) # [?, 30, 300] decoder_embedding = decoder_embedding_layer(decoder_inputs) decoder_lstm = LSTM(units=2*LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout) # [?, 30, 300] decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states) # [?, 30, 100] attention_weight = dot([decoder_outputs, encoder_outputs], axes=[2, 2]) attention = Activation('softmax')(attention_weight) # [?, 30, 300] context = dot([attention, encoder_outputs], axes=[2,1]) #[?, 100, 300] = dot([?,?,100] , [?, 100, 300]) # [?, 30, 600] decoder_combined_context = concatenate([context, decoder_outputs]) # [?, 30, 64] att_output = TimeDistributed(Dense(128, activation="tanh"))(decoder_combined_context) # [?, 30, 39093] output = TimeDistributed(Dense(num_words, activation="softmax"))(att_output) model = Model(inputs=[encoder_inputs,decoder_inputs], outputs=output) return model
def build_fpn_mask_graph(rois, feature_maps, image_shape, pool_size, num_classes): """Builds the computation graph of the mask head of Feature Pyramid Network. rois: [batch, num_rois, (y1, x1, y2, x2)] Proposal boxes in normalized coordinates. feature_maps: List of feature maps from diffent layers of the pyramid, [P2, P3, P4, P5]. Each has a different resolution. image_shape: [height, width, depth] pool_size: The width of the square feature map generated from ROI Pooling. num_classes: number of classes, which determines the depth of the results Returns: Masks [batch, roi_count, height, width, num_classes] """ # ROI Pooling # Shape: [batch, boxes, pool_height, pool_width, channels] x = PyramidROIAlign([pool_size, pool_size], image_shape, name="roi_align_mask")([rois] + feature_maps) # Conv layers x = TimeDistributed(Conv2D(256, (3, 3), padding="same"), name="mrcnn_mask_conv1")(x) x = TimeDistributed(BatchNorm(axis=3), name='mrcnn_mask_bn1')(x) x = Activation('relu')(x) x = TimeDistributed(Conv2D(256, (3, 3), padding="same"), name="mrcnn_mask_conv2")(x) x = TimeDistributed(BatchNorm(axis=3), name='mrcnn_mask_bn2')(x) x = Activation('relu')(x) x = TimeDistributed(Conv2D(256, (3, 3), padding="same"), name="mrcnn_mask_conv3")(x) x = TimeDistributed(BatchNorm(axis=3), name='mrcnn_mask_bn3')(x) x = Activation('relu')(x) x = TimeDistributed(Conv2D(256, (3, 3), padding="same"), name="mrcnn_mask_conv4")(x) x = TimeDistributed(BatchNorm(axis=3), name='mrcnn_mask_bn4')(x) x = Activation('relu')(x) x = TimeDistributed(Conv2DTranspose(256, (2, 2), strides=2, activation="relu"), name="mrcnn_mask_deconv")(x) x = TimeDistributed(Conv2D(num_classes, (1, 1), strides=1, activation="sigmoid"), name="mrcnn_mask")(x) return x
def CRNN(input_shape): Input_Tr = Input(input_shape, dtype='float', name='Input_Tr') conv_layer1 = Conv2D(32, kernel_size=3, strides=1, padding='SAME')(Input_Tr) batch_layer1 = BatchNormalization(axis=-1)(conv_layer1) conv_layer1_out = Activation('relu')(batch_layer1) pooling_layer1 = MaxPooling2D((1, 4))(conv_layer1_out) dropout_layer1 = Dropout(0.5)(pooling_layer1) conv_layer2 = Conv2D(64, kernel_size=3, strides=1, padding='SAME')(dropout_layer1) batch_layer2 = BatchNormalization(axis=-1)(conv_layer2) conv_layer2_out = Activation('relu')(batch_layer2) pooling_layer2 = MaxPooling2D((1, 4))(conv_layer2_out) dropout_layer2 = Dropout(0.5)(pooling_layer2) print(dropout_layer2.shape) reshape_layer3 = Reshape( (600, 64 * int(round(n_mel / 4 / 4))))(dropout_layer2) print(reshape_layer3.shape) bidir_layer3 = Bidirectional( GRU(64, return_sequences=True, activation='tanh'))(reshape_layer3) output = TimeDistributed(Dense(1, activation='sigmoid'))(bidir_layer3) model = Model(inputs=[Input_Tr], outputs=[output]) return model
def create_model(steps_before, steps_after, feature_count): """ creates, compiles and returns a RNN model @param steps_before: the number of previous time steps (input) @param steps_after: the number of posterior time steps (output or predictions) @param feature_count: the number of features in the model @param hidden_neurons: the number of hidden neurons per LSTM layer """ DROPOUT = 0.5 LAYERS = 2 hidden_neurons = 300 model = Sequential() model.add( LSTM(input_dim=feature_count, output_dim=hidden_neurons, return_sequences=False)) model.add(RepeatVector(steps_after)) model.add(LSTM(output_dim=hidden_neurons, return_sequences=True)) model.add(TimeDistributed(Dense(feature_count))) model.add(Activation('linear')) model.compile(loss='mean_squared_error', optimizer='rmsprop', metrics=['accuracy']) return model