def __built_multi_cell_Layer(self): """ :return: the output tensor. """ o1 = RNN(MinimalRNNCell(32, "tanh"), return_sequences=True)(self.input) o1 = BatchNormalization(1)(o1) o2 = RNN(MinimalRNNCell(32, "tanh"), return_sequences=True)(o1) o2 = BatchNormalization(1)(o2) o3 = RNN(MinimalRNNCell(32, "tanh"), return_sequences=True)(o2) o3 = BatchNormalization(1)(o3) o4 = RNN(MinimalRNNCell(32, "tanh"), return_sequences=False)(o3) o5 = Dense(21, activation="relu")(o4) self.output = o5
def get_test_model(seq_len, stacked_features_size, edges_features_matrix_depth, neighbourhood_size, units): # INPUTS stacked_base_features_input = Input(shape=(seq_len, stacked_features_size), name='stacked_base_features') adjacency_matrix_input = Input(shape=(seq_len, seq_len), name='adjacency_matrix') edges_features_matrix_input = Input(shape=(seq_len, seq_len, edges_features_matrix_depth), name='edges_features_matrix') inputs = (stacked_base_features_input, adjacency_matrix_input, edges_features_matrix_input) # ACTUAL MODEL x = Subgraphing(neighbourhood_size)(inputs) x = RNN(GraphReduceCell(units), return_sequences=True)(x) # OUTPUTS reactivity_pred = TimeDistributed(Dense(1), name='reactivity')(x) deg_Mg_pH10_pred = TimeDistributed(Dense(1), name='deg_Mg_pH10')(x) deg_Mg_50C_pred = TimeDistributed(Dense(1), name='deg_Mg_50C')(x) scored_outputs = [reactivity_pred, deg_Mg_pH10_pred, deg_Mg_50C_pred] stacked_outputs = Concatenate(axis=2, name='stacked_outputs')(scored_outputs) # MODEL DEFINING model = Model(inputs=inputs, outputs={'stacked_scored_labels': stacked_outputs}, name='graph_reduce_model') return model
def set_model(self): input_layer = Input(shape=(self.seq_len, self.items_len,)) rnn_part = RNN(ExtendedMemoryRNNCell(f_block_num_layers=self.f_block_num_layers, f_block_units=self.f_block_units, s_block_num_layers=self.s_block_num_layers, s_block_units=self.s_block_units, state_shape=self.state_shape, batch_size=self.batch_size), return_sequences=True)( input_layer ) layer1_1 = TimeDistributed(Dense(units=64, activation='relu'))(rnn_part) layer1_2 = TimeDistributed(Dense(units=64, activation='sigmoid'))(layer1_1) y1_output = TimeDistributed(Dense(units=1, activation='sigmoid'), name='y1_output')(layer1_2) layer2_1 = TimeDistributed(Dense(units=64, activation='relu'))(rnn_part) dropout_1 = Dropout(0.2)(layer2_1) layer2_2 = TimeDistributed(Dense(units=64, activation='relu'))(dropout_1) layer2_3 = TimeDistributed(Dense(units=64, activation='relu'))(layer2_2) y2_output = TimeDistributed(Dense(units=self.features_range, activation='softmax'), name='y2_output')(layer2_3) # Define the model with the input layer and a list of output layers model = tf.keras.models.Model(inputs=input_layer, outputs=[y1_output, y2_output]) # optimizer = tf.keras.optimizers.SGD(lr=0.001) optimizer = tf.keras.optimizers.Adam(clipvalue=0.5) model.compile(optimizer=optimizer, loss={'y1_output': 'binary_crossentropy', 'y2_output': self.missed_value_loss}, metrics={'y1_output': 'accuracy', 'y2_output': self.missed_value_acc}) print(model.summary()) return model
def add_lstm(self, inputs): if 'lstm_conf' in self.nn_config: lstm_conf = self.nn_config['lstm_conf'] activation = None if lstm_conf['batch_norm'] else lstm_conf[ 'lstm_activation'] return_seq = True if '1dCNN' in self.nn_config else False cell = LSTMCell(units=lstm_conf['lstm_units'], activation=activation) if lstm_conf['method'] == 'dynamic_rnn': rnn_outputs1, states = dynamic_rnn(cell, inputs, dtype=tf.float32) lstm_outputs = tf.reshape(rnn_outputs1[:, -1, :], [-1, lstm_conf['lstm_units']]) elif lstm_conf['method'] == 'keras_lstm_layer': lstm_outputs = LSTM(lstm_conf['lstm_units'], name="first_lstm", activation=activation, input_shape=(self.data_config['lookback'], self.ins), return_sequences=return_seq)(inputs) else: rnn_layer = RNN(cell, return_sequences=return_seq) lstm_outputs = rnn_layer(inputs) # [batch_size, neurons] if self.verbose > 0: print(lstm_outputs.shape, 'before reshaping', K.eval(tf.rank(lstm_outputs))) lstm_outputs = tf.reshape(lstm_outputs[:, :], [-1, lstm_conf['lstm_units']]) if self.verbose > 0: print(lstm_outputs.shape, 'after reshaping', K.eval(tf.rank(lstm_outputs))) if lstm_conf['batch_norm']: rnn_outputs3 = BatchNormalization()(lstm_outputs) lstm_outputs = Activation('relu')(rnn_outputs3) if lstm_conf['dropout'] is not None: lstm_outputs = Dropout(lstm_conf['dropout'])(lstm_outputs) else: lstm_outputs = inputs return lstm_outputs
def make_model(batch_size=None): source = Input(shape=(maxlen, ), batch_size=batch_size, dtype=tf.int32, name='Input') embedding = Embedding(input_dim=max_features, output_dim=embedding_size, input_length=X.shape[1], name='Embedding')(source) drop = SpatialDropout1D(0.5)(embedding) #rnn = Bidirectional(LSTM(lstm_out, name = 'LSTM',dropout=0.50, recurrent_dropout=0.50))(drop) rnn = RNN(lstm_out, name='RNN', dropout=0.40, recurrent_dropout=0.40)(drop) predicted_var = Dense(2, activation='sigmoid', name='Output')(rnn) model = tf.keras.Model(inputs=[source], outputs=[predicted_var]) model.compile( #optimizer='rmsprop', optimizer=tf.keras.optimizers.RMSprop(decay=1e-3), loss='categorical_crossentropy', metrics=['acc']) return model
def lstm_gaussian(batch_shape, h_dim=10, n_layers=2, units=256, reparameterize=False, name=None, reg_lambda=0.0, **kwargs): def make_cell(lstm_size): return LSTMCell(lstm_size, activation='tanh', kernel_initializer='he_normal', recurrent_regularizer=l2(reg_lambda)) # ===== Define the lstm model lstm_cells = [make_cell(units) for _ in range(n_layers)] lstm = RNN(lstm_cells, return_sequences=True, return_state=True, name='lstm_model') embed_net = Dense(units=units, activation='linear') sample = Sample(output_dim=h_dim, reparameterization_flag=reparameterize) _in = Input(batch_shape=[batch_shape[0], None, batch_shape[-1]]) initial_state = initial_state_placeholder(units, n_layers, batch_size=batch_shape[0]) embed = TimeDistributed(embed_net)(_in) out = lstm(embed, initial_state=initial_state) h, state = out[0], out[1:] # z, mu, logvar = sample(h) z, mu, logvar = TimeDistributed(sample)(h) model = Model(inputs=[_in, initial_state], outputs=[z, mu, logvar, state], name=name) return model
def simple_lstm(batch_shape, h_dim=10, n_layers=2, units=256, name=None, reg_lambda=0.0, **kwargs): def make_cell(lstm_size): return LSTMCell(lstm_size, activation='tanh', kernel_initializer='glorot_uniform', unit_forget_bias=False, recurrent_regularizer=l2(reg_lambda)) # ===== Define the lstm model lstm_cells = [make_cell(units) for _ in range(n_layers)] lstm = RNN(lstm_cells, return_sequences=True, return_state=True) embed_net = Dense(units=units, activation='linear') output_net = Dense(units=h_dim, activation='tanh') _in = Input(batch_shape=[batch_shape[0], None, batch_shape[-1]]) initial_state = initial_state_placeholder(units, n_layers, batch_size=batch_shape[0]) embed = TimeDistributed(embed_net)(_in) # embed = BatchNormalization()(embed) # --> !!!!!! out = lstm(embed, initial_state=initial_state) predictions, state = out[0], out[1:] predictions = TimeDistributed(output_net)(predictions) model = Model(inputs=[_in, initial_state], outputs=[predictions, state], name=name) return model
def add_model(self, input_data, target_data=None): """Implements core of model that transforms input_data into predictions. The core transformation for this model which transforms a batch of input data into a batch of predictions. Args: input_data: A tensor of shape (batch_size, num_steps, time_stamps). target_data: A tensor of shape (batch_size, num_steps, time_stamps). Returns: predict: A tensor of shape (batch_size, num_steps, time_stamps) """ with tf.variable_scope('embedding_layer'): input_embeddings = TimeDistributed( Dense(self.config.rnn_size, dtype=tf.float32, activity_regularizer=self.config.regularizer, kernel_regularizer=self.config.regularizer, bias_regularizer=self.config.regularizer), input_shape=(self.config.num_steps, self.config.time_stamps), dtype=tf.float32, activity_regularizer=self.config.regularizer)( self.input_placeholder) input_embeddings = tf.keras.layers.Dropout( self.config.dropout)(input_embeddings) with tf.variable_scope('rnn_layer'): if self.config.model_type == 'gru': cell_fun = GRUCell elif self.config.model_type == 'lstm': cell_fun = LSTMCell else: cell_fun = SimpleRNNCell cell = cell_fun(self.config.rnn_size, dtype=tf.float32, activity_regularizer=self.config.regularizer, bias_regularizer=self.config.regularizer, recurrent_regularizer=self.config.regularizer, kernel_regularizer=self.config.regularizer) cell = StackedRNNCells([cell] * self.config.num_layers) # outputs: (batch_size, num_steps * 2, rnn_size) outputs = Bidirectional( RNN(cell, return_sequences=True, dtype=tf.float32))(input_embeddings) # output: (batch_size * (num_steps * 2), rnn_size) output = tf.reshape(outputs, [-1, self.config.rnn_size]) output = tf.keras.layers.Dropout(self.config.dropout)(output) with tf.variable_scope('output_layer'): # weights: (rnn_size, time_stamps) weights = tf.Variable(tf.truncated_normal( [self.config.rnn_size, self.config.time_stamps], dtype=tf.float32, ), dtype=tf.float32) # bias: (time_stamps, ) bias = tf.Variable(tf.zeros(shape=[self.config.time_stamps], dtype=tf.float32), dtype=tf.float32) # output: (batch_size * (num_steps * 2), time_stamps) predict_all = tf.nn.bias_add(tf.matmul(output, weights), bias=bias) predict_up = tf.slice(predict_all, [0, 0], [ self.config.batch_size * self.config.num_steps, self.config.time_stamps ]) predict_down = tf.slice( predict_all, [self.config.batch_size * self.config.num_steps, 0], [ self.config.batch_size * self.config.num_steps, self.config.time_stamps ]) predict = (predict_up + predict_down) / 2 return predict
def add_model(self, input_data): """Implements core of model that transforms input_data into predictions. The core transformation for this model which transforms a batch of input data into a batch of predictions. Args: input_data: A tensor of shape (batch_size, num_steps, time_stamps). Returns: predict: A tensor of shape (batch_size, num_steps, time_stamps) """ with tf.variable_scope('conv_layer'): # add dimension, input_data: (batch_size, num_steps, time_stamps, channels=1) input_data = tf.expand_dims(input_data, -1) max_filters = self.config.num_steps # conv1: (batch_size, W1, H1, F1=32) conv1 = Conv2D(filters=max_filters, kernel_size=(3, 3), padding='same', activation='relu')(input_data) # W2 = (W1 - F) / S + 1 # H2 = (H1 - F) / S + 1 # conv1: (batch_size, W2, H2, F1) conv1 = MaxPooling2D(pool_size=(2, 2))(conv1) conv1 = tf.reshape(conv1, (self.config.batch_size, self.config.num_steps, -1)) with tf.variable_scope('embedding_layer'): input_embeddings = TimeDistributed( Dense(self.config.rnn_size, dtype=tf.float64, activation='tanh', activity_regularizer=self.config.regularizer, kernel_regularizer=self.config.regularizer, bias_regularizer=self.config.regularizer), input_shape=(self.config.num_steps, self.config.time_stamps), dtype=tf.float64, activity_regularizer=self.config.regularizer)( conv1) # embeddings: (batch_size, num_steps, rnn_size) input_embeddings = tf.keras.layers.Dropout(self.config.dropout)(input_embeddings) with tf.variable_scope('rnn_layer'): if self.config.model_type == 'gru': cell_fun = GRUCell elif self.config.model_type == 'lstm': cell_fun = LSTMCell else: cell_fun = SimpleRNNCell cell = cell_fun(self.config.rnn_size, dtype=tf.float64, activity_regularizer=self.config.regularizer, bias_regularizer=self.config.regularizer, recurrent_regularizer=self.config.regularizer, kernel_regularizer=self.config.regularizer) cell = StackedRNNCells([cell] * self.config.num_layers) # outputs: (batch_size, num_steps * 2, rnn_size) outputs = Bidirectional(RNN(cell, return_sequences=True, dtype=tf.float64))(input_embeddings) # output: (batch_size * (num_steps * 2), rnn_size) output = tf.reshape(outputs, [-1, self.config.rnn_size]) output = tf.keras.layers.Dropout(self.config.dropout)(output) with tf.variable_scope('output_layer'): # weights: (rnn_size, time_stamps) weights = tf.Variable( tf.truncated_normal([self.config.rnn_size, self.config.time_stamps], dtype=tf.float64, ), dtype=tf.float64) # bias: (time_stamps, ) bias = tf.Variable(tf.zeros(shape=[self.config.time_stamps], dtype=tf.float64), dtype=tf.float64) # output: (batch_size * (num_steps * 2), time_stamps) predict_all = tf.nn.bias_add(tf.matmul(output, weights), bias=bias) predict_all = tf.nn.tanh(predict_all) predict_up = tf.slice(predict_all, [0, 0], [self.config.batch_size * self.config.num_steps, self.config.time_stamps]) predict_down = tf.slice(predict_all, [self.config.batch_size * self.config.num_steps, 0], [self.config.batch_size * self.config.num_steps, self.config.time_stamps]) predict = (predict_up + predict_down) / 2 self.global_step = tf.Variable(0, trainable=False, name='global_step') return predict
xval = [k for k in train_val['audio_embedding']] yval = train_val['is_turkey'].values # Pad the audio features so that all are "10 seconds" long x_train = pad_sequences(xtrain, maxlen=10) x_val = pad_sequences(xval, maxlen=10) y_train = np.asarray(ytrain) y_val = np.asarray(yval) #Define a basic LSTM model model = Sequential() model.add(BatchNormalization(input_shape=(10, 128))) model.add(Dropout(.5)) model.add(Bidirectional(RNN(128, activation='relu'))) model.add(Dense(1, activation='sigmoid')) #maybe there is something better to use, but let's use binary_crossentropy model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) #fit on a portion of the training data, and validate on the rest model.fit(x_train, y_train, batch_size=300, nb_epoch=4,validation_data=(x_val, y_val)) model.save('./turkeydetection-RNN.h5') # Get accuracy of model on validation data. It's not AUC but it's something at least! score, acc = model.evaluate(x_val, y_val, batch_size=300)