def mnist_irnn_model(inputShape, nb_classes): # inputShape 2dim model = Sequential() model.add(SimpleRNN(output_dim=100, init=lambda shape, name: normal(shape, scale=0.001, name=name), inner_init=lambda shape, name: identity(shape, scale=1.0, name=name), activation='relu', input_shape=inputShape)) model.add(Dense(nb_classes)) model.add(Activation('softmax')) model.summary()
def irnn(shape_orig): hidden_units = 100 learning_rate = 1e-6 model = Sequential() model.add( SimpleRNN( output_dim=hidden_units, init=lambda shape, name: normal(shape, scale=0.001, name=name), inner_init=lambda shape, name: identity( shape, scale=1.0, name=name), activation='relu', input_shape=shape_orig)) model.add(Dense(nb_classes)) model.add(Activation('softmax')) rmsprop = RMSprop(lr=learning_rate) model.compile(loss='categorical_crossentropy', optimizer=rmsprop, metrics=['accuracy']) return model
def buildAttention(self): q_relu = self.tensors['q_relu'] a_relu = self.tensors['a_relu'] with tf.name_scope("attention"): W = identity([self.params['nb_filter'], self.params['nb_filter']], name='W') batch = tf.shape(q_relu)[0] q_matmul = tf.batch_matmul(q_relu, tf.tile(tf.expand_dims(W,[0]), tf.pack([batch, tf.constant(1), tf.constant(1)]))) qa_attention = tf.batch_matmul(q_matmul, a_relu, adj_x=False, adj_y=True, name="attention") # shape = (batch, q_length, 1) qa_attention = tf.tanh(qa_attention) q_max = tf.reduce_max(qa_attention, reduction_indices=[2], keep_dims=True, name='q_max') # shape = (batch, 1, a_length) a_max = tf.reduce_max(qa_attention, reduction_indices=[1], keep_dims=True, name='a_max') # shape = (batch, q_length, 1) q_softmax = tf.expand_dims(tf.nn.softmax(tf.squeeze(q_max, [2])), -1) # shape = (batch, a_length, 1) a_softmax = tf.expand_dims(tf.nn.softmax(tf.squeeze(a_max, [1])), -1) # https://www.tensorflow.org/versions/r0.9/api_docs/python/math_ops.html#batch_matmul # shape = (batch, NUM_FILTERS, 1) q_feature = tf.batch_matmul(q_relu, q_softmax, adj_x=True, adj_y=False) a_feature = tf.batch_matmul(a_relu, a_softmax, adj_x=True, adj_y=False) self.tensors['q_feature'] = q_feature self.tensors['a_feature'] = a_feature self.tensors.setdefault('weights', []).append(W)
X_test = X_test.astype('float32') X_train /= 255 X_test /= 255 print('X_train shape:', X_train.shape) print(X_train.shape[0], 'train samples') print(X_test.shape[0], 'test samples') # convert class vectors to binary class matrices Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) print('Evaluate IRNN...') model = Sequential() model.add(SimpleRNN(output_dim=hidden_units, init=lambda shape, name: normal(shape, scale=0.001, name=name), inner_init=lambda shape, name: identity(shape, scale=1.0, name=name), activation='relu', input_shape=X_train.shape[1:])) model.add(Dense(nb_classes)) model.add(Activation('softmax')) rmsprop = RMSprop(lr=learning_rate) model.compile(loss='categorical_crossentropy', optimizer=rmsprop, metrics=['accuracy']) model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epochs, verbose=1, validation_data=(X_test, Y_test)) scores = model.evaluate(X_test, Y_test, verbose=0) print('IRNN test score:', scores[0]) print('IRNN test accuracy:', scores[1])
y_train= to_categorical(y_train1) y_test= to_categorical(y_test1) # reshape input to be [samples, time steps, features] X_train = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) X_test = np.reshape(testT, (testT.shape[0], 1, testT.shape[1])) batch_size = 64 learning_rate = 0.1 # 1. define the network model = Sequential() model.add(SimpleRNN(64,init=lambda shape, name: normal(shape, scale=0.001, name=name),inner_init=lambda shape, name: identity(shape, scale=1.0, name=name),input_dim=22,activation='relu',return_sequences=True)) # try using a GRU instead, for fun model.add(SimpleRNN(64,activation='relu',return_sequences=True)) model.add(SimpleRNN(64,activation='relu',return_sequences=True)) model.add(SimpleRNN(64,activation='relu',return_sequences=True)) model.add(SimpleRNN(64,activation='relu',return_sequences=True)) model.add(SimpleRNN(64,activation='relu',return_sequences=False)) model.add(Dense(11)) model.add(Activation('softmax')) rmsprop = RMSprop(lr=learning_rate) # try using different optimizers and different optimizer configs model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy']) checkpointer = callbacks.ModelCheckpoint(filepath="kddresults/lstm6layer/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='val_acc',mode='max') csv_logger = CSVLogger('training_set_iranalysis6.csv',separator=',', append=False) model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=1000, validation_data=(X_test, y_test),callbacks=[checkpointer,csv_logger]) model.save("kddresults/lstm6layer/lstm1layer_model.hdf5")
class LossHistory(keras.callbacks.Callback): def on_train_begin(self, logs={}): self.losses = [] self.times = [] self.start_time = time.time() def on_epoch_end(self, batch, logs={}): self.times.append(time.time()-self.start_time) self.losses.append(logs.get("val_acc")) history = LossHistory() print('Evaluate %s...' % ClassModel.__class__.__name__) model = Sequential() model.add(ClassModel(input_dim=1, output_dim=hidden_units, init=lambda shape: normal(shape, scale=0.001), inner_init=lambda shape: identity(shape, scale=1.0), activation='relu', truncate_gradient=BPTT_truncate)) model.add(Dense(hidden_units, nb_classes)) model.add(Activation('sigmoid')) rmsprop = RMSprop(lr=learning_rate) model.compile(loss='categorical_crossentropy', optimizer=rmsprop) model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epochs, show_accuracy=True, verbose=1, validation_data=(X_test, Y_test), callbacks=[history]) scores = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0) print('%s test score:' % ClassModel.__class__.__name__, scores[0]) print('%s test accuracy:' % ClassModel.__class__.__name__, scores[1]) record_file = FileIO() record_file.save_pickle(history.losses, "%s_record_titan_x_losses" % ClassModel.__class__.__name__.lower())
X_train /= 255 X_test /= 255 print('X_train shape:', X_train.shape) print(X_train.shape[0], 'train samples') print(X_test.shape[0], 'test samples') # convert class vectors to binary class matrices Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) print('Evaluate IRNN...') model = Sequential() model.add( SimpleRNN(output_dim=hidden_units, init=lambda shape: normal(shape, scale=0.001), inner_init=lambda shape: identity(shape, scale=1.0), activation='relu', truncate_gradient=BPTT_truncate, input_shape=(None, 1))) model.add(Dense(nb_classes)) model.add(Activation('softmax')) rmsprop = RMSprop(lr=learning_rate) model.compile(loss='categorical_crossentropy', optimizer=rmsprop) model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epochs, show_accuracy=True, verbose=1, validation_data=(X_test, Y_test))
T = [[valid_chars[y] for y in x] for x in T] X_test = sequence.pad_sequences(T, maxlen=max_len) y_train = np.array(trainlabel) y_test = np.array(testlabel) embedding_vecor_length = 128 # 1. define the network model = Sequential() model.add(Embedding(max_features, embedding_vecor_length, input_length=max_len)) model.add(SimpleRNN(128,init=lambda shape, name: normal(shape, scale=0.001, name=name),inner_init=lambda shape, name: identity(shape, scale=1.0, name=name),activation='relu',)) # try using a GRU instead, for fun #model.add(Dropout(0.1)) model.add(Dense(1)) model.add(Activation('sigmoid')) rmsprop = RMSprop(lr=learning_rate) model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy']) checkpointer = callbacks.ModelCheckpoint(filepath="logs/irnn/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='val_acc',mode='max') csv_logger = CSVLogger('logs/irnn/training_set_irnnanalysis.csv',separator=',', append=False) model.fit(X_train, y_train, batch_size=32, nb_epoch=1000,validation_split=0.33, shuffle=True,callbacks=[checkpointer,csv_logger]) score, acc = model.evaluate(X_test, y_test, batch_size=32) print('Test score:', score) print('Test accuracy:', acc)