#%% print('Build model...') # Inputs review_input = Input(shape=(maxsents,maxlen), dtype='int32') # Embedding Layer embedding_layer = Embedding(max_features, embedding_dims, input_length=maxlen) # WORD-LEVEL sentence_input = Input(shape=(maxlen,), dtype='int32') embedded_sequences = embedding_layer(sentence_input) # Bidirectional GRU l_gru = Bidirectional(GRU(gru_output_size, return_sequences=True))(embedded_sequences) l_dense = TimeDistributed(Dense(units=gru_output_size))(l_gru) # Word-Level Attention Layer l_att = AttLayer()(l_dense) sentEncoder = Model(sentence_input, l_att) review_encoder = TimeDistributed(sentEncoder)(review_input) # SENTENCE_LEVEL # Bidirectional GRU l_gru_sent = Bidirectional(GRU(gru_output_size, return_sequences=True))(review_encoder) l_dense_sent = TimeDistributed(Dense(units=gru_output_size))(l_gru_sent) # Sentence-Level Attention Layer postp = AttLayer()(l_dense_sent) # Embedding Average sentEmbed = Model(sentence_input, embedded_sequences)
def construct_model(label_name): output_parameters() model = Sequential() # mask_zero 在 MaxPooling 层中不能支持 model.add( Embedding(creative_id_window, embedding_size, input_length=max_len)) if model_type == 'MLP': model.add(Flatten()) model.add(Dropout(0.2)) model.add( Dense(embedding_size * max_len // 4, activation='relu', kernel_regularizer=l2(0.001))) model.add(Dropout(0.2)) elif model_type == 'Conv1D': model.add( Conv1D(32, 7, activation='relu', kernel_regularizer=l2(0.001))) model.add( Conv1D(32, 7, activation='relu', kernel_regularizer=l2(0.001))) model.add(GlobalMaxPooling1D()) elif model_type == 'GlobalMaxPooling1D': model.add(GlobalMaxPooling1D()) elif model_type == 'GRU': model.add(GRU(embedding_size, dropout=0.2, recurrent_dropout=0.2)) # model.add(LSTM(128, dropout = 0.5, recurrent_dropout = 0.5)) elif model_type == 'Conv1D+LSTM': model.add( Conv1D(32, 5, activation='relu', kernel_regularizer=l2(0.001))) model.add( Conv1D(32, 5, activation='relu', kernel_regularizer=l2(0.001))) model.add(LSTM(16, dropout=0.5, recurrent_dropout=0.5)) elif model_type == 'Bidirectional-LSTM': model.add( Bidirectional( LSTM(embedding_size, dropout=0.2, recurrent_dropout=0.2))) else: raise Exception("错误的网络模型类型") if label_name == "age": model.add( Dense(embedding_size * 10, activation='relu', kernel_regularizer=l2(0.001))) model.add(Dense(10, activation='softmax', kernel_regularizer=l2(0.001))) print("%s——模型构建完成!" % model_type) print("* 编译模型") model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr), loss=losses.sparse_categorical_crossentropy, metrics=[metrics.sparse_categorical_accuracy]) elif label_name == 'gender': model.add( Dense(embedding_size, activation='relu', kernel_regularizer=l2(0.001))) model.add(Dense(1, activation='sigmoid', kernel_regularizer=l2(0.001))) print("%s——模型构建完成!" % model_type) print("* 编译模型") model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr), loss=losses.binary_crossentropy, metrics=[metrics.binary_accuracy]) else: raise Exception("错误的标签类型!") return model
def getModel(maxWordLen, data_dim): # mlp = Sequential() # mlp.add(Dense(512, input_dim=data_dim, init='uniform', activation='tanh')) # mlp.add(Dropout(0.5)) # mlp.add(Dense(128, init='uniform', activation='tanh')) # mlp.add(Dropout(0.5)) # mlp.add(Dense(1, init='uniform', activation='sigmoid')) # sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) # mlp.compile(loss='mean_squared_error', # optimizer=sgd, # metrics=['accuracy']) # # train # mlp.fit(x_train, y_train, # batch_size=68, nb_epoch=5, # validation_data=(x_val, y_val)) # # test # score = mlp.evaluate(x_test, y_test, show_accuracy=True, verbose=0) # print('Test score:', score[0]) # print('Test accuracy:', score[1]) # , dtype='int32' # inp = Input(shape=(maxWordLen+1, data_dim)) # # embedded = Embedding(data_dim, 128, input_length=maxWordLen+1)(inp) # # # # apply forwards LSTM # forwards = LSTM( # 128, return_sequences=False)(inp) # # # apply backwards LSTM # backwards = LSTM( # 128, return_sequences=False, go_backwards=True)(inp) # # # # concatenate the outputs of the 2 LSTMs # merged = merge([forwards, backwards], mode='concat', concat_axis=-1) # # after_dp = Dropout(0.5)(merged) # # # l2 = LSTM(16)(merged) # output = Dense(2, activation='softmax')(merged) # lstm = Model(input=inp, output=output) lstm = Sequential() # lstm.add(Embedding(data_dim, 128, input_length=maxWordLen+1)) # embedded = Embedding(max_features, 128, input_length=maxlen)(sequence) # lecun_uniform lstm.add(GRU( output_dim=62, return_sequences=False, # batch_input_shape=(), input_shape=(maxWordLen+1, data_dim), consume_less="cpu", go_backwards=False, dropout_W=0.25 # W_regularizer=l2(0.01), # b_regularizer=l2(0.01) )) # lstm.add(GRU(8, return_sequences=False, consume_less="cpu")) # lstm.add(LSTMpeephole( # output_dim=1, # return_sequences=False, # # batch_input_shape=(), # input_shape=(maxWordLen+1, data_dim), # consume_less="cpu", # go_backwards=False # # dropout_W=0.5 # # W_regularizer=l2(0.01) # # b_regularizer=l2(0.01) # )) # lstm.add(LSTM( # 128, return_sequences=False, consume_less="cpu", # go_backwards=True)) # lstm.add(LSTM( # hidden_units, return_sequences=True, consume_less="cpu", # go_backwards=False)) # lstm.add(LSTM( # hidden_units, return_sequences=False, consume_less="cpu", # go_backwards=False)) # lstm.add(LSTM( # hidden_units, return_sequences=True, consume_less="cpu", # go_backwards=True)) # lstm.add(LSTM( # hidden_units, # return_sequences=True, # # Varies by word length # input_length=None, # input_dim=data_dim)) # lstm.add(LSTM(hidden_units, return_sequences=True)) # lstm.add(LSTM(hidden2_units, return_sequences=False, consume_less="cpu")) # lstm.add(Dense(256, activation="tanh")) # lstm.add(Dense(32, activation="tanh")) lstm.add(Dense(1, activation="sigmoid")) # lstm.add(Dense(2, activation="softmax")) return lstm
def create_model(self, ckpt_file=None): if ckpt_file is None: if self.arch == "vanilla": self.model = Sequential() self.model.add(self.embedding_layer) self.model.add( LSTM(LSTM_SIZE, dropout=self.drop_prob, recurrent_dropout=self.drop_prob, implementation=2, unroll=True)) self.model.add(Dense(2, activation='softmax')) elif self.arch == "gru": self.model = Sequential() self.model.add(self.embedding_layer) self.model.add( GRU(LSTM_SIZE, dropout=self.drop_prob, recurrent_dropout=self.drop_prob, implementation=2, unroll=True)) self.model.add(Dense(2, activation='softmax')) elif self.arch == "conv": conv_filters = [] for filter_size in self.filter_sizes: conv_filters.append(Sequential()) conv_filters[-1].add(self.embedding_layer) conv_filters[-1].add( Conv1D(filters=self.num_filters, kernel_size=filter_size, strides=1, padding='valid', activation='relu')) conv_filters[-1].add( MaxPooling1D(pool_size=(self.seq_length - filter_size + 1))) self.model = Sequential() self.model.add(Merge(conv_filters, mode='concat')) self.model.add(Flatten()) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(512, activation='softmax')) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(2, activation='softmax')) elif self.arch == "seq_conv1": self.model = Sequential() filter_size = 3 self.model.add(self.embedding_layer) self.model.add( Conv1D(filters=self.num_filters / 2, kernel_size=filter_size, strides=1, padding='valid', activation='relu')) self.model.add(MaxPooling1D(pool_size=2)) self.model.add( Conv1D(filters=int(self.num_filters / 4), kernel_size=filter_size, strides=1, padding='valid', activation='relu')) self.model.add(MaxPooling1D(pool_size=2)) self.model.add(Flatten()) self.model.add(Dense(1024, activation='relu')) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(2, activation='softmax')) elif self.arch == "seq_conv2": self.model = Sequential() filter_size = 3 self.model.add(self.embedding_layer) self.model.add( Conv1D(filters=self.num_filters, kernel_size=filter_size, strides=1, padding='same', activation='relu')) self.model.add( Conv1D(filters=int(self.num_filters / 2), kernel_size=filter_size, strides=1, padding='same', activation='relu')) self.model.add(MaxPooling1D(pool_size=2)) self.model.add( Conv1D(filters=int(self.num_filters / 4), kernel_size=filter_size, strides=1, padding='same', activation='relu')) self.model.add( Conv1D(filters=int(self.num_filters / 4), kernel_size=filter_size, strides=1, padding='same', activation='relu')) self.model.add(MaxPooling1D(pool_size=2)) self.model.add(Flatten()) self.model.add(Dense(1024, activation='relu')) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(512, activation='relu')) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(2, activation='softmax')) elif self.arch == "conv_lstm": # Ensemble of sentence-level embedding obtained from an LSTM and # a sentence-level embedding obtained from a CNN. print("Start") branch1 = Sequential() branch1.add(self.embedding_layer) branch1.add( LSTM(LSTM_SIZE, dropout=self.drop_prob, recurrent_dropout=self.drop_prob, implementation=2, unroll=True)) print("Created first model") conv_filters = [] for filter_size in self.filter_sizes: conv_filters.append(Sequential()) conv_filters[-1].add(self.embedding_layer) conv_filters[-1].add( Conv1D(filters=self.num_filters, kernel_size=filter_size, strides=1, padding='valid', activation='relu')) conv_filters[-1].add( MaxPooling1D(pool_size=(self.seq_length - filter_size + 1))) branch2 = Sequential() branch2.add(Merge(conv_filters, mode='concat')) branch2.add(Flatten()) branch2.add(Dropout(2 * DROPOUT)) print("Created second model") self.model = Sequential() self.model.add(Merge([branch1, branch2], mode='concat')) self.model.add(Dense(1024, activation='relu')) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(2, activation='softmax')) elif self.arch == "ensemble": # Ensemble of sentence-level embedding obtained from an LSTM and # a sentence-level embedding obtained from the OpenAI features. print("Start") branch1 = Sequential() branch1.add(self.embedding_layer) branch1.add( LSTM(LSTM_SIZE, dropout=self.drop_prob, recurrent_dropout=self.drop_prob, implementation=2, unroll=True)) print("Created first model") branch2 = Sequential() branch2.add( Dense(OPENAI_REDUCED_SIZE, activation="linear", input_shape=(OPENAI_FEATURE_SIZE, ))) print("Created second model") self.model = Sequential() self.model.add(Merge([branch1, branch2], mode='concat')) self.model.add(Dense(1024, activation='relu')) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(2, activation='softmax')) elif self.arch == "multi_layer": self.model = Sequential() self.model.add(self.embedding_layer) # Two LSTM layers. self.model.add( LSTM(LSTM_SIZE, dropout=self.drop_prob, recurrent_dropout=self.drop_prob, implementation=2, unroll=True, return_sequences=True)) self.model.add( LSTM(LSTM_SIZE, dropout=self.drop_prob, recurrent_dropout=self.drop_prob, implementation=2, unroll=True)) # Add 2 fully-connected layers. self.model.add(Dense(1024, activation='relu')) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(512, activation='relu')) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(2, activation='softmax')) elif self.arch == "bidi": self.model = Sequential() self.model.add(self.embedding_layer) lstm_layer = LSTM(LSTM_SIZE, dropout=self.drop_prob, recurrent_dropout=self.drop_prob, implementation=2, unroll=True) self.model.add(Bidirectional(lstm_layer, merge_mode='sum')) self.model.add(Dense(512, activation='relu')) self.model.add(Dropout(2 * DROPOUT)) self.model.add(Dense(2, activation='softmax')) else: raise NotImplementedError("Architecture is not implemented:", self.arch) else: # Load model from checkpoint file. self.model = load_model(ckpt_file)
sen_batch1 = BatchNormalization()(sen_conv1) sen_max1 = MaxPooling1D(pool_size=MAX1_SIZE)(sen_batch1) sen_conv3 = Conv1D(CONV_NUM2, kernel_size=(CONV_MERGE_LEN3), strides=(STRIDE_MERGE_LEN3), padding='causal', activation='relu')(sen_batch1) sen_batch3 = BatchNormalization()(sen_conv3) sen_max3 = MaxPooling1D(pool_size=MAX2_SIZE)(sen_batch3) sen_flatten = Flatten()(sen_max3) merged_model = Model(inputs=win_input, outputs=sen_flatten) timed = TimeDistributed(merged_model)(input) #elkészült conv struktúra print(merged_model.summary()) grucell = GRU(INTER_DIM, return_sequences=True)(timed) grucell2 = GRU(INTER_DIM, return_sequences=True)(grucell) avg = Lambda(avg_layer)(grucell2) if (len(PERSONS) > 2): dens = Dense(len(PERSONS), activation="softmax")( avg) #sigmoid binárisnál, softmax categorinál else: dens = Dense(1, activation="sigmoid")(avg) model = Model(inputs=input, outputs=dens) #teljes modell print(model.summary()) if (len(PERSONS) > 2): model.compile(loss='categorical_crossentropy', optimizer='RMSprop', metrics=['accuracy']) # rms prop # binary hogyha bináris
#model.add(Bidirectional(GRU(64,dropout_W=dropout, dropout_U=dropout, return_sequences=True), merge_mode='concat')) model.add(Bidirectional(GRU(64, return_sequences=True), merge_mode='concat', input_shape=(50, 10))) #model.add(GRU(128,dropout_W=dropout, dropout_U=dropout, return_sequences=False)) #model.add(GRU(64, return_sequences=False)) #model.add(Dropout(dropout)) #model.add(Dense(64, activation="tanh")) model.add(AttentionLayer()) model.add(Dense(num_classes, activation="softmax")) ''' x_input = Input(shape=input_shape, dtype='float32') #x_input = Input(shape=(50,10), dtype='float32') l_lstm = Bidirectional(GRU(64, return_sequences=True), merge_mode='concat')(x_input) #l_dense = TimeDistributed(Dense(256))(l_lstm) att = AttentionLayer()(l_lstm) preds = Dense(num_classes, activation="softmax")(att) model = Model(x_input, preds) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) model.summary() #exit() model.fit( train_features,
def lstm(n_features=4, n_timesteps=12, n_train=10, n_window=5, n_units=100, n_epochs=50, with_att=False, methods='lstm', lr=0.001): """ :param n_features: 4 or 10, using 4 features or 10 features :param n_train: training timesteps :param n_window: width of training window, for example, [0 1 2 3 4]->[5], n_window = 5 :param n_units: LSTM units :param n_epochs: trainning epochs :return: """ data = [] for i in range(len(name_node_pairs)): f = open('./data/features_{}/{}_temp_link_ft.csv'.format( n_features, name_node_pairs[i])) df = pd.read_csv(f) data.append(df.values) data = np.array(data) print('data: {}, \ndata.shape(): {}'.format(data, data.shape)) # define train, test # scaler = MinMaxScaler(feature_range=(0, 1)) n_samples, n_timesteps, n_features = data.shape scaled_data = data.reshape((n_samples, n_timesteps * n_features)) # scaled_data = scaler.fit_transform(scaled_data) scaled_data = scaled_data.reshape((n_samples, n_timesteps, n_features)) # define problem properties n_test = 12 - n_train # define LSTM # sequential # model = Sequential() # model.add(Bidirectional(LSTM(n_units, input_shape=(n_window, n_features)))) # model.add(Dense(1)) # # model.compile(loss='mse', optimizer='adam') # Model inputs = Input(shape=(n_window, n_features)) return_sequences = False if with_att == True: return_sequences = True if methods == 'lstm': att_in = Bidirectional( LSTM(n_units, input_shape=(n_window, n_features), return_sequences=return_sequences))(inputs) elif methods == 'gru': att_in = Bidirectional( GRU(n_units, input_shape=(n_window, n_features), return_sequences=return_sequences))(inputs) elif methods == 'rnn': att_in = Bidirectional( SimpleRNN(n_units, input_shape=(n_window, n_features), return_sequences=return_sequences))(inputs) if with_att == True: att_out = attention()(att_in) outputs = Dense(1)(att_out) else: outputs = Dense(1)(att_in) model = Model(inputs, outputs) opt = optimizers.Adam(lr=lr) model.compile(loss='mse', optimizer=opt) # fit network for i in range(n_train - n_window): history = model.fit(scaled_data[:, i:i + n_window, :], scaled_data[:, i + n_window, 1], epochs=n_epochs) # plot history # plt.plot(history.history['loss']) # plt.show() # make prediction inv_yhat = [] for i in range(n_test): yhat = model.predict(scaled_data[:, n_train - n_window + i:n_train + i, :]) inv_yhat.append(yhat) inv_yhat = np.array(inv_yhat) print('inv_yhat.shape:{}'.format( inv_yhat.shape)) #inv_yhat.shape:(3, 736, 1) inv_yhat = inv_yhat.reshape((inv_yhat.shape[0], inv_yhat.shape[1])) print('inv_yhat.shape:{}'.format(inv_yhat.shape)) #inv_yhat.shape:(3, 736) inv_yhat = inv_yhat.T print('inv_yhat.shape:{}'.format(inv_yhat.shape)) #inv_yhat.shape:(736, 3) inv_yhat = np.concatenate((scaled_data[:, n_train:, 0], inv_yhat), axis=1) inv_yhat = inv_yhat.reshape((n_samples, n_test, 2)) inv_yhat = np.concatenate((inv_yhat, scaled_data[:, n_train:, 2:]), axis=2) print('inv_yhat.shape:{}'.format(inv_yhat.shape)) inv_yhat = np.concatenate((scaled_data[:, :n_train, :], inv_yhat), axis=1) print('inv_yhat.shape:{}'.format(inv_yhat.shape)) inv_yhat = inv_yhat.reshape(n_samples, n_timesteps * n_features) print('inv_yhat.shape:{}'.format(inv_yhat.shape)) # inv_yhat = scaler.inverse_transform(inv_yhat) inv_yhat = inv_yhat.reshape(n_samples, n_timesteps, n_features) inv_yhat[inv_yhat < 0] = 0 # transform negative values to zero prediction = inv_yhat[:, -3:, 1] prediction = prediction.reshape(prediction.shape[0], prediction.shape[1], 1) original = data[:, -3:, 1] original = original.reshape(original.shape[0], original.shape[1], 1) concat = np.concatenate((original, prediction), axis=2) print('concat.shape:{}'.format(concat.shape)) np.set_printoptions(threshold=1e6) print('concat\n{}'.format(concat)) concat = concat.reshape(concat.shape[0] * concat.shape[1], concat.shape[2]) df = pd.DataFrame(concat) df.columns = ['original', 'prediction'] df.to_csv('./data/LSTM/prediction_LSTM_{}.csv'.format(n_features), index=False) rmse = sqrt(mean_squared_error(inv_yhat[:, -3:, 1], data[:, -3:, 1])) print('rmse: {}'.format(rmse))
print("------------------------------------------------------") print(Y[:10]) #print(Y.shape) MAX_LENGTH = 6 X = pad_sequences(X, maxlen=MAX_LENGTH, padding='post') print(X.shape) vocab_size = len(vocab) Y = to_categorical(Y, num_classes=vocab_size) print(Y.shape) model = Sequential() model.add(Embedding(vocab_size, 50, input_length=6)) model.add(GRU(150, recurrent_dropout=0.1, dropout=0.1)) model.add(Dense(vocab_size, activation='softmax')) model.compile(loss='categorical_crossentropy', metrics=['acc'], optimizer='adam') print(model.summary()) model.fit(X, Y, epochs=100, verbose=2) def spell_checking(sentence): X, _ = generate_training_data(sentence, word_to_id, 3) for i in range(len(X)): yhat = model.predict_classes(np.array([X[i]])) word = id_to_word[int(yhat)] print(i)
from keras.layers import GRU from keras.layers import SimpleRNN from keras.layers import Dropout from keras import losses import h5py from keras import backend as K import numpy as np print('Build model...') main_input = Input(shape=(None, 22), name='main_input') #x = Dense(44, activation='relu')(main_input) #x = GRU(44, dropout=0.0, recurrent_dropout=0.0, activation='tanh', recurrent_activation='sigmoid', return_sequences=True)(x) x = main_input x = GRU(128, activation='tanh', recurrent_activation='sigmoid', return_sequences=True)(x) #x = GRU(128, return_sequences=True)(x) #x = GRU(22, activation='relu', return_sequences=True)(x) x = Dense(22, activation='sigmoid')(x) #x = Dense(22, activation='softplus')(x) model = Model(inputs=main_input, outputs=x) batch_size = 32 print('Loading data...') with h5py.File('denoise_data.h5', 'r') as hf: all_data = hf['denoise_data'][:] print('done.') window_size = 500
for n, tags in enumerate(Y_test): for t, tag in enumerate(tags): Y_test_onehot[n, t, tag] = 1 # 参数 epochs = 30 batch_size = 32 hidden_layer_size = 10 embedding_dim = 10 # 建立模型 Input只说明输入列维度 # NxT input_ = Input(shape=(max_sequence_length, )) # NxTxD x = Embedding(vocab_size, embedding_dim)(input_) x = GRU(hidden_layer_size, return_sequences=True)(x) # NxTxM output = Dense(tag_size, activation='softmax')(x) model = Model(input_, output) model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1e-2), metrics=['accuracy']) print('Training model...') # keras 的好处就是不用像tensorflow一样,输入前后都要进行维度转换 r = model.fit(X_train, Y_train_onehot, batch_size=batch_size, epochs=epochs, validation_data=(X_test, Y_test_onehot))
# words not found in embedding index will be all-zeros. embedding_matrix[i] = embedding_vector y_tr_one = get_onehot(y_train, 4) y_ts_one = get_onehot(y_test, 4) # creating the model gru_input = Input(shape=(maxlen, )) x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(gru_input) x = SpatialDropout1D(0.2)(x) x = Bidirectional( GRU(128, return_sequences=True, reset_after=True, recurrent_activation='sigmoid'))(x) avg_pool = GlobalAveragePooling1D()(x) max_pool = GlobalMaxPooling1D()(x) x = concatenate([avg_pool, max_pool]) x = Dense(128, activation='relu')(x) x = Dropout(0.1)(x) gru_output = Dense(4, activation="sigmoid")(x) gru_model = Model(gru_input, gru_output) gru_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # training parameters epochs = 20 batch_size = 128
def main(rnn_model): def message_to_array(msg): msg = msg.lower().split(' ') test_seq = np.array([word_index[word] for word in msg]) test_seq = np.pad(test_seq, (500-len(test_seq), 0), 'constant', constant_values=(0)) test_seq = test_seq.reshape(1, 500) return test_seq data = pd.read_csv("./spam_text_message_data.csv") print(data.head()) print(data.tail()) messages = [] labels = [] for index, row in data.iterrows(): messages.append(row['Message']) if row['Category'] == 'ham': labels.append(0) else: labels.append(1) messages = np.asarray(messages) labels = np.asarray(labels) # print("Number of messages: ", len(messages)) #print("Number of labels: ", len(labels)) max_vocab = 10000 max_len = 500 # Ignore all words except the 10000 most common words tokenizer = Tokenizer(num_words=max_vocab) # Calculate the frequency of words tokenizer.fit_on_texts(messages) # Convert array of messages to list of sequences of integers sequences = tokenizer.texts_to_sequences(messages) # Dict keeping track of words to integer index word_index = tokenizer.word_index # Convert the array of sequences(of integers) to 2D array with padding # maxlen specifies the maximum length of sequence (truncated if longer, padded if shorter) data = pad_sequences(sequences, maxlen=max_len) print("data shape: ", data.shape) # We will use 80% of data for training & validation(80% train, 20% validation) and 20% for testing train_samples = int(len(messages)*0.8) messages_train = data[:train_samples] labels_train = labels[:train_samples] messages_test = data[train_samples:len(messages)-2] labels_test = labels[train_samples:len(messages)-2] embedding_mat_columns=40 # Construct the SimpleRNN model model = Sequential() ## Add embedding layer to convert integer encoding to word embeddings(the model learns the ## embedding matrix during training), embedding matrix has max_vocab as no. of rows and chosen ## no. of columns model.add(Embedding(input_dim=max_vocab, output_dim=embedding_mat_columns, input_length=max_len)) model.add(Dropout(0.2)) model.add(Conv1D(64, 5, activation='sigmoid')) model.add(MaxPooling1D(pool_size=4)) if rnn_model == 'SimpleRNN': model.add(SimpleRNN(units=embedding_mat_columns)) elif rnn_model == 'LSTM': model.add(LSTM(units=embedding_mat_columns)) else: model.add(GRU(units=embedding_mat_columns)) model.add(Dense(1, activation='sigmoid')) model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc']) model.summary() #plot_model(model, to_file='model.png', show_shapes=True, show_layer_names=True) # Training the model model.fit(messages_train, labels_train, epochs=1, batch_size=60, validation_split=0.2) # Testing the model pred = model.predict_classes(messages_test) acc = model.evaluate(messages_test, labels_test) print("Test loss is {0:.2f} accuracy is {1:.2f} ".format(acc[0],acc[1])) # ============================================================================= # print(pred) # print(labels_test) # ============================================================================= print("precision is about ", metrics.precision_score(labels_test, pred,average="macro")) print("recall is about ", metrics.recall_score(labels_test, pred, average="macro")) print("f-score is about ", metrics.f1_score(labels_test, pred, average="macro")) print(metrics.confusion_matrix(labels_test, pred))
if len(sys.argv) == 2: print("load " + sys.argv[1] + " to keep training") model = load_model(sys.argv[1]) else: model = Sequential() wm = np.load('wm.npy') model.add( Embedding(wm.shape[0], output_dim=wm.shape[1], weights=[wm], input_length=MAX_LENGTH, trainable=False)) model.add( Bidirectional( GRU(units=256, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))) model.add(Dropout(0.4)) model.add( Bidirectional( GRU(units=256, return_sequences=False, dropout=0.2, recurrent_dropout=0.2))) model.add(Dropout(0.6)) model.add(Dense(units=1, activation='sigmoid')) rms = optimizers.RMSprop(clipnorm=1.) model.compile(optimizer=rms, loss='binary_crossentropy', metrics=['accuracy']) print(model.summary())
def get_model_5(params): input_tensor=None include_top=True # Determine proper input shape if K.image_dim_ordering() == 'th': input_shape = (1, 96, 1366) else: input_shape = (96, 1366, 1) if input_tensor is None: melgram_input = Input(shape=input_shape) else: if not K.is_keras_tensor(input_tensor): melgram_input = Input(tensor=input_tensor, shape=input_shape) else: melgram_input = input_tensor # Determine input axis if K.image_dim_ordering() == 'th': channel_axis = 1 freq_axis = 2 time_axis = 3 else: channel_axis = 3 freq_axis = 1 time_axis = 2 # Input block x = ZeroPadding2D(padding=(0, 37))(melgram_input) x = BatchNormalization(axis=freq_axis, name='bn_0_freq')(x) x = Permute((1, 3, 2))(x) # Conv block 1 x = Convolution2D(64, 3, 3, border_mode='same', name='conv1')(x) x = BatchNormalization(axis=channel_axis, mode=0, name='bn1')(x) x = ELU()(x) x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x) x = Dropout(0.1, name='dropout1')(x) # Conv block 2 x = Convolution2D(128, 3, 3, border_mode='same', name='conv2')(x) x = BatchNormalization(axis=channel_axis, mode=0, name='bn2')(x) x = ELU()(x) x = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(x) x = Dropout(0.1, name='dropout2')(x) # Conv block 3 x = Convolution2D(128, 3, 3, border_mode='same', name='conv3')(x) x = BatchNormalization(axis=channel_axis, mode=0, name='bn3')(x) x = ELU()(x) x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(x) x = Dropout(0.1, name='dropout3')(x) # Conv block 4 x = Convolution2D(128, 3, 3, border_mode='same', name='conv4')(x) x = BatchNormalization(axis=channel_axis, mode=0, name='bn4')(x) x = ELU()(x) x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(x) x = Dropout(0.1, name='dropout4')(x) # reshaping if K.image_dim_ordering() == 'th': x = Permute((3, 2, 1))(x) x = Reshape((15, 128))(x) # GRU block 1, 2, output x = GRU(32, return_sequences=True, name='gru1')(x) x = GRU(32, return_sequences=False, name='gru2')(x) x = Dropout(0.3)(x) if include_top: x = Dense(params["n_out"], activation=params['final_activation'], name='output')(x) if params['final_activation'] == 'linear': reg = Lambda(lambda x :K.l2_normalize(x, axis=1)) x = reg(x) # Create model model = Model(melgram_input, x) return model
test_data = X[trainSize:len(training_data), :, :] test_labels = y[trainSize:len(training_data), :] test_data2 = test_features[trainSize:len(training_data), :] test_labels2 = labels[trainSize:len(training_data)] #network #tanh/sigmoid scale[0,1]/[-1,1] #class_weight=class_weight.compute_class_weight('balanced',np.unique(labels),labels) nb_epoch = 5 BS = 64 my_init = keras.initializers.glorot_uniform(seed=seed) model = Sequential() model.add( Masking(mask_value=0, input_shape=(train_data.shape[1], train_data.shape[2]))) model.add(GRU(units=200, kernel_initializer=my_init, stateful=False)) model.add(Activation('tanh')) #model.add(Dropout(0.2)) ''' model.add(LSTM(units=200, kernel_initializer=my_init, return_sequences=True)) model.add(Activation('tanh')) ''' model.add(Dense(units=len(itemId2), kernel_initializer=my_init)) #model.add(Dense(units = len(itemId2), kernel_initializer=my_init)) model.add(Activation('softmax')) #Output #model.add(Activation('tanh'))#Output #activation function #sgd = optimizers.Adagrad(lr=0.001, epsilon=None, decay=1e-6) #sgd = optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.98, nesterov=True ) #sgd = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-8, decay=0.0) #sgd = optimizers.adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=1e-6, amsgrad=False )
eij = K.tanh(K.dot(x, self.W)) ai = K.exp(eij) weights = ai / K.sum(ai, axis=1).dimshuffle(0, 'x') weighted_input = x * weights.dimshuffle(0, 1, 'x') #.decode('utf-8') print(weighted_input) return weighted_input.sum(axis=1) def get_output_shape_for(self, input_shape): return (input_shape[0], input_shape[-1]) sentence_input = Input(shape=(max_text_len, ), dtype='int32') embedded_sequences = embedding_layer(sentence_input) l_lstm = Bidirectional(GRU(100, return_sequences=True))(embedded_sequences) l_dense = TimeDistributed(Dense(200))(l_lstm) l_att = AttLayer()(l_dense) sentEncoder = Model(sentence_input, l_att) review_input = Input(shape=(max_text_num, max_text_len), dtype='int32') review_encoder = TimeDistributed(sentEncoder)(review_input) l_lstm_sent = Bidirectional(GRU(100, return_sequences=True))(review_encoder) l_dense_sent = TimeDistributed(Dense(200))(l_lstm_sent) l_att_sent = AttLayer()(l_dense_sent) preds = Dense(2, activation='softmax')(l_att_sent) model = Model(review_input, preds) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])
def get_model(): #Inputs name = Input(shape=[X_train["name"].shape[1]], name="name") item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc") brand = Input(shape=[1], name="brand") #category = Input(shape=[1], name="category") category_name_split = Input( shape=[X_train["category_name_split"].shape[1]], name="category_name_split") item_condition = Input(shape=[1], name="item_condition") num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars") #Embeddings layers emb_size = 60 emb_name = Embedding(MAX_TEXT, emb_size)(name) # , mask_zero=True emb_item_desc = Embedding(MAX_TEXT, emb_size)(item_desc) # , mask_zero=True# emb_category_name_split = Embedding(MAX_TEXT, emb_size // 3)( category_name_split) # , mask_zero=True emb_brand = Embedding(MAX_BRAND, 8)(brand) emb_item_condition = Embedding(MAX_CONDITION, 5)(item_condition) conv1 = Conv1D(16, 3, activation='relu', padding="same")(emb_item_desc) conv1 = MaxPooling1D(2)(conv1) conv1 = Conv1D(32, 3, activation='relu', padding="same")(conv1) conv1 = MaxPooling1D(2)(conv1) conv1 = Conv1D(32, 3, activation='relu', padding="same")(conv1) conv1 = MaxPooling1D(2)(conv1) conv2 = Conv1D(16, 3, activation='relu', padding="same")(emb_category_name_split) conv2 = MaxPooling1D(2)(conv2) conv2 = Conv1D(32, 3, activation='relu', padding="same")(conv2) #conv2 = MaxPooling1D(2)(conv2) conv3 = Conv1D(16, 3, activation='relu', padding="same")(emb_name) conv3 = MaxPooling1D(2)(conv3) conv3 = Conv1D(32, 3, activation='relu', padding="same")(conv3) #conv3 = MaxPooling1D(2)(conv3) rnn_layer3 = GRU(32, recurrent_dropout=0.0)(concatenate([conv3, conv2, conv1])) #main layer main_l = concatenate([ Flatten()(emb_brand) #, Flatten() (emb_category) , Flatten()(emb_item_condition) #, rnn_layer1 #, rnn_layer2 , rnn_layer3, num_vars ]) main_l = Dropout(dr)(Dense(512, activation='relu')(main_l)) main_l = Dropout(dr)(Dense(64, activation='relu')(main_l)) #output output = Dense(1, activation="linear")(main_l) #model model = Model( [ name, item_desc, brand #, cat1, cat2, cat3 , category_name_split # category_name, category, , item_condition, num_vars ], output) #optimizer = optimizers.RMSprop() optimizer = optimizers.Adam() model.compile(loss='mse', optimizer=optimizer) return model
maxlen = np.max([len(x) for x in T]) print(maxlen) # Convert characters to int and pad T = [[valid_chars[y] for y in x] for x in T] X_test = sequence.pad_sequences(T, maxlen=max_len) #y_train = np.array(trainlabel) y_test = np.array(testlabel) embedding_vecor_length = 128 model = Sequential() model.add(Embedding(max_features, embedding_vecor_length, input_length=max_len)) model.add(GRU(128)) model.add(Dropout(0.1)) model.add(Dense(1)) model.add(Activation('sigmoid')) ''' model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy']) checkpointer = callbacks.ModelCheckpoint(filepath="logs/gru/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='val_acc',mode='max') csv_logger = CSVLogger('logs/gru/training_set_gruanalysis.csv',separator=',', append=False) model.fit(X_train, y_train, batch_size=32, nb_epoch=1000,validation_split=0.33, shuffle=True,callbacks=[checkpointer,csv_logger]) score, acc = model.evaluate(X_test, y_test, batch_size=128) print('Test score:', score) print('Test accuracy:', acc) '''
embedding_matrix = np.random.random((len(word_index) + 1, EMBEDDING_DIM)) for word, i in word_index.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None: # words not found in embedding index will be all-zeros. embedding_matrix[i] = embedding_vector embedding_layer = Embedding(len(word_index) + 1, EMBEDDING_DIM, weights=[embedding_matrix], input_length=MAX_SEQUENCE_LENGTH, trainable=True) sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32') embedded_sequences = embedding_layer(sequence_input) l_gru = Bidirectional(GRU(100, return_sequences=True))(embedded_sequences) l_att = AttLayer()(l_gru) preds = Dense(2, activation='softmax')(l_att) model = Model(sequence_input, preds) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc']) print("model fitting - attention GRU network") model.summary() model.fit(x_train, y_train, validation_data=(x_val, y_val), nb_epoch=10, batch_size=50)
# pad documents to a max length of 4 words tpadded_docs = pad_sequences(tencoded_docs, maxlen=max_length, padding='post') #print(padded_docs) # define model model = Sequential() e = Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=max_length, trainable=False) model.add(e) model.add( Bidirectional( GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))) model.add(Dense(nclass, activation='softmax')) # compile the model model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy']) # summarize the model print(model.summary()) # fit the model model.fit(padded_docs, y_train, epochs=1, verbose=0, validation_data=(vpadded_docs, y_valid))
from sklearn.model_selection import train_test_split X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) #Reshape for LSTM X_train = X_train.reshape(X_train.shape + (1, )) X_test = X_test.reshape(X_test.shape + (1, )) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) # create the model model = Sequential() model.add(GRU(200, input_shape=X_train.shape[1:])) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) history = model.fit(X_train, y_train, batch_size=64, nb_epoch=3, validation_data=(X_test, y_test)) """embedding_vecor_length = 32 model = Sequential() model.add(Embedding(1000, embedding_vecor_length, input_length=10)) model.add(Convolution1D(nb_filter=32, filter_length=3, border_mode='same', activation='relu')) model.add(MaxPooling1D(pool_length=2))
mask_zero=mask) sequence_input_c = Input(shape=( padsize, MAX_WORD_LENGTH, ), dtype='int32') embedded_sequences_c = embedding_layer_c(sequence_input_c) rone = Lambda(reshape_one)(embedded_sequences_c) merge_m = raw_input('Enter merge mode for GRU Karakter: ') dropout = input('Enter dropout for GRU: ') rec_dropout = dropout # input('Enter GRU Karakter recurrent dropout: ') gru_karakter = Bidirectional(GRU(CHAR_EMBEDDING_DIM, return_sequences=False, dropout=dropout, recurrent_dropout=rec_dropout), merge_mode=merge_m, weights=None)(rone) rtwo = Lambda(reshape_two)(gru_karakter) """ Combine word + char model """ from keras.layers import Add, Subtract, Multiply, Average, Maximum print "Model Choice:" model_choice = input('Enter 1 for WE only, 2 for CE only, 3 for both: ') merge_m = raw_input('Enter merge mode for GRU Kata: ') # dropout = input('Enter GRU Karakter dropout: ') # rec_dropout = input('Enter GRU Karakter recurrent dropout: ')
max_sequence_length = max_recipe_length + 1 max_name_sequence_length = max_name_length + 1 print('Building Inference model...') hiddenStateSize = 512 hiddenStateSize2 = 256 hiddenStateSize3 = 128 hiddenLayerSize = 128 recipe_node = Sequential() recipe_node.add( GRU(hiddenStateSize, batch_input_shape=(1, max_sequence_length, len(char2id)), stateful=True)) recipe_node.add(Dense(hiddenStateSize3)) recipe_node.add(Activation('relu')) title_node = Sequential() title_node.add( GRU(hiddenStateSize2, batch_input_shape=(1, 1, len(char2id)), stateful=True)) title_node.add(Dense(hiddenStateSize3)) title_node.add(Activation('relu')) inference_model = Sequential() inference_model.add(
def cinq_wordEmb_model(args, embedding_matrix, isTeacherForcing=True): '''Inputs P and Q''' P_Input_word = Input(shape=(args.c_max_len, ), name='Pword') Q_Input_word = Input(shape=(args.q_max_len, ), name='Qword') P_Input_char = Input(shape=(args.c_max_len, ), name='Pchar') Q_Input_char = Input(shape=(args.q_max_len, ), name='Qchar') '''Input context vector''' cinq_vector = Input(shape=(args.c_max_len, args.context_vector_level)) gru_cinq = Bidirectional(GRU(64, return_sequences=True))(cinq_vector) punctuation_vector = Input(shape=(args.c_max_len, args.punctuation_level)) P_char_encode, P_word_encode, passage_encoding, \ Q_char_encode, Q_word_encode, question_encoding \ = sentence_encode(args, P_Input_word, Q_Input_word, P_Input_char, Q_Input_char, embedding_matrix) q_char_attention_vector, q_word_attention_vector, question_attention_vector \ = get_attentions(args, P_char_encode, P_word_encode, passage_encoding, Q_char_encode, Q_word_encode, question_encoding) # Answer start prediction output_start = Lambda( lambda arg: concatenate([arg[i] for i in range(len(arg))]))([ passage_encoding, q_char_attention_vector, q_word_attention_vector, question_attention_vector, gru_cinq, punctuation_vector, multiply([P_char_encode, q_char_attention_vector]), multiply([P_word_encode, q_word_attention_vector]), multiply([passage_encoding, question_attention_vector]) ]) output_start = TimeDistributed(Dense(args.hidden_size, activation='relu'))(output_start) output_start = TimeDistributed(Dense(1))(output_start) output_start = Flatten()(output_start) output_start = Activation(K.softmax)(output_start) '''Inputs answer_start ''' answer_start_input = Input(shape=(args.c_max_len, )) if isTeacherForcing == True: answer_start = answer_start_input else: answer_start = output_start # answer_start = Reshape((args.c_max_len, 1), input_shape=(args.c_max_len, ))(answer_start) # Answer end prediction depends on the start prediction def s_answer_feature(x): maxind = K.argmax( x, axis=1, ) return maxind x = Lambda(lambda x: K.tf.cast(s_answer_feature(x), dtype=K.tf.int32))( answer_start) start_feature = Lambda(lambda arg: K.tf.gather_nd( arg[0], K.tf.stack( [K.tf.range(K.tf.shape(arg[1])[0]), K.tf.cast(arg[1], K.tf.int32)], axis=1)))([passage_encoding, x]) start_feature = RepeatVector(args.c_max_len)(start_feature) start_position = Lambda( lambda x: K.tf.one_hot(K.argmax(x), args.c_max_len))(answer_start) start_position = Reshape((args.c_max_len, 1), input_shape=(args.c_max_len, ))(start_position) start_position = Bidirectional(GRU(8, return_sequences=True))(start_position) # Answer end prediction output_end = Lambda( lambda arg: concatenate([arg[i] for i in range(len(arg))]))([ start_position, start_feature, passage_encoding, q_char_attention_vector, q_word_attention_vector, question_attention_vector, gru_cinq, punctuation_vector, multiply([P_char_encode, q_char_attention_vector]), multiply([P_word_encode, q_word_attention_vector]), multiply([passage_encoding, question_attention_vector]), multiply([passage_encoding, start_feature]) ]) output_end = TimeDistributed(Dense(args.hidden_size, activation='relu'))(output_end) output_end = TimeDistributed(Dense(1))(output_end) output_end = Flatten()(output_end) output_end = Activation(K.softmax)(output_end) # define model in/out and compile inputs = [ cinq_vector, punctuation_vector, P_Input_word, Q_Input_word, P_Input_char, Q_Input_char, answer_start_input ] outputs = [output_start, output_end] model = Model(inputs, outputs) model.compile(optimizer=args.optimizer, loss=args.loss, metrics=['acc']) return model
def get_model(height, nclass): input = Input(shape=(height, None, 1), name='the_input') m = Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same', name='conv1')(input) m = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(m) m = Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same', name='conv2')(m) m = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool2')(m) m = Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', name='conv3')(m) m = Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', name='conv4')(m) m = ZeroPadding2D(padding=(0, 1))(m) m = MaxPooling2D(pool_size=(2, 2), strides=(2, 1), padding='valid', name='pool3')(m) m = Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', name='conv5')(m) m = BatchNormalization(axis=1)(m) m = Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', name='conv6')(m) m = BatchNormalization(axis=1)(m) m = ZeroPadding2D(padding=(0, 1))(m) m = MaxPooling2D(pool_size=(2, 2), strides=(2, 1), padding='valid', name='pool4')(m) m = Conv2D(512, kernel_size=(2, 2), activation='relu', padding='valid', name='conv7')(m) m = Permute((2, 1, 3), name='permute')(m) m = TimeDistributed(Flatten(), name='timedistrib')(m) m = Bidirectional(GRU(rnnunit, return_sequences=True), name='blstm1')(m) m = Dense(rnnunit, name='blstm1_out', activation='linear')(m) m = Bidirectional(GRU(rnnunit, return_sequences=True), name='blstm2')(m) y_pred = Dense(nclass, name='blstm2_out', activation='softmax')(m) basemodel = Model(inputs=input, outputs=y_pred) labels = Input(name='the_labels', shape=[ None, ], dtype='float32') input_length = Input(name='input_length', shape=[1], dtype='int64') label_length = Input(name='label_length', shape=[1], dtype='int64') loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')([y_pred, labels, input_length, label_length]) model = Model(inputs=[input, labels, input_length, label_length], outputs=[loss_out]) sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5) #model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer='adadelta') model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd) model.summary() return model, basemodel
def __init__(self, param_dict, Item_size): config = tf.ConfigProto( intra_op_parallelism_threads=THREAD, inter_op_parallelism_threads=THREAD, allow_soft_placement=True, ) session = tf.Session(config=config) backend.set_session(session) self.batch_size = int(param_dict['batch_size'] * 1.25) hand_feature_cols = len(Item_size['hand_feature']) name_seq_len = param_dict['name_Len'] desc_seq_len = param_dict['description_Len'] denselayer_units = param_dict['denselayer_units'] embed_name = param_dict['embed_name'] embed_desc = param_dict['embed_desc'] embed_brand = param_dict['embed_brand'] embed_cat_2 = param_dict['embed_cat_2'] embed_cat_3 = param_dict['embed_cat_3'] rnn_dim_name = param_dict['rnn_dim_name'] rnn_dim_desc = param_dict['rnn_dim_desc'] dense_drop = param_dict['dense_drop'] name_voc_size = Item_size['name'] desc_voc_size = Item_size['item_description'] brand_voc_size = Item_size['brand_name'] cat1_voc_size = Item_size['category_1'] cat2_voc_size = Item_size['category_2'] cat3_voc_size = Item_size['category_name'] # Inputs X_seq_name = Input(shape=[name_seq_len], name="X_seq_name", dtype='int32') X_seq_item_description = Input(shape=[desc_seq_len], name="X_seq_item_description", dtype='int32') X_brand_name = Input(shape=[1], name="X_brand_name", dtype='int32') X_category_1 = Input(shape=[1], name="X_category_1", dtype='int32') X_category_2 = Input(shape=[1], name="X_category_2", dtype='int32') X_category_name = Input(shape=[1], name="X_category_name", dtype='int32') X_item_condition_id = Input(shape=[1], name="X_item_condition_id", dtype='uint8') X_shipping = Input(shape=[1], name="X_shipping", dtype='float32') X_hand_feature = Input(shape=[hand_feature_cols], name="X_hand_feature", dtype='float32') # Embeddings layers name = Embedding(name_voc_size, embed_name)(X_seq_name) item_desc = Embedding(desc_voc_size, embed_desc)(X_seq_item_description) brand = Embedding(brand_voc_size, embed_brand)(X_brand_name) cat_2 = Embedding(cat2_voc_size, embed_cat_2)(X_category_2) cat_3 = Embedding(cat3_voc_size, embed_cat_3)(X_category_name) # RNN layers name = GRU(rnn_dim_name)(name) item_desc = GRU(rnn_dim_desc)(item_desc) # OneHot layers cond = Lambda(one_hot, arguments={'num_classes': 5}, output_shape=(1, 5))(X_item_condition_id) cat_1 = Lambda(one_hot, arguments={'num_classes': cat1_voc_size}, output_shape=(1, cat1_voc_size))(X_category_1) # main layer main_l = concatenate([ name, item_desc, Flatten()(cat_1), Flatten()(cat_2), Flatten()(cat_3), Flatten()(brand), Flatten()(cond), X_shipping, X_hand_feature, ]) main_l = Dropout(dense_drop)(Dense(denselayer_units, activation='relu')(main_l)) output = Dense(1, activation="linear")(main_l) # model model = Model([ X_seq_name, X_seq_item_description, X_brand_name, X_category_1, X_category_2, X_category_name, X_item_condition_id, X_shipping, X_hand_feature ], output) optimizer = optimizers.Adam(lr=param_dict['lr']) model.compile(loss='mse', optimizer=optimizer) self.model = model
embedding_vector = embedding_index.get(word) if embedding_vector is not None: embedding_matrix[i] = embedding_vector print(num_words) #define model model_nn = Sequential() embedding_layer = Embedding(num_words, EMBEDDING_DIM, embeddings_initializer=Constant(embedding_matrix), input_length=max_length, trainable=False) model_nn.add(embedding_layer) model_nn.add(GRU(units=32, dropout=0.2, recurrent_dropout=0.2)) model_nn.add(Dense(1, activation='sigmoid')) model_nn.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model_nn.summary() VALIDATION_SPLIT = 0.2 indices = np.arange(review_pad.shape[0]) #np.random.shuffle(indices) review_pad = review_pad[indices] sentiment = sentiment[indices] num_validation_samples = int(VALIDATION_SPLIT * review_pad.shape[0])
X = X.reshape((X.shape[0], X.shape[1], n_features)) X_train, X_test = X[:split], X[split:] y_train, y_test = y[:split], y[split:] model = Sequential() # model.add(LSTM(100,activation='relu',input_shape=(n_steps_in,n_features))) # model.add(LSTM(100,activation='relu',return_sequences=True,input_shape=(n_steps_in,n_features))) # model.add(LSTM(100,activation='relu')) # model.add(Bidirectional( LSTM(200,activation='relu',input_shape=(n_steps_in,n_features)) )) model.add( GRU(150, activation='relu', return_sequences=True, input_shape=(n_steps_in, n_features))) model.add(GRU(150, activation='relu', return_sequences=True)) model.add(GRU(150, activation='relu')) model.add(Dense(n_steps_out)) model.compile(loss='mse', optimizer='adam') model.fit(X_train, y_train, epochs=50, batch_size=1000, shuffle=False, verbose=1) y_pred = model.predict(X_test)
print(decoder_inputs[:,:,0]) decoder_lstm = LSTM(n_units, return_sequences=True, return_state=True) decoder_outputs, _, _ = decoder_lstm(decoder_inputs, initial_state=encoder_states) decoder_dense = Dense(n_headway_features,activation='relu') decoder_outputs = decoder_dense(decoder_outputs) model = Model(inputs = [encoder_inputs, decoder_inputs],outputs = decoder_outputs) ''' encoder_inputs = Input(shape=(None, n_stop_features)) encoder = GRU(n_units, return_state=True) encoder_outputs, state_h = encoder(encoder_inputs) decoder_inputs = Input(shape=(None, n_headway_features)) decoder_gru = GRU(n_units, return_sequences=True) decoder_outputs = decoder_gru(decoder_inputs, initial_state=state_h) decoder_dense = Dense(n_headway_features, activation='relu') decoder_outputs = decoder_dense(decoder_outputs) model = Model([encoder_inputs, decoder_inputs], decoder_outputs) model.compile(optimizer=optimiser, loss='mse', metrics=['acc']) X1_test_stopfeatures, X2_test_headway, y_test_target_headway = get_test_dataset( ) model.fit([X1_train_stopfeatures, X2_train_headway],
def build_seq2seq_model(word_emb_dim, hidden_state_dim, encoder_seq_len, num_encoder_tokens, num_decoder_tokens): """ Builds architecture for sequence to sequence model. Encoder and Decoder layer consist of one GRU Layer each. User can specify the dimensionality of the word embedding and the hidden state. Parameters ---------- word_emb_dim : int dimensionality of the word embeddings hidden_state_dim : int dimensionality of the hidden state in the encoder and decoder. encoder_seq_len : int the length of the sequences that are input into the encoder. The sequences are expected to all be padded to the same size. num_encoder_tokens : int the vocabulary size of the corpus relevant to the encoder. num_decoder_tokens : int the vocabulary size of the corpus relevant to the decoder. Returns ------- Keras.models.Model """ #### Encoder Model #### encoder_inputs = Input(shape=(encoder_seq_len, ), name='Encoder-Input') # Word embeding for encoder (ex: Issue Titles, Code) x = Embedding(num_encoder_tokens, word_emb_dim, name='Body-Word-Embedding', mask_zero=False)(encoder_inputs) x = BatchNormalization(name='Encoder-Batchnorm-1')(x) # We do not need the `encoder_output` just the hidden state. _, state_h = GRU(hidden_state_dim, return_state=True, name='Encoder-Last-GRU', dropout=.5)(x) # Encapsulate the encoder as a separate entity so we can just # encode without decoding if we want to. encoder_model = Model(inputs=encoder_inputs, outputs=state_h, name='Encoder-Model') seq2seq_encoder_out = encoder_model(encoder_inputs) #### Decoder Model #### decoder_inputs = Input(shape=(None, ), name='Decoder-Input') # for teacher forcing # Word Embedding For Decoder (ex: Issue Titles, Docstrings) dec_emb = Embedding(num_decoder_tokens, word_emb_dim, name='Decoder-Word-Embedding', mask_zero=False)(decoder_inputs) dec_bn = BatchNormalization(name='Decoder-Batchnorm-1')(dec_emb) # Set up the decoder, using `decoder_state_input` as initial state. decoder_gru = GRU(hidden_state_dim, return_state=True, return_sequences=True, name='Decoder-GRU', dropout=.5) decoder_gru_output, _ = decoder_gru(dec_bn, initial_state=seq2seq_encoder_out) x = BatchNormalization(name='Decoder-Batchnorm-2')(decoder_gru_output) # Dense layer for prediction decoder_dense = Dense(num_decoder_tokens, activation='softmax', name='Final-Output-Dense') decoder_outputs = decoder_dense(x) #### Seq2Seq Model #### seq2seq_Model = Model([encoder_inputs, decoder_inputs], decoder_outputs) return seq2seq_Model