def do_birnn_wordbag(trainX, testX, trainY, testY): y_test=testY #trainX = pad_sequences(trainX, maxlen=100, value=0.) #testX = pad_sequences(testX, maxlen=100, value=0.) # Converting labels to binary vectors trainY = to_categorical(trainY, nb_classes=2) testY = to_categorical(testY, nb_classes=2) # Network building # Network building net = input_data(shape=[None, 100]) net = tflearn.embedding(net, input_dim=10000, output_dim=128) net = tflearn.bidirectional_rnn(net, BasicLSTMCell(128), BasicLSTMCell(128)) net = dropout(net, 0.5) net = fully_connected(net, 2, activation='softmax') net = regression(net, optimizer='adam', loss='categorical_crossentropy') # Training model = tflearn.DNN(net, tensorboard_verbose=0) model.fit(trainX, trainY, validation_set=(testX, testY), show_metric=True, batch_size=1,run_id="uba",n_epoch=10) y_predict_list = model.predict(testX) #print y_predict_list y_predict = [] for i in y_predict_list: #print i[0] if i[0] >= 0.5: y_predict.append(0) else: y_predict.append(1) print(classification_report(y_test, y_predict)) print metrics.confusion_matrix(y_test, y_predict)
def recurrent_net(net, rec_type, rec_size, return_sequence): """ A quick if else block to build a recurrent layer, based on the type specified by the user. """ if rec_type == 'lstm': net = tflearn.layers.recurrent.lstm(net, rec_size, return_seq=return_sequence) elif rec_type == 'gru': net = tflearn.layers.recurrent.gru(net, rec_size, return_seq=return_sequence) elif rec_type == 'bi_lstm': net = bidirectional_rnn(net, BasicLSTMCell(rec_size), BasicLSTMCell(rec_size), return_seq=return_sequence) elif rec_type == 'bi_gru': net = bidirectional_rnn(net, GRUCell(rec_size), GRUCell(rec_size), return_seq=return_sequence) else: raise ValueError( 'Incorrect rnn type passed. Try lstm, gru, bi_lstm or bi_gru.') return net
def bi_lstm(trainX, trainY, testX, testY): trainX = pad_sequences(trainX, maxlen=200, value=0.) testX = pad_sequences(testX, maxlen=200, value=0.) # Converting labels to binary vectors trainY = to_categorical(trainY, nb_classes=2) testY = to_categorical(testY, nb_classes=2) # Network building net = tflearn.input_data(shape=[None, 200]) net = tflearn.embedding(net, input_dim=20000, output_dim=128) net = tflearn.bidirectional_rnn(net, BasicLSTMCell(128), BasicLSTMCell(128)) net = tflearn.dropout(net, 0.5) net = tflearn.fully_connected(net, 2, activation='softmax') net = tflearn.regression(net, optimizer='adam', loss='categorical_crossentropy') # Training model = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2) model.fit(trainX, trainY, validation_set=0.1, show_metric=True, batch_size=64, run_id="rnn-bilstm")
def LoadModel(self): self.window_height = 16 self.window_width = 4 self.threshold = 0.03 self.ground_height = 2 symbol_count = 13 network = input_data( shape=[None, self.window_height * self.window_width, symbol_count]) network = bidirectional_rnn(network, BasicLSTMCell(2), BasicLSTMCell(2)) network = dropout(network, 0.8) network = fully_connected(network, self.window_height * symbol_count, activation='prelu') network = tf.reshape(network, [-1, self.window_height, symbol_count]) network = regression(network, optimizer='adagrad', learning_rate=0.005, loss='mean_square', name='target', batch_size=64) self.model = tflearn.DNN(network) self.model.load('./LSTMmodel/model.tfl')
def learning(self): # Network building net = input_data(shape=[None, MAXIMUM_LENGTH_DATA, 1]) index = tf.placeholder(shape=[None], dtype=tf.int32) # net = bidirectional_rnn(net, BasicLSTMCell(200), BasicLSTMCell(200),return_seq=True) # net = dropout(net, 0.5) # net = tflearn.time_distributed(net, tflearn.fully_connected, [1,'softmax']) # net = tflearn.regression(net, optimizer='adam', learning_rate=0.001, loss='binary_crossentropy') # model = tflearn.DNN(net, tensorboard_verbose=3) # model.fit(self.trainingData.data, self.trainingData.label, validation_set=0.1, show_metric=True, batch_size=1000,n_epoch=100) net = bidirectional_rnn(net, BasicLSTMCell(200), BasicLSTMCell(200), return_seq=True, dynamic=True) net = fully_connected(net[index], 2, activation='softmax') net = regression(net, optimizer='adam', loss='categorical_crossentropy') # # Training model = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2) model.fit(self.trainingData.data, self.trainingData.label, validation_set=0.1, show_metric=True, batch_size=64)
def build_network(self, ilearning_rate=0.001): ''' :param ilearning_rate: :return: Build network ''' #tf.reset_default_graph() net = input_data(shape=[None, 200]) net = embedding(net, input_dim=51887, output_dim=200, trainable=False, name="EmbeddingLayer") #net = embedding(net, input_dim=20000, output_dim=128, trainable=False, weights_init=W, # name="EmbeddingLayer") #net = tflearn.embedding(net, input_dim=20000, output_dim=128, trainable=False, weights_init = W, name="EmbeddingLayer") net = bidirectional_rnn(net, BasicLSTMCell(128), BasicLSTMCell(128)) net = dropout(net, 0.5) net = fully_connected(net, 2, activation='softmax') net = regression(net, optimizer='adam', loss='categorical_crossentropy', learning_rate=ilearning_rate) return net
def lstm_block(input, hidden_units=128, dropout=0.5, reuse=False, layers=1, dynamic=True, return_seq=False, bidirectional=False, return_state=False): output = None prev_output = input for n_layer in range(layers): if not bidirectional: if n_layer < layers - 1: output = tflearn.lstm(prev_output, hidden_units, dropout=dropout, dynamic=dynamic, reuse=reuse, scope='lstm_{}'.format(n_layer), return_seq=True) output = tf.stack(output, axis=0) output = tf.transpose(output, perm=[1, 0, 2]) prev_output = output continue output = tflearn.lstm(prev_output, hidden_units, dropout=dropout, dynamic=dynamic, reuse=reuse, scope='lstm_{}'.format(n_layer), return_seq=return_seq, return_state=return_state) else: if n_layer < layers - 1: output = bidirectional_rnn(prev_output, BasicLSTMCell(hidden_units, reuse=reuse), BasicLSTMCell(hidden_units, reuse=reuse), dynamic=dynamic, scope='blstm_{}'.format(n_layer), return_seq=True) output = tf.stack(output, axis=0) output = tf.transpose(output, perm=[1, 0, 2]) prev_output = output continue output = bidirectional_rnn(prev_output, BasicLSTMCell(hidden_units, reuse=reuse), BasicLSTMCell(hidden_units, reuse=reuse), dynamic=dynamic, scope='blstm_{}'.format(n_layer), return_seq=return_seq, return_states=return_state) return output
def bi_LSTM(): # Network building net = input_data(shape=[None, charvec_len]) net = embedding(net, input_dim=in_dim, output_dim=nn_dim) net = dropout(net, drop1) net = bidirectional_rnn(net, BasicLSTMCell(nn_dim, forget_bias=1.), BasicLSTMCell(nn_dim, forget_bias=1.)) net = dropout(net, drop2) net = fully_connected(net, 2, activation='softmax') net = regression(net, optimizer='adam', loss='categorical_crossentropy', learning_rate=lrate) return net
def bi_LSTM(): # Network building net = input_data(shape=[None, 440]) net = embedding(net, input_dim=20000, output_dim=128) net = dropout(net, 0.9) net = bidirectional_rnn(net, BasicLSTMCell(128, forget_bias=1.), BasicLSTMCell(128, forget_bias=1.)) net = dropout(net, 0.7) net = fully_connected(net, 2, activation='softmax') net = regression(net, optimizer='adam', loss='categorical_crossentropy', learning_rate=0.001) return net
def encoder(x: tf.Tensor, noise, initial_state: LSTMStateTuple, seq_len, n_joints, motion_selection): x = tf.expand_dims(x, axis=1) x = tf.tile(x, [1, seq_len, 1]) x = tf.concat([x, tf.expand_dims(noise, axis=2)], axis=2) with tf.variable_scope('lstm1'): lstm = BasicLSTMCell(2**n_joints + motion_selection) x, final_state = tf.nn.dynamic_rnn(lstm, x, dtype=tf.float32, initial_state=initial_state) x = tf.layers.dense(x, 2**n_joints + motion_selection) with tf.variable_scope('lstm2'): lstm = BasicLSTMCell(2**n_joints + motion_selection) x, final_state = tf.nn.dynamic_rnn(lstm, x, dtype=tf.float32, initial_state=initial_state) x = tf.layers.dense(x, n_joints) x = tf.clip_by_value(x, -pi, pi) return x, final_state
downsample=True) print("resn16", net.get_shape()) '''net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn18", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn20", net.get_shape())''' net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') #net = tflearn.global_avg_pool(net) # LSTM print("before LSTM, before reshape", net.get_shape()) ############ reshape for sub_seq net = tf.reshape(net, [-1, n_dim // n_split, 512]) print("before LSTM", net.get_shape()) net = bidirectional_rnn(net, BasicLSTMCell(256), BasicLSTMCell(256)) print("after LSTM", net.get_shape()) #net = tflearn.layers.recurrent.lstm(net, n_units=512) #print("after LSTM", net.get_shape()) net = dropout(net, 0.5) # Regression net = tflearn.fully_connected(net, 32, activation='sigmoid') net = tflearn.dropout(net, 0.5) net = tflearn.fully_connected(net, 4, activation='softmax') print("dense", net.get_shape()) net = tflearn.regression( net, optimizer='adam', #momentum', loss='categorical_crossentropy') #,learning_rate=0.05)
x_frames = np.array(x_frames) y_frames = np.array(y_frames) x_frames, y_frames = unison_shuffled_copies(x_frames, y_frames) split = int(0.05 * x_frames.shape[0]) valid_x = x_frames[0:split] valid_y = y_frames[0:split] train_x = x_frames[split:] train_y = y_frames[split:] print x_frames.shape print x_frames.shape[1], x_frames.shape[2] net = tflearn.input_data([None, x_frames.shape[1], x_frames.shape[2]]) print net.get_shape().as_list() net = bidirectional_rnn(net, BasicLSTMCell(number_hidden), BasicLSTMCell(number_hidden)) net = dropout(net, 0.8) fc = tflearn.fully_connected(net, highway_size, activation='elu', regularizer='L2', weight_decay=0.001) net = fc for i in xrange(highway_layer_amount): net = tflearn.highway(net, highway_size, activation='elu', regularizer='L2', weight_decay=0.001, transform_dropout=0.8) net = tflearn.fully_connected(net, y_frames.shape[1], activation='elu') net = tflearn.regression(net, optimizer='adam', learning_rate=learning_rate, loss='mean_square')
testX, testY = test # Data preprocessing # Sequence padding trainX = pad_sequences(trainX, maxlen=input_length_each_seq, value=0.) testX = pad_sequences(testX, maxlen=input_length_each_seq, value=0.) # Converting labels to binary vectors trainY = to_categorical(trainY,n_class) testY = to_categorical(testY,n_class) # Network building net = input_data(shape=[None, input_length_each_seq]) # [none,200] net = embedding(net, input_dim=n_datas, output_dim=hiddle_layes) # [none,200,128] net=tf.unstack(net,input_length_each_seq,1) print(net);exit(-1) net = bidirectional_rnn(net, BasicLSTMCell(hiddle_layes), BasicLSTMCell(hiddle_layes)) net = dropout(net, 0.5) net = fully_connected(net, n_class, activation='softmax') net = regression(net, optimizer='adam', loss='categorical_crossentropy') # Training model = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2) model.fit(trainX, trainY, validation_set=0.1, show_metric=True, batch_size=64) 2、tensorflow 参考:TensorFlow-Examples """ Bi-directional Recurrent Neural Network. A Bi-directional Recurrent Neural Network (LSTM) implementation example using TensorFlow library. This example is using the MNIST database of handwritten
def SlstmLayer(incoming, seq_length, input_dim, output_dim, policy, dropout_keepprob=0.5, pooling=False, update="straight", scope=None, name="SlstmLayer"): ''' incomming: [batch_size, sen_length, choose_length, input_dim[0]] [batch_size, sen_length, choose_length, input_dim[1]] output: [batch_size, output_dim[0]] [batch_size, sen_length] ''' with tf.variable_scope(scope, default_name=name, values=[incoming]) as scope: name = scope.name batch_size = tf.shape(incoming[0])[0] sen_length = incoming[0].get_shape()[1].value choose_length = incoming[0].get_shape()[2].value cell = BasicLSTMCell(output_dim[0]) def call_cell(inputs, status): with tf.variable_scope("cell") as scope: ans = cell(inputs, status, scope=scope)[1] cell.reuse = True return ans cell_p = BasicLSTMCell(output_dim[1]) def call_cell_p(inputs, status): with tf.variable_scope("cell_p") as scope: ans = cell_p(inputs, status, scope=scope)[1] cell_p.reuse = True return ans x_seq = tf.unstack(incoming[0], axis=1) h_seq = [tf.zeros([batch_size, output_dim[0]])] * choose_length c_seq = [tf.zeros([batch_size, output_dim[0]])] * choose_length if incoming[1]: x_p_seq = tf.unstack(incoming[1], axis=1) h_p_seq = [tf.zeros([batch_size, output_dim[1]])] * choose_length c_p_seq = [tf.zeros([batch_size, output_dim[1]])] * choose_length else: x_p_seq = x_seq h_p_seq = h_seq c_p_seq = c_seq action_continous_seq = [] action_seq = [] for time, (x, x_p) in enumerate(zip(x_seq, x_p_seq)): h_pre = tf.stack(h_seq[:-choose_length - 1:-1], axis=1) h_pre_s = tf.stop_gradient(h_pre) x_s = tf.stop_gradient(x) if incoming[1]: h_p_pre = tf.stack(h_p_seq[:-choose_length - 1:-1], axis=1) action = policy(tf.concat([h_pre_s, h_p_pre], axis=2), tf.concat([x_s, x_p], axis=2), scope=scope) else: action = policy(h_pre_s, x_s, scope=scope) action_continous_seq.append(action) g = tf.get_default_graph() with ops.name_scope("MultinomialSample") as name: with g.gradient_override_map({ "OneHot": "ST_OneHot", "Multinomial": "ST_Multinomial" }): action = tf.one_hot(tf.multinomial(action, 1), choose_length, on_value=1.0, off_value=0.0, dtype=tf.float32, axis=-1) action_seq.append(action) h_pre = h_seq[:-choose_length - 1:-1] c_pre = c_seq[:-choose_length - 1:-1] x_seq = tf.unstack(x, axis=1) h_p_pre = h_p_seq[:-choose_length - 1:-1] c_p_pre = c_p_seq[:-choose_length - 1:-1] x_p_seq = tf.unstack(x_p, axis=1) all_h = [] all_c = [] all_h_p = [] all_c_p = [] for h, c, h_p, c_p, r, r_p in zip(h_pre, c_pre, h_p_pre, c_p_pre, x_seq, x_p_seq): state = call_cell(r, (c, h)) all_c.append(state[0]) all_h.append(state[1]) if input_dim[1] == 0: all_c_p.append(c_p_seq[-1]) all_h_p.append(h_p_seq[-1]) else: state_p = call_cell_p( tf.concat([tf.stop_gradient(r), r_p], axis=1), (c_p, h_p)) all_c_p.append(state_p[0]) all_h_p.append(state_p[1]) action = tf.reshape(action, [batch_size, choose_length, 1]) now_h = all_h[0] now_c = all_c[0] #now_h = tf.reduce_sum(tf.stack(all_h, axis = 1) * action, axis = 1) # now_h = tf.where(tf.less(time, seq_length), now_h, h_seq[-1]) #now_c = tf.reduce_sum(tf.stack(all_c, axis = 1) * action, axis = 1) # now_c = tf.where(tf.less(time, seq_length), now_c, c_seq[-1]) h_seq.append(now_h) c_seq.append(now_c) if incoming[1]: now_h_p = tf.reduce_sum(tf.stack(all_h_p, axis=1) * action, axis=1) now_h_p = tf.where(tf.less(time, seq_length), now_h_p, h_p_seq[-1]) now_c_p = tf.reduce_sum(tf.stack(all_c_p, axis=1) * action, axis=1) now_c_p = tf.where(tf.less(time, seq_length), now_c_p, c_p_seq[-1]) h_p_seq.append(now_h_p) c_p_seq.append(now_c_p) if pooling: output_h = tf.reduce_max(tf.stack(h_seq, axis=2), axis=2) else: output_h = h_seq[-1] output_h = tflearn.dropout(output_h, dropout_keepprob, name="dropOut") output_action = tf.stack(action_seq, axis=1) action_continous = tf.stack(action_continous_seq, axis=1) tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, output_h) tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, output_action) tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, action_continous) return output_h, output_action
testX, testY = test # Data preprocessing # Sequence padding trainX = pad_sequences(trainX, maxlen=200, value=0.) testX = pad_sequences(testX, maxlen=200, value=0.) # Converting labels to binary vectors trainY = to_categorical(trainY, nb_classes=2) testY = to_categorical(testY, nb_classes=2) # Network building net = input_data(shape=[None, 200]) #same shape as the max length net = embedding( net, input_dim=20000, output_dim=128 ) #creates embedding matrix. Not sure if I need this for project 4 net = bidirectional_rnn(net, BasicLSTMCell(128), BasicLSTMCell( 128)) #has two LSTMs with 128 units go in forward and backward directions. net = dropout(net, 0.5) #dropout with keep probability of 0.5. All are finalized net = fully_connected( net, 2, activation='softmax' ) #makes softmax probabilities over 2 categories, true and false net = regression( net, optimizer='adam', loss='categorical_crossentropy' ) #runs adam optimizer on net minimizing catagorical cross entropy loss # Training model = tflearn.DNN( net, clip_gradients=5, tensorboard_verbose=1 ) # clips gradients at 5. Prints out loss, accuracy and gradients. All are finalized model.fit(
def get_resnet_feature(test_data, test_label, test_pid, pid_map): n_dim = 6000 n_split = 300 tf.reset_default_graph() sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) ### split #X = X.reshape([-1, n_split, 1]) #testX = testX.reshape([-1, n_split, 1]) # Building Residual Network net = tflearn.input_data(shape=[None, n_dim, 1]) ############ reshape for sub_seq net = tf.reshape(net, [-1, n_split, 1]) net = tflearn.conv_1d(net, 64, 16, 2) #net = tflearn.conv_1d(net, 64, 16, 2, regularizer='L2', weight_decay=0.0001) net = tflearn.batch_normalization(net) # Residual blocks net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides=2, downsample=True, is_first_block=True) net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides=2, downsample=True) net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') #net = tflearn.global_avg_pool(net) # LSTM ############ reshape for sub_seq net = tf.reshape(net, [-1, n_dim // n_split, 512]) net = bidirectional_rnn(net, BasicLSTMCell(256), BasicLSTMCell(256)) #net = tflearn.layers.recurrent.lstm(net, n_units=512) #print("after LSTM", net.get_shape()) net = dropout(net, 0.5) # Regression feature_layer = tflearn.fully_connected(net, 32, activation='sigmoid') net = tflearn.dropout(feature_layer, 0.5) net = tflearn.fully_connected(net, 4, activation='softmax') net = tflearn.regression( net, optimizer='adam', #momentum', loss='categorical_crossentropy') #,learning_rate=0.1) ## save model ### load model = tflearn.DNN(net) run_id = 'resnet_6000_500_10_5_v1' model.load('../model/resNet/' + run_id) ### create new model, and get features m2 = tflearn.DNN(feature_layer, session=model.session) tmp_feature = [] num_of_test = len(test_data) cur_data = [] pre = [] for i in range(num_of_test): cur_data.append(test_data[i]) if (i % 2000 == 0 or i == (num_of_test - 1)) and i != 0: #tmp_test_data = test_data[i].reshape([-1, n_dim, 1]) tmp_testX = np.array(cur_data, dtype=np.float32) tmp_feature.extend(m2.predict(tmp_testX.reshape([-1, n_dim, 1]))) cur_data = [] pre.extend(model.predict(tmp_testX)) tmp_feature = np.array(tmp_feature) test_pid = np.array(test_pid, dtype=np.string_) y_num = len(pid_map) features = [[0. for j in range(32)] for i in range(y_num)] re_labels = [[0. for j in range(4)] for i in range(y_num)] y_pre = [[0. for j in range(4)] for i in range(y_num)] y_sec_pre = [[0. for j in range(4)] for i in range(y_num)] y_third_pre = [[0. for j in range(4)] for i in range(y_num)] y_fourth_pre = [[0. for j in range(4)] for i in range(y_num)] y_fifth_pre = [[0. for j in range(4)] for i in range(y_num)] y_sixth_pre = [[0. for j in range(4)] for i in range(y_num)] y_seventh_pre = [[0. for j in range(4)] for i in range(y_num)] y_groundtruth = [[0. for j in range(4)] for i in range(y_num)] #print(y_num) for j in range(len(tmp_feature)): feature_pred = np.array(tmp_feature[j], dtype=np.float32) #print(len(feature_pred)) i_pred = np.array(pre[j], dtype=np.float32) cur_pid = str(test_pid[j], 'utf-8') list_id = pid_map[cur_pid] #print (list_id) temp_feature = np.array(features[list_id], dtype=np.float32) temp_pre = np.array(y_pre[list_id], dtype=np.float32) temp_sec_pre = np.array(y_sec_pre[list_id], dtype=np.float32) temp_third_pre = np.array(y_third_pre[list_id], dtype=np.float32) #print(temp_pre) max_p = temp_pre[np.argmax(temp_pre)] max_sec_p = temp_sec_pre[np.argmax(temp_sec_pre)] max_third_p = temp_third_pre[np.argmax(temp_third_pre)] sec_p = 0 sec_sec_p = 0 sec_third_p = 0 for k in range(len(temp_pre)): if temp_pre[k] == max_p: continue if temp_pre[k] > sec_p: sec_p = temp_pre[k] if temp_sec_pre[k] == max_sec_p: continue if temp_sec_pre[k] > sec_sec_p: sec_sec_p = temp_sec_pre[k] if temp_third_pre[k] == max_third_p: continue if temp_third_pre[k] > sec_third_p: sec_third_p = temp_third_pre[k] cur_max_p = i_pred[np.argmax(i_pred)] cur_sec_p = 0 for k in range(len(i_pred)): if i_pred[k] == cur_max_p: continue if i_pred[k] > cur_sec_p: cur_sec_p = i_pred[k] if (cur_max_p - cur_sec_p) > (max_p - sec_p): y_seventh_pre[list_id] = y_sixth_pre[list_id] y_sixth_pre[list_id] = y_fifth_pre[list_id] y_fifth_pre[list_id] = y_fourth_pre[list_id] y_fourth_pre[list_id] = y_third_pre[list_id] y_third_pre[list_id] = y_sec_pre[list_id] y_sec_pre[list_id] = y_pre[list_id] y_pre[list_id] = i_pred elif (cur_max_p - cur_sec_p) > (max_sec_p - sec_sec_p): y_seventh_pre[list_id] = y_sixth_pre[list_id] y_sixth_pre[list_id] = y_fifth_pre[list_id] y_fifth_pre[list_id] = y_fourth_pre[list_id] y_fourth_pre[list_id] = y_third_pre[list_id] y_third_pre[list_id] = y_sec_pre[list_id] y_sec_pre[list_id] = i_pred elif (cur_max_p - cur_sec_p) > (max_third_p - sec_third_p): y_seventh_pre[list_id] = y_sixth_pre[list_id] y_sixth_pre[list_id] = y_fifth_pre[list_id] y_fifth_pre[list_id] = y_fourth_pre[list_id] y_fourth_pre[list_id] = y_third_pre[list_id] y_third_pre[list_id] = i_pred elif (cur_max_p - cur_sec_p) > (max_fourth_p - sec_fourth_p): y_seventh_pre[list_id] = y_sixth_pre[list_id] y_sixth_pre[list_id] = y_fifth_pre[list_id] y_fifth_pre[list_id] = y_fourth_pre[list_id] y_fourth_pre[list_id] = i_pred elif (cur_max_p - cur_sec_p) > (max_fifth_p - sec_fifth_p): y_seventh_pre[list_id] = y_sixth_pre[list_id] y_sixth_pre[list_id] = y_fifth_pre[list_id] y_fifth_pre[list_id] = i_pred elif (cur_max_p - cur_sec_p) > (max_sixth_p - sec_sixth_p): y_seventh_pre[list_id] = y_sixth_pre[list_id] y_sixth_pre[list_id] = i_pred elif (cur_max_p - cur_sec_p) > (max_seventh_p - sec_seventh_p): y_seventh_pre[list_id] = i_pred max_f = 0 for k in range(len(temp_feature)): if temp_feature[k] > max_f: max_f = temp_feature[k] if max_f > 0: feature_pred = (feature_pred + temp_feature) / 2 #for k in range(len(temp_feature)): # feature_pred[k] = (feature_pred[k]+temp_feature[k])/2 features[list_id] = feature_pred y_groundtruth[list_id] = test_label[j] gt_list = ["N", "A", "O", "~"] pred_1 = gt_list[np.argmax(i_pred)] if pred_1 == 'N': re_labels[list_id][0] += 1 elif pred_1 == 'A': re_labels[list_id][1] += 1 elif pred_1 == 'O': re_labels[list_id][2] += 1 elif pred_1 == '~': re_labels[list_id][3] += 1 else: print('wrong label') out_feature = [] for i in range(len(features)): out_feature.append(features[i]) out_feature = np.array(out_feature) for k in range(len(y_pre)): labels = [0. for j in range(4)] pred_1 = np.argmax(y_pre[k]) labels[pred_1] += 1 pred_2 = np.argmax(y_sec_pre[k]) labels[pred_2] += 1 pred_3 = np.argmax(y_third_pre[k]) labels[pred_3] += 1 if pred_1 == 2: print("O was selected!") continue elif pred_2 == 2: y_pre[k] = y_sec_pre[k] print("O was selected!") elif pred_3 == 2: y_pre[k] = y_third_pre[k] print("O was selected!") if pred_1 != np.argmax(labels): if pred_2 == np.argmax(labels): y_pre[k] = y_sec_pre[k] print("Second was selected!") MyEval.F1Score3_num(pre, test_label[:len(pre)]) MyEval.F1Score3_num(y_pre, y_groundtruth) MyEval.F1Score3_num(re_labels, y_groundtruth) return out_feature
def get_deep_mimic_feats(test_data): n_dim = 6000 n_split = 300 tf.reset_default_graph() sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) # Building Residual Network net = tflearn.input_data(shape=[None, n_dim, 1]) print("input", net.get_shape()) ############ reshape for sub_seq net = tf.reshape(net, [-1, n_split, 1]) print("reshaped input", net.get_shape()) net = tflearn.conv_1d(net, 64, 16, 2) #net = tflearn.conv_1d(net, 64, 16, 2, regularizer='L2', weight_decay=0.0001) print("cov1", net.get_shape()) net = tflearn.batch_normalization(net) print("bn1", net.get_shape()) net = tflearn.activation(net, 'relu') print("relu1", net.get_shape()) # Residual blocks net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides = 2, downsample=True, is_first_block = True) print("resn2", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides = 4, downsample=True) print("resn4", net.get_shape()) # net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides = 4, downsample=True) # print("resn6", net.get_shape()) # net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides = 4, downsample=True) # print("resn8", net.get_shape()) net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') print("before reshape", net.get_shape()) # net = tf.reshape(net, [-1, n_dim//n_split*net.get_shape()[-2], net.get_shape()[-1]]) # LSTM ############ reshape for sub_seq before_reshaped_shape = net.get_shape().as_list() net = tf.reshape(net, [-1, n_dim//n_split, before_reshaped_shape[1]*before_reshaped_shape[2]]) print("before LSTM", net.get_shape()) net = bidirectional_rnn(net, BasicLSTMCell(64), BasicLSTMCell(64)) print("after LSTM", net.get_shape()) net = dropout(net, 0.5) # Regression net = tflearn.fully_connected(net, 32, activation='sigmoid') net = tflearn.fully_connected(net, 4, activation='softmax') print("dense", net.get_shape()) net = tflearn.regression(net, optimizer='adam', loss='mean_square') # Training model = tflearn.DNN(net) model.load('../model/mimic/mimic_model_offline_v4.1') pred = [] num_of_test = len(test_data) cur_data = [] for i in range(num_of_test): cur_data.append(test_data[i]) if (i % 2000 == 0 or i == (num_of_test - 1)) and i !=0: tmp_testX = np.array(cur_data, dtype=np.float32) pred.extend(model.predict(tmp_testX.reshape([-1, n_dim, 1]))) cur_data = [] return pred
# Sequence padding trainX = pad_sequences(trainX, maxlen=maxLen, value=0.) #Converting labels to binary vectors trainY = pad_sequences(trainY, maxlen=2, value=0.) embeddings = concat_2Dvectors(embeddings, Flatten_3Dto2D(POS_vectors)) # Network building print("Beginning neural network") net = input_data(shape=[None, maxLen]) net = embedding(net, input_dim=len(embeddings), output_dim=len(embeddings[0]), trainable=False, name="EmbeddingLayer") print(net.get_shape().as_list()) net = bidirectional_rnn(net, BasicLSTMCell(1024), BasicLSTMCell(1024)) net = dropout(net, 0.5) net = fully_connected(net, 2, activation='softmax') net = regression(net, optimizer='adam', loss='categorical_crossentropy', learning_rate=0.005) testX = trainX[int(0.3 * len(trainY)):] testY = trainY[int(0.3 * len(trainY)):] # Training model = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2) embeddingWeights = tflearn.get_layer_variables_by_name('EmbeddingLayer')[0] # Assign your own weights (for example, a numpy array [input_dim, output_dim]) model.set_weights(embeddingWeights, embeddings)
def get_model(): n_dim = 6000 n_split = 300 tf.reset_default_graph() sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) ### split #X = X.reshape([-1, n_split, 1]) #testX = testX.reshape([-1, n_split, 1]) # Building Residual Network net = tflearn.input_data(shape=[None, n_dim, 1]) print("input", net.get_shape()) ############ reshape for sub_seq net = tf.reshape(net, [-1, n_split, 1]) print("reshaped input", net.get_shape()) net = tflearn.conv_1d(net, 64, 16, 2) #net = tflearn.conv_1d(net, 64, 16, 2, regularizer='L2', weight_decay=0.0001) print("cov1", net.get_shape()) net = tflearn.batch_normalization(net) print("bn1", net.get_shape()) net = tflearn.activation(net, 'relu') print("relu1", net.get_shape()) # Residual blocks '''net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides = 2, downsample=True, is_first_block = True) print("resn2", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides = 2, downsample=True) print("resn4", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides = 2, downsample=True) print("resn6", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides = 2, downsample=True) print("resn8", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn10", net.get_shape())''' net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides=2, downsample=True, is_first_block=True) print("resn2", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides=2, downsample=True) print("resn4", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides=2, downsample=True) print("resn6", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides=2, downsample=True) print("resn8", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides=2, downsample=True) print("resn10", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides=2, downsample=True) print("resn12", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides=2, downsample=True) print("resn14", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides=2, downsample=True) print("resn16", net.get_shape()) '''net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn18", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn20", net.get_shape())''' net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') #net = tflearn.global_avg_pool(net) # LSTM print("before LSTM, before reshape", net.get_shape()) ############ reshape for sub_seq net = tf.reshape(net, [-1, n_dim // n_split, 512]) print("before LSTM", net.get_shape()) net = bidirectional_rnn(net, BasicLSTMCell(256), BasicLSTMCell(256)) print("after LSTM", net.get_shape()) #net = tflearn.layers.recurrent.lstm(net, n_units=512) #print("after LSTM", net.get_shape()) net = dropout(net, 0.5) # Regression feature_layer = tflearn.fully_connected(net, 32, activation='sigmoid') net = tflearn.dropout(feature_layer, 0.5) net = tflearn.fully_connected(net, 4, activation='softmax') print("dense", net.get_shape()) net = tflearn.regression( net, optimizer='adam', #momentum', loss='categorical_crossentropy') #,learning_rate=0.1) ## save model ### load model = tflearn.DNN(net) run_id = 'resnet_6000_500_10_5_v1' model.load('../model/resNet/' + run_id) all_names = tflearn.variables.get_all_variables() print(all_names[0]) ttt = model.get_weights(all_names[0]) print(type(ttt)) print(ttt) # tflearn.variables.get_value(all_names[0], xxx) return all_names
# IMDB Dataset loading train, test, _ = imdb.load_data(path='imdb.pkl', n_words=10000, valid_portion=0.1) trainX, trainY = train testX, testY = test # Data preprocessing # Sequence padding trainX = pad_sequences(trainX, maxlen=200, value=0.) testX = pad_sequences(testX, maxlen=200, value=0.) # Converting labels to binary vectors trainY = to_categorical(trainY,2) testY = to_categorical(testY,2) # Network building net = input_data(shape=[None, 200]) net = embedding(net, input_dim=20000, output_dim=128) net = bidirectional_rnn(net, BasicLSTMCell(128), BasicLSTMCell(128)) net = dropout(net, 0.5) net = fully_connected(net, 2, activation='softmax') net = regression(net, optimizer='adam', loss='categorical_crossentropy') # Training model = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2) model.fit(trainX, trainY, n_epoch=100,validation_set=0.1, show_metric=True, batch_size=64) model.save('sentiment_model.tfl')
def get_resnet_feature(test_data): n_dim = 6000 n_split = 300 tf.reset_default_graph() sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) ### split #X = X.reshape([-1, n_split, 1]) #testX = testX.reshape([-1, n_split, 1]) # Building Residual Network net = tflearn.input_data(shape=[None, n_dim, 1]) print("input", net.get_shape()) ############ reshape for sub_seq net = tf.reshape(net, [-1, n_split, 1]) print("reshaped input", net.get_shape()) net = tflearn.conv_1d(net, 64, 16, 2) #net = tflearn.conv_1d(net, 64, 16, 2, regularizer='L2', weight_decay=0.0001) print("cov1", net.get_shape()) net = tflearn.batch_normalization(net) print("bn1", net.get_shape()) net = tflearn.activation(net, 'relu') print("relu1", net.get_shape()) # Residual blocks '''net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides = 2, downsample=True, is_first_block = True) print("resn2", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides = 2, downsample=True) print("resn4", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides = 2, downsample=True) print("resn6", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides = 2, downsample=True) print("resn8", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn10", net.get_shape())''' net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides = 2, downsample=True, is_first_block = True) print("resn2", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides = 2, downsample=True) print("resn4", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides = 2, downsample=True) print("resn6", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides = 2, downsample=True) print("resn8", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides = 2, downsample=True) print("resn10", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides = 2, downsample=True) print("resn12", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides = 2, downsample=True) print("resn14", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides = 2, downsample=True) print("resn16", net.get_shape()) '''net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn18", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn20", net.get_shape())''' net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') #net = tflearn.global_avg_pool(net) # LSTM print("before LSTM, before reshape", net.get_shape()) ############ reshape for sub_seq net = tf.reshape(net, [-1, n_dim//n_split, 512]) print("before LSTM", net.get_shape()) net = bidirectional_rnn(net, BasicLSTMCell(256), BasicLSTMCell(256)) print("after LSTM", net.get_shape()) #net = tflearn.layers.recurrent.lstm(net, n_units=512) #print("after LSTM", net.get_shape()) net = dropout(net, 0.5) # Regression net = tflearn.fully_connected(net, 32, activation='sigmoid') net = tflearn.dropout(net, 0.5) # net, feature_layer = tflearn.fully_connected(net, 4, activation='softmax', return_logit = True) feature_layer = tflearn.fully_connected(net, 4, activation='softmax') print('feature_layer: ', feature_layer.get_shape()) print("dense", net.get_shape()) net = tflearn.regression(net, optimizer='adam',#momentum', loss='categorical_crossentropy') #,learning_rate=0.1) print('final output: ', net.get_shape()) ## save model ### load model = tflearn.DNN(net) run_id = 'resnet_6000_500_10_5_v1' model.load('../model/resNet/'+run_id) # print(tflearn.variables.get_all_variables()) ### create new model, and get features m2 = tflearn.DNN(feature_layer, session=model.session) tmp_feature = [] num_of_test = len(test_data) cur_data = [] pre = [] for i in range(num_of_test): cur_data.append(test_data[i]) if (i % 2000 == 0 or i == (num_of_test - 1)) and i !=0: #tmp_test_data = test_data[i].reshape([-1, n_dim, 1]) tmp_testX = np.array(cur_data, dtype=np.float32) tmp_feature.extend(m2.predict(tmp_testX.reshape([-1, n_dim, 1]))) cur_data = [] pre.extend(model.predict(tmp_testX)) print(i, len(tmp_feature), len(tmp_feature[0])) tmp_feature = np.array(tmp_feature) return tmp_feature
def lstm(incoming, n_units, activation='tanh', inner_activation='sigmoid', dropout=None, bias=True, weights_init=None, forget_bias=1.0, return_seq=False, return_state=False, initial_state=None, dynamic=False, trainable=True, restore=True, reuse=False, scope=None, name="LSTM"): """ LSTM. Long Short Term Memory Recurrent Layer. Input: 3-D Tensor [samples, timesteps, input dim]. Output: if `return_seq`: 3-D Tensor [samples, timesteps, output dim]. else: 2-D Tensor [samples, output dim]. Arguments: incoming: `Tensor`. Incoming 3-D Tensor. n_units: `int`, number of units for this layer. activation: `str` (name) or `function` (returning a `Tensor`). Activation applied to this layer (see tflearn.activations). Default: 'tanh'. inner_activation: `str` (name) or `function` (returning a `Tensor`). LSTM inner activation. Default: 'sigmoid'. dropout: `tuple` of `float`: (input_keep_prob, output_keep_prob). The input and output keep probability. bias: `bool`. If True, a bias is used. weights_init: `str` (name) or `Tensor`. Weights initialization. (See tflearn.initializations). forget_bias: `float`. Bias of the forget gate. Default: 1.0. return_seq: `bool`. If True, returns the full sequence instead of last sequence output only. return_state: `bool`. If True, returns a tuple with output and states: (output, states). initial_state: `Tensor`. An initial state for the RNN. This must be a tensor of appropriate type and shape [batch_size x cell.state_size]. dynamic: `bool`. If True, dynamic computation is performed. It will not compute RNN steps above the sequence length. Note that because TF requires to feed sequences of same length, 0 is used as a mask. So a sequence padded with 0 at the end must be provided. When computation is performed, it will stop when it meets a step with a value of 0. trainable: `bool`. If True, weights will be trainable. restore: `bool`. If True, this layer weights will be restored when loading a model. reuse: `bool`. If True and 'scope' is provided, this layer variables will be reused (shared). scope: `str`. Define this layer scope (optional). A scope can be used to share variables between layers. Note that scope will override name. name: `str`. A name for this layer (optional). References: Long Short Term Memory, Sepp Hochreiter & Jurgen Schmidhuber, Neural Computation 9(8): 1735-1780, 1997. Links: [http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf] (http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf) """ cell = BasicLSTMCell(n_units, activation=activation, inner_activation=inner_activation, forget_bias=forget_bias, bias=bias, weights_init=weights_init, trainable=trainable, restore=restore, reuse=reuse) x = _rnn_template(incoming, cell=cell, dropout=dropout, return_seq=return_seq, return_state=return_state, initial_state=initial_state, dynamic=dynamic, scope=scope, name=name) return x
# Converting labels to binary vectors trainY = to_categorical(trainY, nb_classes=2) testY = to_categorical(testY, nb_classes=2) # Building network network = input_data(shape=[None, maxlen], name='input') network = embedding(network, input_dim=vocab_size, output_dim=embedding_dim, trainable=True) network = bidirectional_rnn(network, BasicLSTMCell(rnn_hidden_size, activation='tanh', inner_activation='sigmoid'), BasicLSTMCell(rnn_hidden_size, activation='tanh', inner_activation='sigmoid'), return_seq=True, dynamic=True) network = tf.pack(network, axis=1) fw_outputs, bw_outputs = tf.split(split_dim=2, num_split=2, value=network) network = tf.add(fw_outputs, bw_outputs) branch1 = conv_1d(network, num_filters, 3, padding='valid',
def get_resNet_proba(long_data, long_pid, model_path): all_pid = np.array(long_pid) all_feature = np.array(long_data) all_label = np.array([]) test_data, test_label, test_pid = slide_and_cut(all_feature, all_label, all_pid) pid_map = {} for i in range(len(all_pid)): pid_map[all_pid[i]] = i n_dim = 6000 n_split = 300 tf.reset_default_graph() sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) # Building Residual Network net = tflearn.input_data(shape=[None, n_dim, 1]) ############ reshape for sub_seq net = tf.reshape(net, [-1, n_split, 1]) net = tflearn.conv_1d(net, 64, 16, 2) net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') # Residual blocks net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides=2, downsample=True, is_first_block=True) net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides=2, downsample=True) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides=2, downsample=True) net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') #net = tflearn.global_avg_pool(net) # LSTM ############ reshape for sub_seq net = tf.reshape(net, [-1, n_dim // n_split, 512]) net = bidirectional_rnn(net, BasicLSTMCell(256), BasicLSTMCell(256)) net = dropout(net, 0.5) # Regression feature_layer = tflearn.fully_connected(net, 32, activation='sigmoid') net = tflearn.dropout(feature_layer, 0.5) net = tflearn.fully_connected(net, 4, activation='softmax') net = tflearn.regression( net, optimizer='adam', #momentum', loss='categorical_crossentropy') #,learning_rate=0.1) ## save model ### load model = tflearn.DNN(net) model.load(model_path) ### create new model, and get features num_of_test = len(test_data) cur_data = [] pre = [] for i in range(num_of_test): cur_data.append(test_data[i]) if (num_of_test > 1 and (i % 2000 == 0 or i == (num_of_test - 1)) and i != 0) or (num_of_test == 1): tmp_testX = np.array(cur_data, dtype=np.float32) cur_data = [] pre.extend(model.predict(tmp_testX)) test_pid = np.array(test_pid, dtype=np.string_) y_num = len(pid_map) y_pre = [[0. for j in range(4)] for i in range(y_num)] y_sec_pre = [[0. for j in range(4)] for i in range(y_num)] y_third_pre = [[0. for j in range(4)] for i in range(y_num)] #print(y_num) for j in range(len(pre)): i_pred = np.array(pre[j], dtype=np.float32) cur_pid = str(test_pid[j], 'utf-8') list_id = pid_map[cur_pid] temp_pre = np.array(y_pre[list_id], dtype=np.float32) temp_sec_pre = np.array(y_sec_pre[list_id], dtype=np.float32) temp_third_pre = np.array(y_third_pre[list_id], dtype=np.float32) max_p = temp_pre[np.argmax(temp_pre)] max_sec_p = temp_sec_pre[np.argmax(temp_sec_pre)] max_third_p = temp_third_pre[np.argmax(temp_third_pre)] sec_p = 0 sec_sec_p = 0 sec_third_p = 0 for k in range(len(temp_pre)): if temp_pre[k] == max_p: continue if temp_pre[k] > sec_p: sec_p = temp_pre[k] if temp_sec_pre[k] == max_sec_p: continue if temp_sec_pre[k] > sec_sec_p: sec_sec_p = temp_sec_pre[k] if temp_third_pre[k] == max_third_p: continue if temp_third_pre[k] > sec_third_p: sec_third_p = temp_third_pre[k] cur_max_p = i_pred[np.argmax(i_pred)] cur_sec_p = 0 for k in range(len(i_pred)): if i_pred[k] == cur_max_p: continue if i_pred[k] > cur_sec_p: cur_sec_p = i_pred[k] if (cur_max_p - cur_sec_p) > (max_p - sec_p): y_third_pre[list_id] = y_sec_pre[list_id] y_sec_pre[list_id] = y_pre[list_id] y_pre[list_id] = i_pred elif (cur_max_p - cur_sec_p) > (max_sec_p - sec_sec_p): y_third_pre[list_id] = y_sec_pre[list_id] y_sec_pre[list_id] = i_pred elif (cur_max_p - cur_sec_p) > (max_third_p - sec_third_p): y_third_pre[list_id] = i_pred for k in range(len(y_pre)): labels = [0. for j in range(4)] pred_1 = np.argmax(y_pre[k]) labels[pred_1] += 1 pred_2 = np.argmax(y_sec_pre[k]) labels[pred_2] += 1 pred_3 = np.argmax(y_third_pre[k]) labels[pred_3] += 1 if pred_1 == 2: # and (abs(y_pre[k][np.argmax(labels)] - y_pre[k][2])/y_pre[k][np.argmax(labels)] <= 0.2): continue elif pred_2 == 2: # and (abs(y_pre[k][np.argmax(labels)] - y_sec_pre[k][2])/y_pre[k][np.argmax(labels)] <= 0.2): y_pre[k] = y_sec_pre[k] elif pred_3 == 2: # and (abs(y_pre[k][np.argmax(labels)] - y_third_pre[k][2])/y_pre[k][np.argmax(labels)] <= 0.2): y_pre[k] = y_third_pre[k] elif pred_1 != np.argmax(labels): if pred_2 == np.argmax(labels): y_pre[k] = y_sec_pre[k] return y_pre
# pickle.dump (X_test, open ("xtest.p", b)) # X_train = pickle.load (open ("xtrain.p", rb)) # X_test = pickle.load (open ("xtest.p", rb)) ### Models print('Build model') net = input_data([None, model_size]) net = embedding(net, input_dim=n_words, output_dim=embedding_size) if cell_type == "lstm": for i in range(len(cell_size)): if i < len(cell_size) - 1: net = bidirectional_rnn(net, BasicLSTMCell(cell_size[i]), BasicLSTMCell(cell_size[i]), return_seq=True) net = dropout(net, dropout_ratio) else: net = bidirectional_rnn(net, BasicLSTMCell(cell_size[i]), BasicLSTMCell(cell_size[i])) net = dropout(net, dropout_ratio) elif cell_type == "gru": for i in range(len(cell_size)): if i < len(cell_size) - 1: net = bidirectional_rnn(net, GRUCell(cell_size[i]), GRUCell(cell_size[i]), return_seq=True) net = dropout(net, dropout_ratio)
#Get data trainX, trainY, maxLen, vocabSize, _ = data.get_Data_Vectors() # Data preprocessing # Sequence padding trainX = pad_sequences(trainX, maxlen=maxLen, value=0.) #Converting labels to binary vectors trainY = pad_sequences(trainY, maxlen=maxLen, value=0.) # Network building print("Beginning neural network") net = input_data(shape=[None, maxLen]) net = embedding(net, input_dim=vocabSize, output_dim=128) #print(net.get_shape().as_list()) net = bidirectional_rnn(net, BasicLSTMCell(512), BasicLSTMCell(512)) net = dropout(net, 0.5) net = fully_connected(net, maxLen, activation='softmax') net = regression(net, optimizer='adam', loss='categorical_crossentropy') #, learning_rate=0.001) #""" # Training model = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2) model.fit(trainX, trainY, validation_set=0.2, show_metric=True, batch_size=64, shuffle=True) #"""
trainX = pad_sequences(trainX, maxlen=maxLen, value=0) #Converting labels to binary vectors trainY = pad_sequences(trainY, maxlen=maxLen, value=0) embeddings = concat_2Dvectors(embeddings, Flatten_3Dto2D(POS_vectors)) # Network building print("Beginning neural network") net = input_data(shape=[None, maxLen]) net = embedding(net, input_dim=len(embeddings), output_dim=len(embeddings[0]), trainable=False, name="EmbeddingLayer") print("After embeddings : ", net.get_shape().as_list()) net = bidirectional_rnn(net, BasicLSTMCell(1024), BasicLSTMCell(1024), return_seq=True) #net = [dropout(net[i], 0.5) for i in range(len(net))] net = [ fully_connected(net[i], 1, activation='sigmoid') for i in range(len(net)) ] net = merge(net, mode='concat') print("After RNN : ", net.get_shape().as_list()) print("After Dropout : ", net.get_shape().as_list()) net = regression(net, optimizer='adam', loss='binary_crossentropy', learning_rate=0.005) print("After regression : ", net.get_shape().as_list())
def make_core_network(net, regularizer='L2'): ############ reshape for sub_seq net = tf.reshape(net, [-1, n_split, 1]) print("reshaped input", net.get_shape()) net = tflearn.conv_1d(net, 64, 16, 2) #net = tflearn.conv_1d(net, 64, 16, 2, regularizer='L2', weight_decay=0.0001) print("cov1", net.get_shape()) net = tflearn.batch_normalization(net) print("bn1", net.get_shape()) net = tflearn.activation(net, 'relu') print("relu1", net.get_shape()) # Residual blocks '''net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides = 2, downsample=True, is_first_block = True) print("resn2", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides = 2, downsample=True) print("resn4", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides = 2, downsample=True) print("resn6", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides = 2, downsample=True) print("resn8", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn10", net.get_shape())''' net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides=2, downsample=True, is_first_block=True) print("resn2", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 64, downsample_strides=2, downsample=True) print("resn4", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides=2, downsample=True) print("resn6", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 128, downsample_strides=2, downsample=True) print("resn8", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides=2, downsample=True) print("resn10", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 256, downsample_strides=2, downsample=True) print("resn12", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides=2, downsample=True) print("resn14", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 512, downsample_strides=2, downsample=True) print("resn16", net.get_shape()) '''net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn18", net.get_shape()) net = tflearn.residual_bottleneck(net, 2, 16, 1024, downsample_strides = 2, downsample=True) print("resn20", net.get_shape())''' net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') #net = tflearn.global_avg_pool(net) # LSTM print("before LSTM, before reshape", net.get_shape()) ############ reshape for sub_seq net = tf.reshape(net, [-1, n_dim // n_split, 512]) print("before LSTM", net.get_shape()) net = bidirectional_rnn(net, BasicLSTMCell(256), BasicLSTMCell(256)) print("after LSTM", net.get_shape()) #net = tflearn.layers.recurrent.lstm(net, n_units=512) #print("after LSTM", net.get_shape()) net = dropout(net, 0.5) # Regression feature_layer = tflearn.fully_connected(net, 32, activation='sigmoid') net = tflearn.dropout(feature_layer, 0.5) net = tflearn.fully_connected(net, 4, activation='softmax') print("dense", net.get_shape()) return net, feature_layer