def pre_emph_test(coeff, canvas_size): x_ = tf.placeholder(tf.float32, shape=[ canvas_size, ]) x_preemph = pre_emph(x_, coeff) return x_, x_preemph
dec_full = full_layer (rnn_output, self.w['dec_full'], 'dec_full') dec_full = tf.nn.dropout(dec_full, dropout_p) full_out = full_layer (dec_full, self.w['out'], 'full_out') # full_out = tf.add( self.biases['out'], tf.matmul( dec_full, self.w['out']) ) # full_out = tf.nn.tanh(full_out) return full_out, dec_rnn_state ###############################################################s # Construct the residual model if emphasis: X= pre_emph(X) residue =X # '- Since at the beginning AEoutput is zero AE = codec(weights) init_enc = AE.rnn_enc.zero_state(tf.shape(X)[0], tf.float32) init_dec = AE.rnn_dec.zero_state(tf.shape(X)[0], tf.float32) #%% The entire AUTOENCODER # with tf.variable_scope("ae"): # tf.get_variable_scope().reuse_variables()
def pre_emph_test(coeff, canvas_size): x_ = tf.placeholder(tf.float32, shape=[canvas_size,]) x_preemph = pre_emph(x_, coeff) return x_, x_preemph
def rnn(input_dim=512, learning_rate=.0001, training_epochs=50, batch_size=128, full_width=1024, rnn_width=512, binary_width=192, num_steps=4, dropout_p=1., display_step=100, scope='rnn_self', emphasis='False'): #%% #loading data # We use 10 mat files (each 200 MB) for training and one for test #input_dim =128; with tf.variable_scope(scope): #%% Parameters #learning_rate = .0001 #training_epochs = 1 #batch_size = 128 #full_width = 1024 #rnn_width = 512 #binary_width = 32 #num_steps = 4 #dropout_p = 0.8 #display_step = 100 #%% print('input_dim', input_dim) print('batch_size', batch_size) print('full_width', full_width) print('rnn_width', rnn_width) print('binary_width', binary_width) print('num_steps', num_steps) print('dropout_p', dropout_p) print('learning_rate_new', learning_rate) #%% ############################################################################## # input, weights and biases X = tf.placeholder("float", [None, input_dim]) #drop_out_p=tf.placeholder("float", [1]) mode = tf.placeholder("float", None) # for quantizing neurons std_weight = 0.01 #( 2.0 / max( [full_width, rnn_width] ))**0.5; std_bias = 0.001 #%% def w_b_gen(shape_param, stddev_param): weight = tf.Variable( tf.random_normal(shape_param, mean=0.0, stddev=stddev_param)) return weight #%% Generating weights weights = {} biases = {} weights['enc_full'] = w_b_gen([input_dim, full_width], std_weight) weights['middle'] = w_b_gen([rnn_width, binary_width], std_weight) weights['dec_full'] = w_b_gen([rnn_width, full_width], std_weight) weights['out'] = w_b_gen([full_width, input_dim], std_weight) biases['out'] = w_b_gen([input_dim], std_bias) #%%############################################# ################### RNN layer ################# ################################################ def rnn_block(num_neurons): rnn = tf.contrib.rnn.BasicLSTMCell(num_neurons, state_is_tuple=True) rnn = tf.contrib.rnn.MultiRNNCell([rnn] * 2, state_is_tuple=True) # rnn = tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.BasicLSTMCell(num_neurons, state_is_tuple=True) for _ in range(2)], state_is_tuple=True) return rnn #%%############################################################################## #%% Full layer def full_layer(x, w, scope): try: with tf.variable_scope(scope, reuse=True): tf.get_variable_scope().reuse_variables() layer = batch_norm(x, w, scope) print('reuse') except ValueError: with tf.variable_scope(scope): print('creation') layer = batch_norm(x, w, scope) # layer = tf.matmul(x, w) layer = tf.nn.tanh(layer) return layer #%% def batch_norm(x, W, scope): num_neurons = W.get_shape()[1].value epsilon = 1e-3 z_BN = tf.matmul(x, W) # x * W batch_mean, batch_var = tf.nn.moments(z_BN, [0]) scale = tf.get_variable( scope + 'scale', shape=[num_neurons], dtype=tf.float32) # (tf.ones([num_neurons])) beta = tf.get_variable(scope + 'beta', shape=[num_neurons], dtype=tf.float32) x_BN = tf.nn.batch_normalization(z_BN, batch_mean, batch_var, beta, scale, epsilon) return x_BN #%%############################################################################## # Building the codec b_q = binary_quantizer(tf) #enc=tf.get_variable() class codec(): def __init__(self, w): self.rnn_enc = rnn_block(rnn_width) self.rnn_dec = rnn_block(rnn_width) self.w = w self.biases = biases def encoder(self, x, init_state): enc_full = full_layer(x, self.w['enc_full'], 'enc_full') enc_full = tf.nn.dropout(enc_full, dropout_p) try: with tf.variable_scope('enc', reuse=True): tf.get_variable_scope().reuse_variables() rnn_output, enc_rnn_state = self.rnn_enc( enc_full, init_state) except ValueError: with tf.variable_scope('enc'): rnn_output, enc_rnn_state = self.rnn_enc( enc_full, init_state) # self.enc_rnn_output = tf.reshape(self.enc_rnn_output, [-1, rnn_width]) full_middle = full_layer(rnn_output, self.w['middle'], 'full_middle') full_middle = b_q(full_middle, mode) return full_middle, enc_rnn_state ############################################ def decoder(self, x, init_state): # x = tf.reshape(x, [-1, 1, binary_width]) # with tf.variable_scope('dec', reuse=True): # tf.get_variable_scope().reuse_variables() # rnn_dec = rnn_block(rnn_width) try: with tf.variable_scope('dec', reuse=True): tf.get_variable_scope().reuse_variables() rnn_output, dec_rnn_state = self.rnn_dec(x, init_state) except ValueError: with tf.variable_scope('dec'): rnn_output, dec_rnn_state = self.rnn_dec(x, init_state) # self.dec_rnn_output = tf.reshape(self.dec_rnn_output, [-1, rnn_width]) dec_full = full_layer(rnn_output, self.w['dec_full'], 'dec_full') dec_full = tf.nn.dropout(dec_full, dropout_p) full_out = full_layer(dec_full, self.w['out'], 'full_out') # full_out = tf.add( self.biases['out'], tf.matmul( dec_full, self.w['out']) ) # full_out = tf.nn.tanh(full_out) return full_out, dec_rnn_state ###############################################################s # Construct the residual model if emphasis: X = pre_emph(X) residue = X # '- Since at the beginning output is zero AE = codec(weights) init_enc = AE.rnn_enc.zero_state(tf.shape(X)[0], tf.float32) init_dec = AE.rnn_dec.zero_state(tf.shape(X)[0], tf.float32) #%% The entire AUTOENCODER # with tf.variable_scope("ae"): # tf.get_variable_scope().reuse_variables() encoder_output, state_enc = AE.encoder(residue, init_enc) decoder_output, state_dec = AE.decoder(encoder_output, init_dec) residue = X - decoder_output print('iteration', '1') for _ in range(num_steps - 1): with tf.variable_scope("ae", reuse=True): tf.get_variable_scope().reuse_variables() encoder_output, state_enc = AE.encoder(residue, init_enc) decoder_output, state_dec = AE.decoder(encoder_output, init_dec) residue = X - decoder_output print('iteration', _ + 2) #%% Cost and optimization setup y_true = X cost = tf.reduce_mean(tf.pow(y_true - decoder_output, 2)) optimizer = tf.train.AdamOptimizer(learning_rate, epsilon=1e-8).minimize(cost) #optimizer = tf.train.GradientDescent1024Optimizer(learning_rate).minimize(cost) #optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(cost) #%%############################################################################## # Training init = tf.global_variables_initializer() sess = tf.Session() #sess = tf.Session(config=tf.ConfigProto(log_device_placement=False)) sess.run(init) # Training cycle cost_vector = [] for epoch in range(training_epochs): for file_idx in range(10): file_name = 'clean_{}.mat'.format(file_idx) training_data, n_training = data_loader(file_name, input_dim, overlap=False) n_batch = int(n_training / batch_size) for i in range(n_batch): start_ind = i #np.random.randint(0, n_training-batch_size ); # For shuffling batch_xs = training_data[start_ind * batch_size:(start_ind + 1) * batch_size, :] # 50% overlap # batch_xs= tf.reshape(batch_xs, (batch_size,1,input_dim) ) _, c = sess.run([optimizer, cost], feed_dict={ X: batch_xs, mode: 0.0 }) # Display logs per epoch step if i % display_step == 0: print("epoch:", '%03d' % (epoch), "File:", '%02d' % (file_idx), "iteration:", '%04d' % (i + 1), "cost=", "{:.9f}".format(c)) cost_vector += [c] print("Optimization Finished!") #%%########################################################################## # Testing the network performance training_data, n_training = data_loader('clean_9.mat', input_dim, overlap=False) training_error = sess.run(cost, feed_dict={ X: training_data, mode: 1.0 })**0.5 test_data, n_test = data_loader('clean_10.mat', input_dim, overlap=False) y_pred_test, y_true_test, test_error = sess.run( [decoder_output, y_true, cost], feed_dict={ X: test_data, mode: 1.0 }) test_error = test_error**0.5 print('training_error', "{:.9f}".format(training_error)) print('test_error', "{:.9f}".format(test_error)) #print('learning_rate= ', learning_rate) #print('num_quantization_steps= ', num_steps) #%%########################################################################## # Savings network # AE_output={}; # AE_output['y_pred_test']=de_emph(y_pred_test, input_dim); # AE_output['y_true_test']=de_emph(y_true_test, input_dim); # # si.savemat("/home/hsadeghi/Dropbox/May/past_codes/AE_output.mat", # AE_output); sess.close() return training_error, test_error