#-*- coding: utf-8 -*- import tensorflow as tf x_data = [1, 2, 3] y_data = [1, 2, 3] """ try to find vlaues for W and b compute y_data = W * x_data + b (we know that W should be 1 and b 0, but Tensorflow will figure that out for us) """ W = tf.Variable(tf.random_uniform([1], -1.0, 1.0)) b = tf.Variable(tf.random_uniform([1], -1.0, 1.0)) # X, Y 를 placeholder 로 선언한다. 이제 run 실행 시 값을 지정할 수 있다 X = tf.placeholder(tf.float32) Y = tf.placeholder(tf.float32) # our hypothesis hypothesis = W * X + b # simplified cost function cost = tf.reduce_mean(tf.square(hypothesis - Y)) # minimize a = tf.Variable(0.1) optimizer = tf.train.GradientDescentOptimizer(a) train = optimizer.minimize(cost) # before starting, initilize the variables. we will 'run' this first
def weight_init(shape, name_for_weight): Xavier_init = np.sqrt(2.0) * np.sqrt(2.0 / np.array(shape).sum()) weights = tf.truncated_normal(shape, stddev = Xavier_init) return tf.Variable(weights, name = name_for_weight)
import tensorflow as tf # placeholder for input to the computation x = tf.placeholder(dtype=tf.float32, name="x") # bias variable for the affine weight transformation b = tf.Variable(tf.zeros(100)) # weight variable for the affine wegiht transformation with random values W = tf.Variable(tf.random_uniform([784, 100]), tf.float32) # activation as a function of the weight transformation a = tf.relu(tf.matmul(W, x) + b) # cost computed as a function of the activation # and the target optimization task C = [...] # Start session to run the computational graph session = tf.InteractiveSession() # Initialize all variables, in this example only the weight # matrix depends on an initialization tf.global_variables_initializer() for i in range(epochs): result = session.run(C, feed_dict={x: data[batch_indices]}) print(i, result)
def train(): avg_accuracy = 0 mnist = input_data.read_data_sets(FLAGS.data_dir, fake_data=FLAGS.fake_data) sess = tf.InteractiveSession() # Input placeholders with tf.name_scope('input'): x = tf.placeholder(tf.float32, [None, 784], name='x-input') x_quantized = fake_quantize_tensor(x, input_quantization, 0, 1, name="quantized_input") y_ = tf.placeholder(tf.int64, [None], name='y-input') with tf.name_scope('input_reshape'): image_shaped_input = tf.Variable( [-1, 28, 28, 1], collections=[tf.GraphKeys.GLOBAL_VARIABLES]) image_shaped_input = tf.reshape(x, [-1, 28, 28, 1]) tf.summary.image('input', image_shaped_input, 10) # Functions to create fc and conv layers depending on quantization settings def create_fc_layer(input_tensor, input_dim, output_dim, bits_w, max_w, bits_b, max_b, bits_a, max_a, noise_stddev, layer_name, act=tf.nn.relu): if (quantization_enabled == False): return fc_layer(input_tensor, input_dim, output_dim, layer_name, act) elif (noise_enabled_fc == False): return fc_layer_quantized(input_tensor, input_dim, output_dim, bits_w, max_w, bits_b, max_b, bits_a, max_a, layer_name, act) else: return fc_layer_quantized_add_noise(input_tensor, input_dim, output_dim, bits_w, max_w, bits_b, max_b, bits_a, max_a, noise_stddev, layer_name, act) def create_conv_layer(input_data, num_input_channels, num_filters, filter_shape, pool_shape, bits_w, max_w, bits_b, max_b, bits_a, max_a, noise_stddev, layer_name): if (quantization_enabled == False): return conv_layer(input_data, num_input_channels, num_filters, filter_shape, pool_shape, layer_name) elif (noise_enabled_conv == False): return conv_layer_quantized(input_data, num_input_channels, num_filters, filter_shape, pool_shape, bits_w, max_w, bits_b, max_b, bits_a, max_a, layer_name) else: return conv_layer_quantized_add_noise(input_data, num_input_channels, num_filters, filter_shape, pool_shape, bits_w, max_w, bits_b, max_b, bits_a, max_a, noise_stddev, layer_name) if (conv_enabled): if (quantization_enabled): image_shaped_input_quantized = fake_quantize_tensor( image_shaped_input, input_quantization, 0, 1, name="quantized_input") else: image_shaped_input_quantized = image_shaped_input layer1 = create_conv_layer(image_shaped_input_quantized, 1, 32, [5, 5], [2, 2], conv1_w_bits, conv1_w_max, conv1_b_bits, conv1_b_max, conv1_a_bits, conv1_a_max, noise_stddev, layer_name='conv1') layer2 = create_conv_layer(layer1, 32, 64, [5, 5], [2, 2], conv2_w_bits, conv2_w_max, conv2_b_bits, conv2_b_max, conv2_a_bits, conv2_a_max, noise_stddev, layer_name='conv2') with tf.name_scope('flatten'): x_flattened = tf.reshape(layer2, [-1, 7 * 7 * 64]) x_shape = 7 * 7 * 64 else: if (quantization_enabled): x_flattened = fake_quantize_tensor(x, input_quantization, 0, 1, name="quantized_input") else: x_flattened = x x_shape = 28 * 28 if (num_layers == 3): hidden1 = create_fc_layer(x_flattened, x_shape, fc1_depth, fc1_w_bits, fc1_w_max, fc1_b_bits, fc1_b_max, fc1_a_bits, fc1_a_max, noise_stddev, 'fully_connected1') with tf.name_scope('dropout'): keep_prob = tf.placeholder(tf.float32) tf.summary.scalar('dropout_keep_probability', keep_prob) dropped = tf.nn.dropout(hidden1, keep_prob) # Middle Layer (using settings 3) hidden2 = create_fc_layer(dropped, fc1_depth, fc3_depth, fc3_w_bits, fc3_w_max, fc3_b_bits, fc3_b_max, fc3_a_bits, fc3_a_max, noise_stddev, 'fully_connected_middle') with tf.name_scope('dropout2'): keep_prob2 = tf.placeholder(tf.float32) tf.summary.scalar('dropout2_keep_probability', keep_prob2) dropped2 = tf.nn.dropout(hidden2, keep_prob2) # Do not apply softmax activation yet, see below. y = create_fc_layer(dropped2, fc3_depth, fc2_depth, fc2_w_bits, fc2_w_max, fc2_b_bits, fc2_b_max, fc2_a_bits, fc2_a_max, noise_stddev, 'fully_connected2', act=tf.identity) elif (num_layers == 1): keep_prob = tf.placeholder(tf.float32) # Need to create placeholders keep_prob2 = tf.placeholder( tf.float32) # Even though they wont be used y = create_fc_layer(x_flattened, x_shape, fc2_depth, fc2_w_bits, fc2_w_max, fc2_b_bits, fc2_b_max, fc2_a_bits, fc2_a_max, noise_stddev, layer_name='fully_connected', act=tf.identity) else: # Otherwise create 2 layers hidden1 = create_fc_layer(x_flattened, x_shape, fc1_depth, fc1_w_bits, fc1_w_max, fc1_b_bits, fc1_b_max, fc1_a_bits, fc1_a_max, noise_stddev, 'fully_connected1') with tf.name_scope('dropout'): keep_prob = tf.placeholder(tf.float32) tf.summary.scalar('dropout_keep_probability', keep_prob) dropped = tf.nn.dropout(hidden1, keep_prob) keep_prob2 = tf.placeholder(tf.float32) # Need to create a placholder y = create_fc_layer(dropped, fc1_depth, fc2_depth, fc2_w_bits, fc2_w_max, fc2_b_bits, fc2_b_max, fc2_a_bits, fc2_a_max, noise_stddev, 'fully_connected2', act=tf.identity) with tf.name_scope('cross_entropy'): # The raw formulation of cross-entropy can be numerically unstable. # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)), # reduction_indices=[1])) # # So here we use tf.losses.sparse_softmax_cross_entropy on the # raw logit outputs of the nn_layer above, and then average across the batch. with tf.name_scope('total'): cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_, logits=y) tf.summary.scalar('cross_entropy', cross_entropy) with tf.name_scope('train'): train_step = tf.train.AdamOptimizer( FLAGS.learning_rate).minimize(cross_entropy) with tf.name_scope('accuracy'): with tf.name_scope('correct_prediction'): correct_prediction = tf.equal(tf.argmax(y, 1), y_) with tf.name_scope('accuracy'): accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', accuracy) # Merge all the summaries and write them out merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph) test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test') tf.global_variables_initializer().run() # Train the model, and also write summaries. # Every 10th step, measure test-set accuracy, and write test summaries # All other steps, run train_step on training data, & add training summaries def feed_dict(train, batch_size=100): # 128 """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" if train or FLAGS.fake_data: xs, ys = mnist.train.next_batch(batch_size, fake_data=FLAGS.fake_data) k = FLAGS.dropout else: xs, ys = mnist.test.images, mnist.test.labels k = 1.0 return {x: xs, y_: ys, keep_prob: k, keep_prob2: k} for i in range(FLAGS.max_steps + 5): if (i >= FLAGS.max_steps): sess.run([merged, train_step], feed_dict=feed_dict(True)) summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) test_writer.add_summary(summary, i) print('Accuracy test %s: %s' % (i - FLAGS.max_steps, acc)) avg_accuracy += acc elif (i % 100 == 99 and record_summaries == True and i <= 1000): # Record summaries and test-set accuracy summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) test_writer.add_summary(summary, i) print(datetime.datetime.now().strftime("%H:%M:%S"), 'Accuracy at step %s: %s' % (i, acc)) elif (i % 1000 == 999): # Record summaries and test-set accuracy summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) test_writer.add_summary(summary, i) print(datetime.datetime.now().strftime("%H:%M:%S"), 'Accuracy at step %s: %s' % (i, acc)) elif (i % 100 == 0): # print(datetime.datetime.now().strftime("%H:%M:%S"), "Adding run metadata for Step: ", i) run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() summary, _ = sess.run([merged, train_step], feed_dict=feed_dict(True), options=run_options, run_metadata=run_metadata) train_writer.add_run_metadata(run_metadata, 'step%03d' % i) train_writer.add_summary(summary, i) else: # Train and record summary summary, _ = sess.run([merged, train_step], feed_dict=feed_dict(True)) train_writer.add_summary(summary, i) print("Training Completed: ", datetime.datetime.now().strftime("%H:%M:%S")) print('Final accuracy: ', avg_accuracy / 5) x_in, x_flat, y_out = sess.run([x, x_flattened, y], feed_dict=feed_dict(True, batch_size=1)) np.savetxt("input.csv", x_in[0], delimiter=",", fmt='%f') np.savetxt("input_reshaped_quantized.csv", x_flat[0], delimiter=",", fmt='%f') np.savetxt("output.csv", y_out[0], delimiter=",", fmt='%f') for var in tf.global_variables(): if num_layers == 1: if '/weights/Variable:0' in var.name: print(var) v = sess.run(var) #get_weights(sess.run(var)) np.savetxt("Parameters/weights.csv", v, delimiter=",", fmt='%f') np.savetxt("Parameters/Q.csv", calculate_Q(0.066667, v).astype(int), delimiter=",", fmt='%i') elif '/biases/Variable:0' in var.name: print(var) v = sess.run(var) # get_biases(sess.run(var)) np.savetxt("Parameters/floating_biases.csv", v, delimiter=",", fmt='%f') # Store the floating point biases np.savetxt("Parameters/biases.csv", get_biases(v), delimiter=",", fmt='%f') # Store the fixed point biases else: print( "Fixed point inference not implemented yet for networks with more than one layer" ) if 'fully_connected1/weights/Variable:0' in var.name: print(var) elif 'fully_connected1/biases/Variable:0' in var.name: print(var) elif 'fully_connected2/weights/Variable:0' in var.name: print(var) elif 'fully_connected2/biases/Variable:0' in var.name: print(var) elif 'fully_connected3/weights/Variable:0' in var.name: print(var) elif 'fully_connected3/biases/Variable:0' in var.name: print(var) train_writer.close()
def get_z_var(hparams, batch_size): z = tf.Variable(tf.random_normal((batch_size, hparams.n_z)), name='z') return z
def build(self, input_shape): # We just change the way bias is added and remove it from trainable variable! input_shape = tf.TensorShape(input_shape) if input_shape[-1] is None: raise ValueError('The last dimension of the inputs to `Dense` ' 'should be defined. Found `None`.') if not input_shape.is_fully_defined(): print("the input shape used for build is not fully defined") self.kernel = self.add_weight( 'kernel', shape=[input_shape[-1], self.units], initializer=self.sparseInitializer, regularizer=self.kernel_regularizer, constraint=weightFixedAndClippedConstraint(self.sparseInitializer), dtype=self.dtype, trainable=True) self.bias = None variableShape=(input_shape[-1],self.units) self.mask = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) self.k1 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) self.k1n = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) self.k2 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) self.k3 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) self.k3n = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) self.k4 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) #only one template starts each cascade: self.TA0 = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.TI0 = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) #only one inhibition by outputs (units): self.k5 = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k5n = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k6 = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) #degradation cst for the pseudo-template self.kdpT = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) #degradation cst for the output self.kd = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) #degradation cst for the activation cascaded template self.kdT = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) #degradation cst for the inhibition cascaded template self.kdT2 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) self.E0 = tf.Variable(tf.constant(1,dtype=tf.float32),trainable=False,dtype=tf.float32) self.rescaleFactor = tf.Variable(1,dtype=tf.float32,trainable=False) self.Xglobal = tf.Variable(tf.constant(1,dtype=tf.float32),trainable=False,dtype=tf.float32) #other intermediates variable: self.k1M = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) self.k3M = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False) self.k5M = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k1g = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k1ng = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k2g = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k3g = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k3ng = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k4g = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k1Mg = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.k3Mg = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False) self.built = True print("Layer successfully built")
LEARNING_RATE_DECAY = 0.9 TRAINING_STEPS = 30000 KEEP_PROB = 0.5 #learning_rate = 0.01 batch_size = 128 trainfile_set = [] num_classes = 95 #the classes of our data set butterfly refine_layers = [ 'fc8' ] #we only refine the last layer, other layer is load from a trained one train_layers = [ 'pool1', 'norm1', 'conv2', 'pool2', 'norm2', 'conv3', 'conv4', 'conv5', 'pool5', 'fc6', 'fc7', 'fc8' ] global_step = tf.Variable(0, trainable=False) #Input and output x = tf.placeholder(tf.float32, [None, 227, 227, 3]) y = tf.placeholder(tf.float32, [None, num_classes]) keep_prob = tf.placeholder(tf.float32) #Initialize model model = AlexNet(x, keep_prob, num_classes, refine_layers) # output of the AlexNet score = model.fc8 var_list = [ v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers ] #loss function
sign_names = pd.read_csv('signnames.csv') nb_classes = 43 x = tf.placeholder(tf.float32, (None, 32, 32, 3)) resized = tf.image.resize_images(x, (227, 227)) # NOTE: By setting `feature_extract` to `True` we return # the second to last layer. fc7 = AlexNet(resized, feature_extract=True) # TODO: Define a new fully connected layer followed by a softmax activation to classify # the traffic signs. Assign the result of the softmax activation to `probs` below. # HINT: Look at the final layer definition in alexnet.py to get an idea of what this # should look like. shape = (fc7.get_shape().as_list()[-1], nb_classes) # use this shape for the weight matrix fc8W = tf.Variable(tf.truncated_normal(shape, stddev=1e-2)) fc8b = tf.Variable(tf.zeros(nb_classes)) probs = tf.nn.softmax(tf.nn.xw_plus_b(fc7, fc8W, fc8b)) init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) # Read Images im1 = imread("construction.jpg").astype(np.float32) im1 = im1 - np.mean(im1) im2 = imread("stop.jpg").astype(np.float32) im2 = im2 - np.mean(im2) # Run Inference
data = np.genfromtxt('iris.data', delimiter=",") # iris.data file loading np.random.shuffle(data) # we shuffle the data x_data = data[:, 0:4].astype('f4') # the samples are the four first rows of data y_data = one_hot(data[:, 4].astype(int), 3) # the labels are in the last row. Then we encode them in one hot code print("\nSome samples...") for i in range(20): print(x_data[i], " -> ", y_data[i]) print x = tf.placeholder("float", [None, 4]) # samples y_ = tf.placeholder("float", [None, 3]) # labels W1 = tf.Variable(np.float32(np.random.rand(4, 5)) * 0.1) b1 = tf.Variable(np.float32(np.random.rand(5)) * 0.1) W2 = tf.Variable(np.float32(np.random.rand(5, 3)) * 0.1) b2 = tf.Variable(np.float32(np.random.rand(3)) * 0.1) h = tf.nn.sigmoid(tf.matmul(x, W1) + b1) # h = tf.matmul(x, W1) + b1 # Try this! y = tf.nn.softmax(tf.matmul(h, W2) + b2) loss = tf.reduce_sum(tf.square(y_ - y)) train = tf.train.GradientDescentOptimizer(0.01).minimize(loss) # learning rate: 0.01 init = tf.initialize_all_variables()
import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' import tensorflow as tf import numpy state = tf.Variable(0.0) add_op = tf.assign(state, state+tf.constant(1.0)) #assign的结果也是float32 with tf.Session() as sess: sess.run(tf.global_variables_initializer()) #记得initializaer后面加括号 print ('init state:', sess.run(state), 'type:', type(sess.run(state))) for _ in range(3): sess.run(add_op) print(sess.run(state)) print('sess_op:', sess.run(add_op), 'sess_op_type:', type(sess.run(add_op))) input1 = tf.placeholder(tf.float32) input2 = tf.placeholder(tf.float32) output = tf.add(tf.multiply(input1, input2), 1) with tf.Session() as sess: print(sess.run([output], feed_dict={input1:[7.0], input2:[2.0]}))
import tensorflow as tf import numpy as np # 使用 NumPy 生成假数据(phony data), 总共 100 个点. x_data = np.float32(np.random.rand(2, 100)) # 随机输入 y_data = np.dot([0.100, 0.200], x_data) + 0.300 # 构造一个线性模型 # b = tf.Variable(tf.zeros([1])) W = tf.Variable(tf.random_uniform([1, 2], -1.0, 1.0)) y = tf.matmul(W, x_data) + b # 最小化方差 loss = tf.reduce_mean(tf.square(y - y_data)) optimizer = tf.train.GradientDescentOptimizer(0.5) train = optimizer.minimize(loss) # 初始化变量 #init = tf.initialize_all_variables() init = tf.global_variables_initializer() # 启动图 (graph) sess = tf.Session() sess.run(init) # 拟合平面 for step in range(0, 201): sess.run(train) if step % 20 == 0: print(step,sess.run(W),sess.run(b))
#!/usr/bin/env python """ Source: https://www.oreilly.com/learning/hello-tensorflow """ import tensorflow as tf # 1. TensorFlow Graph - first creating the computation graph # Starting off with the default graph graph = tf.get_default_graph() # Setting the Ops x = tf.constant(1.0, name='input') w = tf.Variable(0.8, name='weight') y = tf.mul(w, x, name='output') # Assume that the correct value == 0.0 y_ = tf.constant(0.0) # Loss is the "wrongness" of the model # The model should lessen the square of the diff b/w current output and desired output. loss = (y - y_)**2 # Adding an optimizer to add the "Learning" component optim = tf.train.GradientDescentOptimizer(learning_rate=0.025) # 2. Sessions - computing in Sessions # Initialize all variables and start the session
''' 练习使用滑动平均模型 ''' import tensorflow as tf v1 = tf.Variable(0, dtype=tf.float32) step = tf.Variable(0, trainable=False) ema = tf.train.ExponentialMovingAverage(0.99, step) maintain_average_op = ema.apply([v1]) with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) print(sess.run([v1, ema.average(v1)])) sess.run(tf.assign(v1, 5)) sess.run(maintain_average_op) print(sess.run([v1, ema.average(v1)])) sess.run(tf.assign(step, 1000)) sess.run(tf.assign(v1, 10)) sess.run(maintain_average_op) print(sess.run([v1, ema.average(v1)])) sess.run(maintain_average_op) print(sess.run([v1, ema.average(v1)]))
training_steps = 10000 batch_size = 128 display_step = 200 # Network Parameters num_input = 28 # MNIST data input (img shape: 28*28) timesteps = 28 # timesteps num_hidden = 128 # hidden layer num of features num_classes = 10 # MNIST total classes (0-9 digits) # tf Graph input X = tf.placeholder("float", [None, timesteps, num_input]) Y = tf.placeholder("float", [None, num_classes]) # Define weights weights = {'out': tf.Variable(tf.random_normal([num_hidden, num_classes]))} biases = {'out': tf.Variable(tf.random_normal([num_classes]))} def RNN(x, weights, biases): # Prepare data shape to match `rnn` function requirements # Current data input shape: (batch_size, timesteps, n_input) # Required shape: 'timesteps' tensors list of shape (batch_size, n_input) # Unstack to get a list of 'timesteps' tensors of shape (batch_size, n_input) x = tf.unstack(x, timesteps, 1) # Define a lstm cell with tensorflow lstm_cell = rnn.BasicLSTMCell(num_hidden, forget_bias=1.0)
def build(self, n_input_features, units_per_hidden_layer, encoder_activation_function='sigmoid', decoder_activation_function='identity'): with self.graph.as_default(): self.n_input_features = n_input_features self.input = self.input = tf.placeholder(tf.float32, shape=(None, n_input_features), name='input') self.units_per_hidden_layer = units_per_hidden_layer self.encoder_activation_function = encoder_activation_function self.decoder_activation_function = decoder_activation_function self.keep_probability = tf.placeholder(tf.float32, name='keep_probability') self.learning_rate = tf.placeholder(tf.float32, name='learning_rate') with tf.name_scope('stack'): n_inputs = self.n_input_features current_input = self.input for i, units in enumerate(self.units_per_hidden_layer): mask = tf.random_uniform( shape=tf.shape(current_input), minval=0, maxval=1, dtype=tf.float32, seed=None, name='weight_initializer_stack_{}'.format(i + 1)) mask = tf.where(mask <= self.keep_probability, tf.ones_like(current_input, dtype=tf.float32), tf.zeros_like(current_input, dtype=tf.float32), name='random_mask_stack_{}'.format(i + 1)) self.corrupted_inputs.append( tf.multiply(current_input, mask, name='corruped_input_stack_{}'.format(i + 1))) with tf.name_scope('encoder_stack_{}'.format(i + 1)): previous_input = self.corrupted_inputs[-1] activation_function = self.get_activation_function( self.encoder_activation_function) weights = tf.Variable( tf.truncated_normal([n_inputs, units]), dtype=tf.float32, name='weights_stack_{}'.format(i + 1)) self.initial_weights.append(weights) bias = tf.Variable(tf.zeros([units], dtype=tf.float32), name='bias_stack_{}'.format(i + 1)) self.initial_biases.append(bias) self.encoders.append( activation_function( tf.add(tf.matmul(previous_input, weights), bias), name='encoder_stack_{}'.format(i + 1))) current_input = self.encoders[-1] with tf.name_scope('decoder_stack_{}'.format(i + 1)): weights = tf.Variable( tf.truncated_normal([units, n_inputs]), dtype=tf.float32, name='weights_stack_{}'.format(i + 1)) bias = tf.Variable(tf.zeros([n_inputs], dtype=tf.float32), name='bias_stack_{}'.format(i + 1)) activation_function = self.get_activation_function( self.decoder_activation_function) self.decoders.append( activation_function( tf.add(tf.matmul(self.encoders[-1], weights), bias), name='decoder_stack_{}'.format(i + 1))) n_inputs = units self.saver = tf.train.Saver()
def weight_variable(shape): initial = tf.truncated_normal(shape, stddev=0.1) return tf.Variable(initial)
def _build_model(self, input_width, input_height, input_channel, output_size, learning_rate, decay): # inputs prev_images = tf.placeholder(dtype=tf.float32, shape=[None, input_height, input_width, input_channel]) tf.summary.image(name='prev_images', tensor=prev_images) next_images = tf.placeholder(dtype=tf.float32, shape=[None, input_height, input_width, input_channel]) tf.summary.image(name='next_images', tensor=next_images) images = tf.concat([prev_images, next_images], axis=3) images = tf.image.resize_area(images, [input_height / 4, input_width / 4]) # labels labels = tf.placeholder(dtype=tf.float32, shape=[None, output_size]) tf.summary.histogram(name='labels', values=labels) keep_prob = tf.placeholder(dtype=tf.float32, shape=()) # learning rate self.learning_rate = tf.Variable(learning_rate, name='learning_rate', trainable=False) self.decay_lr = tf.assign(self.learning_rate, self.learning_rate * decay) tf.summary.scalar(name='learning_rate', tensor=self.learning_rate) # model with tf.name_scope('conv1'): h1_size = 32 w = tf.get_variable(name='conv_w1', shape=[3, 3, input_channel * 2, h1_size], dtype=tf.float32, initializer=tf.random_normal_initializer(stddev=0.01)) b = tf.get_variable(name='conv_b1', shape=[h1_size], dtype=tf.float32, initializer=tf.constant_initializer(value=1e-3)) h = tf.nn.relu(tf.nn.conv2d(images, w, strides=[1, 1, 1, 1], padding='SAME') + b) h = tf.nn.max_pool(h, strides=[1, 2, 2, 1], ksize=[1, 2, 2, 1], padding='SAME') h = tf.nn.dropout(h, keep_prob) with tf.name_scope('conv2'): h2_size = 32 w = tf.get_variable(name='conv_w2', shape=[3, 3, h1_size, h2_size], dtype=tf.float32, initializer=tf.random_normal_initializer(stddev=0.01)) b = tf.get_variable(name='conv_b2', shape=[h2_size], dtype=tf.float32, initializer=tf.constant_initializer(value=1e-3)) h = tf.nn.relu(tf.nn.conv2d(h, w, strides=[1, 1, 1, 1], padding='SAME') + b) h = tf.nn.max_pool(h, strides=[1, 2, 2, 1], ksize=[1, 2, 2, 1], padding='SAME') h = tf.nn.dropout(h, keep_prob) with tf.name_scope('conv3'): h3_size = 32 w = tf.get_variable(name='conv_w3', shape=[3, 3, h2_size, h3_size], dtype=tf.float32, initializer=tf.random_normal_initializer(stddev=0.01)) b = tf.get_variable(name='conv_b3', shape=[h3_size], dtype=tf.float32, initializer=tf.constant_initializer(value=1e-3)) h = tf.nn.relu(tf.nn.conv2d(h, w, strides=[1, 1, 1, 1], padding='SAME') + b) h = tf.nn.max_pool(h, strides=[1, 2, 2, 1], ksize=[1, 2, 2, 1], padding='SAME') h = tf.nn.dropout(h, keep_prob) with tf.name_scope('fc4'): h_size = h.get_shape().as_list() self.logger.info('connect size: %s' % (str(h_size))) connect_size = h_size[1] * h_size[2] * h_size[3] h4_size = 1024 w = tf.get_variable(name='w4', shape=[connect_size, h4_size], dtype=tf.float32, initializer=tf.random_normal_initializer( stddev=np.sqrt(2.0 / connect_size))) b = tf.get_variable(name='b4', shape=[h4_size], dtype=tf.float32, initializer=tf.constant_initializer(value=1e-3)) h = tf.nn.relu(tf.matmul(tf.reshape(h, [-1, connect_size]), w) + b) with tf.name_scope('fc5'): h5_size = 256 w = tf.get_variable(name='w5', shape=[h4_size, h5_size], dtype=tf.float32, initializer=tf.random_normal_initializer( stddev=np.sqrt(2.0 / h4_size))) b = tf.get_variable(name='b5', shape=[h5_size], dtype=tf.float32, initializer=tf.constant_initializer(value=1e-3)) h = tf.nn.relu(tf.matmul(h, w) + b) with tf.name_scope('output'): w = tf.get_variable(name='ow', shape=[h5_size, output_size], dtype=tf.float32, initializer=tf.random_normal_initializer( stddev=np.sqrt(2.0 / h5_size))) b = tf.get_variable(name='ob', shape=[output_size], dtype=tf.float32, initializer=tf.constant_initializer(value=1e-3)) logits = tf.matmul(h, w) + b outputs = logits tf.summary.histogram(name='outputs', values=outputs) # loss and optimizer with tf.name_scope('loss'): # scale up to cm as unit loss = tf.reduce_mean(tf.square(logits - labels)) * output_size * (10 **2) tf.summary.scalar(name='loss', tensor=loss) error = tf.reduce_mean(tf.abs(logits - labels)) tf.summary.scalar(name='error', tensor=error) with tf.name_scope('optimizer'): optimizer = tf.train.GradientDescentOptimizer(self.learning_rate) train_op = optimizer.minimize(loss) return prev_images, next_images, labels, keep_prob, outputs, \ loss, error, train_op
def train_NN(): x = tf.placeholder(tf.float32, [None, 256]) # we input 2 vector as our input 128*2 y_ = tf.placeholder(tf.float32, [None, 5]) # we have four kinds of relationship # #w1 = tf.Variable(tf.zeros([256, hiddenunit])) # how many items in each x,unit of hidden layer] w1 = tf.Variable(tf.truncated_normal([256, hiddenunit], stddev=0.1)) b1 = tf.Variable(tf.zeros([hiddenunit ])) # how many hidden unit in each layer w2 = tf.Variable(tf.zeros( [hiddenunit, 5])) # [how many items in each x,unit of hidden layer],初始值是0 b2 = tf.Variable(tf.zeros([5])) keep_prob = tf.placeholder(tf.float32) #layer 1 #relu nnhlyer1 = tf.nn.softmax(tf.matmul(x, w1) + b1) h_fc1_drop = tf.nn.dropout(nnhlyer1, keep_prob) #avoid overfit #layer 2 y_nn = tf.nn.softmax(tf.matmul(h_fc1_drop, w2) + b2) # loss cross_entropy = tf.reduce_mean( -tf.reduce_sum(y_ * tf.log(y_nn), reduction_indices=[1])) # loss #cross_entropy = -tf.reduce_sum(y_*tf.log(y_nn)) correctPred = tf.equal(tf.argmax(y_nn, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correctPred, tf.float32)) # change loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits( logits=(tf.matmul(h_fc1_drop, w2) + b2), labels=y_)) # update train_step = tf.train.GradientDescentOptimizer(ita).minimize(cross_entropy) # train_step = tf.train.AdamOptimizer(1e-4).minimize(loss) #init init = tf.global_variables_initializer() saver = tf.train.Saver() sess = tf.InteractiveSession() saver = tf.train.Saver() sess.run(init) tf.summary.scalar('Loss', cross_entropy) tf.summary.scalar('Accuracy', accuracy) merged = tf.summary.merge_all() logdir = "tensorboard/" + datetime.datetime.now().strftime( "%Y%m%d-%H%M%S") + "/" writer = tf.summary.FileWriter(logdir, sess.graph) # Session sess = tf.Session() sess.run(init) # for i in range(iterations): #random batch_xs, batch_ys = getimageFeaturebynxtbatch(tr_vectors, tr_labels, batchsize) sess.run([cross_entropy, train_step], feed_dict={ x: batch_xs, y_: batch_ys, keep_prob: keep_pro }) # print('>>>batch train, which iteration = %d' % (i)) if i % 20 == 0: # correct_prediction = tf.equal(tf.argmax(y_nn, 1), tf.argmax(y_, 1)) # accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) # print ("Setp: ", i, "Accuracy: ",sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys})) save_path = saver.save(sess, "models/pretrained_lstm.ckpt", global_step=i) print("saved to %s" % save_path) summary = sess.run(merged, { x: batch_xs, y_: batch_ys, keep_prob: 1.0 }) writer.add_summary(summary, i) writer.close()
def predict(sample): dataset = pd.read_csv("data.csv") def normalise(value,values): m = max(values) return value/m index = 2 for i in range(len(sample[0])): sample[0][i]=normalise(sample[0][i],dataset.iloc[:,index]) index+=1 input = tf.placeholder(tf.float32,[None,12]) W1 = tf.Variable(tf.random.normal([12,20])) b1 = tf.Variable(tf.random.normal([20])) layer1 = tf.add(tf.matmul(input,W1),b1) layer1 = tf.nn.leaky_relu(layer1) W2 = tf.Variable(tf.random.normal([20,20])) b2 = tf.Variable(tf.random.normal([20])) layer2 = tf.add(tf.matmul(layer1,W2),b2) layer2 = tf.nn.leaky_relu(layer2) W3 = tf.Variable(tf.random.normal([20,20])) b3 = tf.Variable(tf.random.normal([20])) layer3 = tf.add(tf.matmul(layer2,W3),b3) layer3 = tf.nn.leaky_relu(layer3) W4 = tf.Variable(tf.random.normal([20,20])) b4 = tf.Variable(tf.random.normal([20])) layer4 = tf.add(tf.matmul(layer3,W4),b4) layer4 = tf.nn.leaky_relu(layer4) W5 = tf.Variable(tf.random.normal([20,20])) b5 = tf.Variable(tf.random.normal([20])) layer5 = tf.add(tf.matmul(layer4,W5),b5) layer5 = tf.nn.leaky_relu(layer5) W6 = tf.Variable(tf.random.normal([20,20])) b6 = tf.Variable(tf.random.normal([20])) layer6 = tf.add(tf.matmul(layer5,W6),b6) layer6 = tf.nn.leaky_relu(layer6) W7 = tf.Variable(tf.random.normal([20,20])) b7 = tf.Variable(tf.random.normal([20])) layer7 = tf.add(tf.matmul(layer6,W7),b7) layer7 = tf.nn.leaky_relu(layer7) W8 = tf.Variable(tf.random.normal([20,20])) b8 = tf.Variable(tf.random.normal([20])) layer8 = tf.add(tf.matmul(layer7,W8),b8) layer8 = tf.nn.leaky_relu(layer8) W9 = tf.Variable(tf.random.normal([20,20])) b9 = tf.Variable(tf.random.normal([20])) layer9 = tf.add(tf.matmul(layer8,W9),b9) layer9 = tf.nn.leaky_relu(layer9) W10= tf.Variable(tf.random.normal([20,1])) b10 = tf.Variable(tf.random.normal([1])) output = tf.add(tf.matmul(layer9,W10),b10) with tf.Session() as sess: saver = tf.train.Saver() saver.restore(sess,"housing_price_model/model") return sess.run(output,feed_dict={input:sample})
def bias_variable(shape): initial = tf.constant(0.1, shape=shape) return tf.Variable(initial)
import tensorflow as tf import os # 调整警告等级 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' a = tf.constant([1, 2, 3, 4, 5]) # 变量必须进行显示初始化 var = tf.Variable(tf.random_normal([2, 3], mean=0, stddev=1)) init_op = tf.global_variables_initializer() print(a, '\n', var) with tf.Session() as sess: sess.run(init_op) print(sess.run([a, var]))
import tensorflow as tf import numpy as np weights = tf.Variable(tf.random_normal([784,20],stddev=0.35),name="weights") heights = tf.Variable(tf.random_normal([20,20]),name="heights") init_op = tf.global_variables_initializer() saver = tf.train.Saver() np.set_printoptions(threshold='nan') with tf.Session() as sess: sess.run(init_op) # saver.restore(sess,"/tmp/tensorflow_data/model.ckpt") print(weights.eval()) save_path = saver.save(sess,"/tmp/tensorflow_data/model.ckpt")
###################### # CGES Configuration # ###################### tf.app.flags.DEFINE_float('mu', 0.8, 'initialized group sparsity ratio') tf.app.flags.DEFINE_float('chvar', 0.2, '\'mu\' change per layer') FLAGS = tf.app.flags.FLAGS gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.memory_usage) sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options)) mnist = input_data.read_data_sets('MNIST_data', one_hot=True) x = tf.placeholder(tf.float32, shape=[None, 784]) # single flattened 28 * 28 pixel MNIST image y_ = tf.placeholder(tf.float32, shape=[None, 10]) # 10 classes output keep_prob = tf.placeholder(tf.float32) W = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) batch = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay( FLAGS.base_lr, # Base learning rate. batch, # Current index. FLAGS.stepsize, # Decay iteration step. FLAGS.decay_rate, # Decay rate. staircase=True) from mnist_model import mnist_conv y_conv = mnist_conv(x, 10, keep_prob) S_vars = [svar for svar in tf.trainable_variables() if 'weight' in svar.name] ff_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y_))
def map_estimate_BFGS(self, max_iter=1000, initialize=True, batch_size=None): """ Runs BFGS on the data :param max_iter: int | number of max iterations for BFGS :param initialize: bool | set to True if alpha values should be initialized :param batch_size: int | needed for batch update :return: alpha """ sess = self.sess max_iter = max_iter if initialize: self.batch = 0 if batch_size is None: Inf = genInfectedTensor(self.data, self.numNodes, self.T) U = genUninfectedTensor(self.data, self.numNodes, self.T) else: Inf = genInfectedTensor( self.data[self.batch:self.batch + batch_size], self.numNodes, self.T) U = genUninfectedTensor( self.data[self.batch:self.batch + batch_size], self.numNodes, self.T) self.batch += batch_size U_ph = tf.placeholder(tf.float32, U.shape) I_ph = tf.placeholder(tf.float32, Inf.shape) if initialize: B = tf.Variable(tf.random_uniform(U.shape[1:]), dtype=tf.float32) else: B = tf.Variable(self.a, dtype=tf.float32) alpha_tensor = tf.nn.relu(B) psi_1 = tf.map_fn(lambda x: f_psi_1(tf.transpose(alpha_tensor), x), I_ph, dtype=tf.float32) psi_2 = tf.map_fn(lambda x: f_psi_2(tf.transpose(alpha_tensor), x), U_ph, dtype=tf.float32) psi_3 = tf.map_fn(lambda x: f_psi_3(tf.transpose(alpha_tensor), x), I_ph, dtype=tf.float32) prior = gamma_prior(alpha_tensor) log_p = -(tf.reduce_sum(psi_1) + tf.reduce_sum(psi_2) + tf.reduce_sum(psi_3) + tf.reduce_sum(prior)) feed_dict = {U_ph: U.eval(session=sess), I_ph: Inf.eval(session=sess)} optimizer = tf.contrib.opt.\ ScipyOptimizerInterface(log_p, method='L-BFGS-B', options={'maxiter': max_iter}) if initialize: model = tf.global_variables_initializer() sess.run(model) optimizer.minimize(sess, feed_dict=feed_dict) self.a = alpha_tensor.eval(session=sess) return self.a
L1 = tf.nn.relu(BN1) L1 = tf.nn.dropout(L1, keep_prob) tf.summary.histogram('W1', W1) with tf.name_scope('layer2'): W2 = weight_init(layer2_shape, 'W2') z2 = tf.matmul(L1, W2) BN2 = tf.contrib.layers.batch_norm(z2, center = True, scale = True, is_training = is_training_holder) L2 = tf.nn.relu(BN2) L2 = tf.nn.dropout(L2, keep_prob) tf.summary.histogram('W2', W2) with tf.name_scope('output'): W3 = weight_init(output_shape, 'W3') b3 = tf.Variable(tf.random_normal([output_shape[1]])) model = tf.matmul(L2, W3) + b3 tf.summary.histogram('W3', W3) with tf.name_scope('optimizer'): base_cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=model, labels=Y)) lossL2 = tf.reduce_mean(tf.nn.l2_loss(W1) + tf.nn.l2_loss(W2) + tf.nn.l2_loss(W3)) * L2beta cost = base_cost + lossL2 update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost) tf.summary.scalar('cost', cost) with tf.name_scope("accuracy"):
def map_estimate_BFGS_topics(self, max_iter=1000, numTopics=2, initialize=True, batch_size=None): """ Runs BFGS on the data and including the topic information :param max_iter: int | number of max iterations for BFGS :param initialize: bool | set to True if alpha values should be initialized :param batch_size: int | needed for batch update :return: alpha """ topics = self.topics sess = self.sess max_iter = max_iter if initialize: self.batch = 0 theta_topics = tf.reshape( tf.divide(tf.ones((1, numTopics)), numTopics), (numTopics, 1)) if batch_size is None: Inf = genInfectedTensor(self.data, self.numNodes, self.T) U = genUninfectedTensor(self.data, self.numNodes, self.T) else: Inf = genInfectedTensor( self.data[self.batch:self.batch + batch_size], self.numNodes, self.T) U = genUninfectedTensor( self.data[self.batch:self.batch + batch_size], self.numNodes, self.T) self.batch += batch_size U_ph = tf.placeholder(tf.float32, U.shape) I_ph = tf.placeholder(tf.float32, Inf.shape) rate_intercept = tf.Variable(tf.random_uniform( (self.numNodes, self.numNodes)), dtype=tf.float32) rate_affinity = tf.Variable(tf.random_uniform( (self.numNodes, self.numNodes, numTopics)), dtype=tf.float32) psi_1 = tf.map_fn(lambda x: f_psi_1_t(rate_intercept, rate_affinity, x[ 0], x[1], self.numNodes, numTopics), (I_ph, topics), dtype=tf.float32) psi_2 = tf.map_fn(lambda x: f_psi_2_t(rate_intercept, rate_affinity, x[ 0], x[1], self.numNodes, numTopics), (U_ph, topics), dtype=tf.float32) psi_3 = tf.map_fn(lambda x: f_psi_3_t(rate_intercept, rate_affinity, x[ 0], x[1], self.numNodes, numTopics), (I_ph, topics), dtype=tf.float32) prior = gamma_prior_t(rate_intercept, rate_affinity, theta_topics, self.numNodes, numTopics) log_p = -(tf.reduce_sum(psi_1) + tf.reduce_sum(psi_2) + tf.reduce_sum(psi_3) + tf.reduce_sum(prior)) feed_dict = {U_ph: U.eval(session=sess), I_ph: Inf.eval(session=sess)} optimizer = tf.contrib.opt.\ ScipyOptimizerInterface(log_p, method='L-BFGS-B', options={'maxiter': max_iter}) if initialize: model = tf.global_variables_initializer() sess.run(model) optimizer.minimize(sess, feed_dict=feed_dict) self.a_t1 = evaluateAlpha( rate_intercept, rate_affinity, np.array([1, 0]), self.numNodes, numTopics).eval(session=sess).transpose().round(1) self.a_t2 = evaluateAlpha( rate_intercept, rate_affinity, np.array([0, 1]), self.numNodes, numTopics).eval(session=sess).transpose().round(1) return self.a_t1, self.a_t2
def run_main(id_data = 0, setting = "dlvm2", link = "nonlinear3", n = 1000, p = 10, d=3, d_miwae = 3, h_miwae=128, add_mask=True, num_samples_zmul=200, num_samples_xmul=50, perc_miss = 0.1, in_folder = "../Data/", out_folder = "../Data/"): ########## id = id_data print(in_folder, out_folder) print('run_main', id_data, n, p) # try: os.makedirs(in_folder, exist_ok=True) except FileExistsError: pass try: os.makedirs(out_folder, exist_ok=True) except FileExistsError: pass in_file_text = setting + "_" + link + "_" + str(n) + "_" + str(p) + "_" + str(d) + "_seed" + str(id) + "_propNA"+"{:3.2f}".format(perc_miss) if add_mask: out_file_text = setting + "_" + link + "_" + str(n) + "_" + str(p) + "_" + str(d) + "_WY_dmiwae" + str(d_miwae) + "_hmiwae" + str(h_miwae) + "_mask_seed" + str(id) + "_propNA"+"{:3.2f}".format(perc_miss) else: out_file_text = setting + "_" + link + "_" + str(n) + "_" + str(p) + "_" + str(d) + "_WY_dmiwae" + str(d_miwae) + "_hmiwae" + str(h_miwae) + "_seed" + str(id) + "_propNA"+"{:3.2f}".format(perc_miss) print(out_folder+out_file_text) data = np.array(pd.read_csv(in_folder+in_file_text+"_xcomp.csv", low_memory=False))[:,1:(p+1)] data_miss = np.array(pd.read_csv(in_folder+in_file_text+"_xmisswy.csv", low_memory=False))[:,1:(p+3)] print('dim(data)', data.shape) print('dim([data_miss , w])', data_miss.shape) # ######### xfull = (data - np.mean(data,0))/np.std(data,0) n = xfull.shape[0] # number of observations p = xfull.shape[1] # number of features print(n) print(p) # ########## np.random.seed(1234) tf.set_random_seed(1234) xmiss = np.copy(data_miss) mask = np.isfinite(xmiss) # binary mask that indicates which values are missing print('mask', mask.shape) # ########## xhat_0 = np.copy(xmiss) xhat_0[np.isnan(xmiss)] = 0 p_mod = p pw = 2 if add_mask: mask_mod = np.copy(mask) #mask_mod = mask_mod.astype(float) #mask_mod = (mask_mod - np.mean(mask_mod,0))/np.std(mask_mod,0) xhat_0 = np.concatenate((xhat_0, mask_mod), axis=1) xfull = np.concatenate((xfull, mask_mod), axis=1) mask = np.concatenate((mask, np.ones_like(mask).astype(bool)), axis = 1) p = p*2 print('[data, w, y, mask, 1, 1]', xhat_0.shape) pw = 4 # ########## x = tf.placeholder(tf.float32, shape=[None, p+pw]) # Placeholder for xhat_0 learning_rate = tf.placeholder(tf.float32, shape=[]) batch_size = tf.shape(x)[0] xmask = tf.placeholder(tf.bool, shape=[None, p+pw]) K= tf.placeholder(tf.int32, shape=[]) # Placeholder for the number of importance weights # ########## p_z = tfd.MultivariateNormalDiag(loc=tf.zeros(d_miwae, tf.float32)) # ########## sigma = "relu" decoder = tfk.Sequential([ tfkl.InputLayer(input_shape=[d_miwae,]), tfkl.Dense(h_miwae, activation=sigma,kernel_initializer="orthogonal"), tfkl.Dense(h_miwae, activation=sigma,kernel_initializer="orthogonal"), tfkl.Dense(3*(p+pw),kernel_initializer="orthogonal") # the decoder will output both the mean, the scale, and the number of degrees of freedoms (hence the 3*p) ]) # ########## tiledmask = tf.tile(xmask,[K,1]) tiledmask_float = tf.cast(tiledmask,tf.float32) mask_not_float = tf.abs(-tf.cast(xmask,tf.float32)) iota = tf.Variable(np.zeros([1,p+pw]),dtype=tf.float32) tilediota = tf.tile(iota,[batch_size,1]) iotax = x + tf.multiply(tilediota,mask_not_float) # ########## encoder = tfk.Sequential([ tfkl.InputLayer(input_shape=[p+pw,]), tfkl.Dense(h_miwae, activation=sigma,kernel_initializer="orthogonal"), tfkl.Dense(h_miwae, activation=sigma,kernel_initializer="orthogonal"), tfkl.Dense(3*d_miwae,kernel_initializer="orthogonal") ]) # ########## out_encoder = encoder(iotax) q_zgivenxobs = tfd.Independent(distribution=tfd.StudentT(loc=out_encoder[..., :d_miwae], scale=tf.nn.softplus(out_encoder[..., d_miwae:(2*d_miwae)]), df=3 + tf.nn.softplus(out_encoder[..., (2*d_miwae):(3*d_miwae)]))) zgivenx = q_zgivenxobs.sample(K) zgivenx_flat = tf.reshape(zgivenx,[K*batch_size,d_miwae]) data_flat = tf.reshape(tf.tile(x,[K,1]),[-1,1]) # ########## out_decoder = decoder(zgivenx_flat) all_means_obs_model = out_decoder[..., :(p+pw)] all_scales_obs_model = tf.nn.softplus(out_decoder[..., (p+pw):(2*(p+pw))]) + 0.001 all_degfreedom_obs_model = tf.nn.softplus(out_decoder[..., (2*(p+pw)):(3*(p+pw))]) + 3 all_log_pxgivenz_flat = tfd.StudentT(loc=tf.reshape(all_means_obs_model,[-1,1]),scale=tf.reshape(all_scales_obs_model,[-1,1]),df=tf.reshape(all_degfreedom_obs_model,[-1,1])).log_prob(data_flat) all_log_pxgivenz = tf.reshape(all_log_pxgivenz_flat,[K*batch_size,p+pw]) # ########## logpxobsgivenz = tf.reshape(tf.reduce_sum(tf.multiply(all_log_pxgivenz[:,0:p_mod],tiledmask_float[:,0:p_mod]),1),[K,batch_size]) logpz = p_z.log_prob(zgivenx) logq = q_zgivenxobs.log_prob(zgivenx) # ########## miwae_loss = -tf.reduce_mean(tf.reduce_logsumexp(logpxobsgivenz + logpz - logq,0)) +tf.log(tf.cast(K,tf.float32)) train_miss = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(miwae_loss) # ########## xgivenz = tfd.Independent( distribution=tfd.StudentT(loc=all_means_obs_model, scale=all_scales_obs_model, df=all_degfreedom_obs_model)) # ########## imp_weights = tf.nn.softmax(logpxobsgivenz + logpz - logq,0) # these are w_1,....,w_L for all observations in the batch xms = tf.reshape(xgivenz.mean(),[K,batch_size,p+pw]) xm=tf.einsum('ki,kij->ij', imp_weights, xms) # ########## z_hat = tf.einsum('ki,kij->ij', imp_weights, zgivenx) # ########## sir_logits = tf.transpose(logpxobsgivenz + logpz - logq) sirx = tfd.Categorical(logits = sir_logits).sample(num_samples_xmul) xmul = tf.reshape(xgivenz.sample(),[K,batch_size,p+pw]) sirz = tfd.Categorical(logits = sir_logits).sample(num_samples_zmul) zmul = tf.reshape(zgivenx,[K,batch_size,d_miwae]) # ########## miwae_loss_train=np.array([]) mse_train=np.array([]) bs = 64 # batch size n_epochs = 602 xhat = np.copy(xhat_0) # This will be out imputed data matrix x_mul_imp = np.tile(xhat_0,[num_samples_xmul,1,1]) zhat = np.zeros([n,d_miwae]) # low-dimensional representations zhat_mul = np.tile(zhat, [num_samples_zmul,1,1]) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for ep in range(1,n_epochs): perm = np.random.permutation(n) # We use the "random reshuffling" version of SGD batches_data = np.array_split(xhat_0[perm,], n/bs) batches_mask = np.array_split(mask[perm,], n/bs) for it in range(len(batches_data)): train_miss.run(feed_dict={x: batches_data[it], learning_rate: 0.001, K:20, xmask: batches_mask[it]}) # Gradient step if ep % 200 == 1: losstrain = np.array([miwae_loss.eval(feed_dict={x: xhat_0, K:20, xmask: mask})]) # MIWAE bound evaluation miwae_loss_train = np.append(miwae_loss_train,-losstrain,axis=0) print('Epoch %g' %ep) print('MIWAE likelihood bound %g' %-losstrain) for i in range(n): # We impute the observations one at a time for memory reasons # # Single imputation: xhat[i,:][~mask[i,:]]=xm.eval(feed_dict={x: xhat_0[i,:].reshape([1,p+pw]), K:10000, xmask: mask[i,:].reshape([1,p+pw])})[~mask[i,:].reshape([1,p+pw])] # # Multiple imputation: # si, xmu = sess.run([sirx, xmul],feed_dict={x: xhat_0[i,:].reshape([1,p+pw]), K:10000, xmask: mask[i,:].reshape([1,p+pw])}) # x_mul_imp[:,i,:][~np.tile(mask[i,:].reshape([1,p+pw]),[num_samples_xmul,1])] = np.squeeze(xmu[si,:,:])[~np.tile(mask[i,:].reshape([1,p+pw]),[num_samples_xmul,1])] # Dimension reduction: zhat[i,:] = z_hat.eval(feed_dict={x: xhat_0[i,:].reshape([1,p+pw]), K:10000, xmask: mask[i,:].reshape([1,p+pw])}) # Z|X* sampling: si, zmu = sess.run([sirz, zmul],feed_dict={x: xhat_0[i,:].reshape([1,p+pw]), K:10000, xmask: mask[i,:].reshape([1,p+pw])}) zhat_mul[:,i,:] = np.squeeze(zmu[si,:,:]) err = np.array([mse(xhat[:,:p_mod],xfull[:,:p_mod],mask[:,:p_mod])]) mse_train = np.append(mse_train,err,axis=0) print('Imputation MSE %g' %err) print('-----') # ########## print('save files') if add_mask: xhat_rescaled = xhat[:,0:int(p/2)]*np.std(data,0) + np.mean(data,0) else: xhat_rescaled = xhat[:,:p]*np.std(data,0) + np.mean(data,0) np.savetxt(out_folder+out_file_text+"_imp.csv", xhat_rescaled, delimiter=";") # xmiss_rescaled = xmiss*np.std(data,0) + np.mean(data,0) # np.savetxt(out_folder+"/"+out_file_text+"_xmiss.csv", xmiss_rescaled, delimiter=";") np.savetxt(out_folder+out_file_text+"_zhat.csv", zhat, delimiter=";") # if add_mask: # x_mul_imp_rescaled = x_mul_imp[:,:,0:int(p/2)] # else: # x_mul_imp_rescaled = x_mul_imp # for i in range(num_samples_xmul): # x_mul_imp_rescaled[i,:,:p] = x_mul_imp_rescaled[i,:,:p]*np.std(data,0) + np.mean(data,0) # np.savetxt(out_folder+"/"+out_file_text+"_imp_m"+str(i)+".csv", x_mul_imp_rescaled[i,:,:p], delimiter=";") zhat_mul_rescaled = zhat_mul for i in range(num_samples_zmul): np.savetxt(out_folder+out_file_text+"_zhat_m"+str(i)+".csv", zhat_mul_rescaled[i,:,:], delimiter=";")
def __init__(self): # ---------------------------------------------------------------- # # PARAMETERS: # Data Parameters: batch_size = 100 # Learning Parameters: LEARNING_RATE = 0.001 LEARNING_THRESHOLD = 0.0001 N_SCREEN_ITERATIONS = 1 # Network Parameters: nInputs = 2 nNeuronsLayer1 = 100 nOutputs = 1 # Network Construction: x_ = tf.placeholder(tf.float32, shape=[None, nInputs]) y_ = tf.placeholder(tf.float32, shape=[None, nOutputs]) W1 = tf.Variable(tf.truncated_normal([nInputs,nNeuronsLayer1], stddev=0.1)) W2 = tf.Variable(tf.truncated_normal([nNeuronsLayer1,nNeuronsLayer1], stddev=0.1)) W3 = tf.Variable(tf.truncated_normal([nNeuronsLayer1,nNeuronsLayer1], stddev=0.1)) WOut = tf.Variable(tf.truncated_normal([nNeuronsLayer1,nOutputs], stddev=0.1)) b1 = tf.Variable(tf.truncated_normal([nNeuronsLayer1], stddev=0.1)) b2 = tf.Variable(tf.truncated_normal([nOutputs], stddev=0.1)) b3 = tf.Variable(tf.truncated_normal([nOutputs], stddev=0.1)) bOut = tf.Variable(tf.truncated_normal([nOutputs], stddev=0.1)) y1 = tf.sigmoid(tf.matmul(x_, W1) + b1) y2 = tf.sigmoid(tf.matmul(y1, W2) + b2) y3 = tf.sigmoid(tf.matmul(y2, W3) + b3) yOut = tf.sigmoid(tf.matmul(y3, WOut) + bOut) # ---------------------------------------------------------------- # # Define loss, optimizer, accuracy: #cost = tf.reduce_mean(tf.nn.l2_loss(y_ - yOut)) cost = tf.reduce_mean(tf.square(y_ - yOut)) train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost) acc, acc_op = tf.metrics.accuracy(labels=tf.round(y_), predictions=tf.round(yOut)) # ---------------------------------------------------------------- # # Session Initialization: sess = tf.InteractiveSession() tf.global_variables_initializer().run() tf.local_variables_initializer().run() # ---------------------------------------------------------------- # # Prepare Data: datasetFeatures, datasetTargets = self.getTrainingSet() batchesLeft = int(len(datasetTargets)/batch_size) # ---------------------------------------------------------------- # # Train: i = 0 k = 0 while True: if batchesLeft > 0: if i % batch_size == 0: batch_x, batch_y, batchesLeft = self.getNextBatch(batch_size, i, batchesLeft,datasetFeatures,datasetTargets) out_batch, _ = sess.run((yOut, train_step), feed_dict={x_: batch_x, y_: batch_y}) else: batchesLeft = int(len(datasetTargets)/batch_size) i = -1 # Print Test: if k % 1000 == 0: v = sess.run([acc, acc_op], feed_dict={yOut: out_batch, y_: batch_y}) print("Accuracy:") print(v[0]) # Condition On Exit: if v[0] > 0.90: break k +=1 i +=1 print("Exited") # ---------------------------------------------------------------- # # Return Trained Neural Network self.nnTrained = NeuralNetwork() self.nnTrained.W1 = sess.run(W1) self.nnTrained.W2 = sess.run(W2) self.nnTrained.b1 = sess.run(b1) self.nnTrained.b2 = sess.run(b2) self.returnTrainedNN()
def __init__(self, n_in=784, n_out=10, hidden_layers_sizes=[25, 25], epsilon=0.25, _batch_size=280, finetuneLR=0.01): ''' :param n_in: int, the dimension of input :param n_out: int, the dimension of output :param hidden_layers_sizes: list or tuple, the number of hidden neurons, the last item will be the number of hidden neurons in the last hidden layer :param epsilon: privacy budget epsilon :param _batch_size: the batch size :param finetuneLR: fine tunning learning rate ''' # Number of layers assert len(hidden_layers_sizes) > 0 self.n_layers = len(hidden_layers_sizes) self.layers = [] # hidden layers self.params = [] # keep track of params for training self.last_n_in = hidden_layers_sizes[ -1] # the number of hidden neurons in the last hidden layer self.pretrain_ops = [] # list of pretrain objective functions for hidden layers self.epsilon = epsilon # privacy budget epsilon epsilon self.batch_size = _batch_size # batch size # Define the input, output, Laplace noise for the output layer self.x = tf.placeholder(tf.float32, shape=[None, n_in], name='x') self.y = tf.placeholder(tf.float32, shape=[None, n_out]) self.LaplaceNoise = tf.placeholder(tf.float32, self.last_n_in) ###### ############################# ##Construct the Model######## ############################# # Create the 1st auto-encoder layer Auto_Layer1 = Autoencoder(inpt=self.x, n_in=784, n_out=hidden_layers_sizes[0], activation=tf.nn.sigmoid) self.layers.append(Auto_Layer1) self.params.extend(Auto_Layer1.params) # get the pretrain objective function self.pretrain_ops.append( Auto_Layer1.get_dp_train_ops(epsilon=self.epsilon, data_size=50000, learning_rate=0.01)) ### # Create the 2nd cauto-encoder layer Auto_Layer2 = Autoencoder(inpt=self.layers[-1].output, n_in=self.layers[-1].n_out, n_out=hidden_layers_sizes[1], activation=tf.nn.sigmoid) self.layers.append(Auto_Layer2) self.params.extend(Auto_Layer2.params) # get the pretrain objective function self.pretrain_ops.append( Auto_Layer2.get_dp_train_ops(epsilon=self.epsilon, data_size=50000, learning_rate=0.01)) ### # Create the flat connected hidden layer flat1 = HiddenLayer(inpt=self.layers[-1].output, n_in=self.layers[-1].n_out, n_out=self.last_n_in, activation=tf.nn.relu) self.layers.append(flat1) self.params.extend(flat1.params) ### # Create the output layer # We use the differentially private Logistic Regression (dpLogisticRegression) layer as the objective function self.output_layer = dpLogisticRegression( inpt=self.layers[-1].output, n_in=self.last_n_in, n_out=n_out, LaplaceNoise=self.LaplaceNoise) # We can also use the non-differentially private layer: LogisticRegression(inpt=self.layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_out) self.params.extend(self.output_layer.params) ### ####################################### ##Define Fine Tune Cost and Optimizer## ####################################### # The finetuning cost self.cost = self.output_layer.cost(self.y) # train_op for finetuning with AdamOptimizer global_step = tf.Variable(0, trainable=False) #learning_rate = tf.train.exponential_decay(finetuneLR, global_step, 700, 0.96, staircase=True); # learning rate decay can be carefully used # Fine tune with AdamOptimizer. Note that we do not fine tune the pre-trained parameters at the auto-encoder layers self.train_op = tf.train.AdamOptimizer(finetuneLR).minimize( self.cost, var_list=[flat1.params, self.output_layer.params], global_step=global_step) # The accuracy self.accuracy = self.output_layer.accuarcy(self.y)
def bias_variable(shape): return tf.Variable(tf.constant(0.05, shape=shape))