#-*- coding: utf-8 -*- import tensorflow as tf x_data = [1, 2, 3] y_data = [1, 2, 3] """ try to find vlaues for W and b compute y_data = W * x_data + b (we know that W should be 1 and b 0, but Tensorflow will figure that out for us) """ W = tf.Variable(tf.random_uniform([1], -1.0, 1.0)) b = tf.Variable(tf.random_uniform([1], -1.0, 1.0)) # X, Y 를 placeholder 로 선언한다. 이제 run 실행 시 값을 지정할 수 있다 X = tf.placeholder(tf.float32) Y = tf.placeholder(tf.float32) # our hypothesis hypothesis = W * X + b # simplified cost function cost = tf.reduce_mean(tf.square(hypothesis - Y)) # minimize a = tf.Variable(0.1) optimizer = tf.train.GradientDescentOptimizer(a) train = optimizer.minimize(cost) # before starting, initilize the variables. we will 'run' this first
def weight_init(shape, name_for_weight):
Xavier_init = np.sqrt(2.0) * np.sqrt(2.0 / np.array(shape).sum())
weights = tf.truncated_normal(shape, stddev = Xavier_init)
return tf.Variable(weights, name = name_for_weight)
import tensorflow as tf
# placeholder for input to the computation
x = tf.placeholder(dtype=tf.float32, name="x")
# bias variable for the affine weight transformation
b = tf.Variable(tf.zeros(100))
# weight variable for the affine wegiht transformation with random values
W = tf.Variable(tf.random_uniform([784, 100]), tf.float32)
# activation as a function of the weight transformation
a = tf.relu(tf.matmul(W, x) + b)
# cost computed as a function of the activation
# and the target optimization task
C = [...]
# Start session to run the computational graph
session = tf.InteractiveSession()
# Initialize all variables, in this example only the weight
# matrix depends on an initialization
tf.global_variables_initializer()
for i in range(epochs):
result = session.run(C, feed_dict={x: data[batch_indices]})
print(i, result)
def train():
avg_accuracy = 0
mnist = input_data.read_data_sets(FLAGS.data_dir,
fake_data=FLAGS.fake_data)
sess = tf.InteractiveSession()
# Input placeholders
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, 784], name='x-input')
x_quantized = fake_quantize_tensor(x,
input_quantization,
0,
1,
name="quantized_input")
y_ = tf.placeholder(tf.int64, [None], name='y-input')
with tf.name_scope('input_reshape'):
image_shaped_input = tf.Variable(
[-1, 28, 28, 1], collections=[tf.GraphKeys.GLOBAL_VARIABLES])
image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])
tf.summary.image('input', image_shaped_input, 10)
# Functions to create fc and conv layers depending on quantization settings
def create_fc_layer(input_tensor,
input_dim,
output_dim,
bits_w,
max_w,
bits_b,
max_b,
bits_a,
max_a,
noise_stddev,
layer_name,
act=tf.nn.relu):
if (quantization_enabled == False):
return fc_layer(input_tensor, input_dim, output_dim, layer_name,
act)
elif (noise_enabled_fc == False):
return fc_layer_quantized(input_tensor, input_dim, output_dim,
bits_w, max_w, bits_b, max_b, bits_a,
max_a, layer_name, act)
else:
return fc_layer_quantized_add_noise(input_tensor, input_dim,
output_dim, bits_w, max_w,
bits_b, max_b, bits_a, max_a,
noise_stddev, layer_name, act)
def create_conv_layer(input_data, num_input_channels, num_filters,
filter_shape, pool_shape, bits_w, max_w, bits_b,
max_b, bits_a, max_a, noise_stddev, layer_name):
if (quantization_enabled == False):
return conv_layer(input_data, num_input_channels, num_filters,
filter_shape, pool_shape, layer_name)
elif (noise_enabled_conv == False):
return conv_layer_quantized(input_data, num_input_channels,
num_filters, filter_shape, pool_shape,
bits_w, max_w, bits_b, max_b, bits_a,
max_a, layer_name)
else:
return conv_layer_quantized_add_noise(input_data,
num_input_channels,
num_filters, filter_shape,
pool_shape, bits_w, max_w,
bits_b, max_b, bits_a, max_a,
noise_stddev, layer_name)
if (conv_enabled):
if (quantization_enabled):
image_shaped_input_quantized = fake_quantize_tensor(
image_shaped_input,
input_quantization,
0,
1,
name="quantized_input")
else:
image_shaped_input_quantized = image_shaped_input
layer1 = create_conv_layer(image_shaped_input_quantized,
1,
32, [5, 5], [2, 2],
conv1_w_bits,
conv1_w_max,
conv1_b_bits,
conv1_b_max,
conv1_a_bits,
conv1_a_max,
noise_stddev,
layer_name='conv1')
layer2 = create_conv_layer(layer1,
32,
64, [5, 5], [2, 2],
conv2_w_bits,
conv2_w_max,
conv2_b_bits,
conv2_b_max,
conv2_a_bits,
conv2_a_max,
noise_stddev,
layer_name='conv2')
with tf.name_scope('flatten'):
x_flattened = tf.reshape(layer2, [-1, 7 * 7 * 64])
x_shape = 7 * 7 * 64
else:
if (quantization_enabled):
x_flattened = fake_quantize_tensor(x,
input_quantization,
0,
1,
name="quantized_input")
else:
x_flattened = x
x_shape = 28 * 28
if (num_layers == 3):
hidden1 = create_fc_layer(x_flattened, x_shape, fc1_depth, fc1_w_bits,
fc1_w_max, fc1_b_bits, fc1_b_max, fc1_a_bits,
fc1_a_max, noise_stddev, 'fully_connected1')
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
tf.summary.scalar('dropout_keep_probability', keep_prob)
dropped = tf.nn.dropout(hidden1, keep_prob)
# Middle Layer (using settings 3)
hidden2 = create_fc_layer(dropped, fc1_depth, fc3_depth, fc3_w_bits,
fc3_w_max, fc3_b_bits, fc3_b_max, fc3_a_bits,
fc3_a_max, noise_stddev,
'fully_connected_middle')
with tf.name_scope('dropout2'):
keep_prob2 = tf.placeholder(tf.float32)
tf.summary.scalar('dropout2_keep_probability', keep_prob2)
dropped2 = tf.nn.dropout(hidden2, keep_prob2)
# Do not apply softmax activation yet, see below.
y = create_fc_layer(dropped2,
fc3_depth,
fc2_depth,
fc2_w_bits,
fc2_w_max,
fc2_b_bits,
fc2_b_max,
fc2_a_bits,
fc2_a_max,
noise_stddev,
'fully_connected2',
act=tf.identity)
elif (num_layers == 1):
keep_prob = tf.placeholder(tf.float32) # Need to create placeholders
keep_prob2 = tf.placeholder(
tf.float32) # Even though they wont be used
y = create_fc_layer(x_flattened,
x_shape,
fc2_depth,
fc2_w_bits,
fc2_w_max,
fc2_b_bits,
fc2_b_max,
fc2_a_bits,
fc2_a_max,
noise_stddev,
layer_name='fully_connected',
act=tf.identity)
else: # Otherwise create 2 layers
hidden1 = create_fc_layer(x_flattened, x_shape, fc1_depth, fc1_w_bits,
fc1_w_max, fc1_b_bits, fc1_b_max, fc1_a_bits,
fc1_a_max, noise_stddev, 'fully_connected1')
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
tf.summary.scalar('dropout_keep_probability', keep_prob)
dropped = tf.nn.dropout(hidden1, keep_prob)
keep_prob2 = tf.placeholder(tf.float32) # Need to create a placholder
y = create_fc_layer(dropped,
fc1_depth,
fc2_depth,
fc2_w_bits,
fc2_w_max,
fc2_b_bits,
fc2_b_max,
fc2_a_bits,
fc2_a_max,
noise_stddev,
'fully_connected2',
act=tf.identity)
with tf.name_scope('cross_entropy'):
# The raw formulation of cross-entropy can be numerically unstable.
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)),
# reduction_indices=[1]))
#
# So here we use tf.losses.sparse_softmax_cross_entropy on the
# raw logit outputs of the nn_layer above, and then average across the batch.
with tf.name_scope('total'):
cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_,
logits=y)
tf.summary.scalar('cross_entropy', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(
FLAGS.learning_rate).minimize(cross_entropy)
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y, 1), y_)
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Merge all the summaries and write them out
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test')
tf.global_variables_initializer().run()
# Train the model, and also write summaries.
# Every 10th step, measure test-set accuracy, and write test summaries
# All other steps, run train_step on training data, & add training summaries
def feed_dict(train, batch_size=100): # 128
"""Make a TensorFlow feed_dict: maps data onto Tensor placeholders."""
if train or FLAGS.fake_data:
xs, ys = mnist.train.next_batch(batch_size,
fake_data=FLAGS.fake_data)
k = FLAGS.dropout
else:
xs, ys = mnist.test.images, mnist.test.labels
k = 1.0
return {x: xs, y_: ys, keep_prob: k, keep_prob2: k}
for i in range(FLAGS.max_steps + 5):
if (i >= FLAGS.max_steps):
sess.run([merged, train_step], feed_dict=feed_dict(True))
summary, acc = sess.run([merged, accuracy],
feed_dict=feed_dict(False))
test_writer.add_summary(summary, i)
print('Accuracy test %s: %s' % (i - FLAGS.max_steps, acc))
avg_accuracy += acc
elif (i % 100 == 99 and record_summaries == True
and i <= 1000): # Record summaries and test-set accuracy
summary, acc = sess.run([merged, accuracy],
feed_dict=feed_dict(False))
test_writer.add_summary(summary, i)
print(datetime.datetime.now().strftime("%H:%M:%S"),
'Accuracy at step %s: %s' % (i, acc))
elif (i % 1000 == 999): # Record summaries and test-set accuracy
summary, acc = sess.run([merged, accuracy],
feed_dict=feed_dict(False))
test_writer.add_summary(summary, i)
print(datetime.datetime.now().strftime("%H:%M:%S"),
'Accuracy at step %s: %s' % (i, acc))
elif (i % 100 == 0):
# print(datetime.datetime.now().strftime("%H:%M:%S"), "Adding run metadata for Step: ", i)
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, _ = sess.run([merged, train_step],
feed_dict=feed_dict(True),
options=run_options,
run_metadata=run_metadata)
train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
train_writer.add_summary(summary, i)
else: # Train and record summary
summary, _ = sess.run([merged, train_step],
feed_dict=feed_dict(True))
train_writer.add_summary(summary, i)
print("Training Completed: ", datetime.datetime.now().strftime("%H:%M:%S"))
print('Final accuracy: ', avg_accuracy / 5)
x_in, x_flat, y_out = sess.run([x, x_flattened, y],
feed_dict=feed_dict(True, batch_size=1))
np.savetxt("input.csv", x_in[0], delimiter=",", fmt='%f')
np.savetxt("input_reshaped_quantized.csv",
x_flat[0],
delimiter=",",
fmt='%f')
np.savetxt("output.csv", y_out[0], delimiter=",", fmt='%f')
for var in tf.global_variables():
if num_layers == 1:
if '/weights/Variable:0' in var.name:
print(var)
v = sess.run(var) #get_weights(sess.run(var))
np.savetxt("Parameters/weights.csv",
v,
delimiter=",",
fmt='%f')
np.savetxt("Parameters/Q.csv",
calculate_Q(0.066667, v).astype(int),
delimiter=",",
fmt='%i')
elif '/biases/Variable:0' in var.name:
print(var)
v = sess.run(var) # get_biases(sess.run(var))
np.savetxt("Parameters/floating_biases.csv",
v,
delimiter=",",
fmt='%f') # Store the floating point biases
np.savetxt("Parameters/biases.csv",
get_biases(v),
delimiter=",",
fmt='%f') # Store the fixed point biases
else:
print(
"Fixed point inference not implemented yet for networks with more than one layer"
)
if 'fully_connected1/weights/Variable:0' in var.name:
print(var)
elif 'fully_connected1/biases/Variable:0' in var.name:
print(var)
elif 'fully_connected2/weights/Variable:0' in var.name:
print(var)
elif 'fully_connected2/biases/Variable:0' in var.name:
print(var)
elif 'fully_connected3/weights/Variable:0' in var.name:
print(var)
elif 'fully_connected3/biases/Variable:0' in var.name:
print(var)
train_writer.close()
def get_z_var(hparams, batch_size):
z = tf.Variable(tf.random_normal((batch_size, hparams.n_z)), name='z')
return z
def build(self, input_shape):
# We just change the way bias is added and remove it from trainable variable!
input_shape = tf.TensorShape(input_shape)
if input_shape[-1] is None:
raise ValueError('The last dimension of the inputs to `Dense` '
'should be defined. Found `None`.')
if not input_shape.is_fully_defined():
print("the input shape used for build is not fully defined")
self.kernel = self.add_weight(
'kernel',
shape=[input_shape[-1], self.units],
initializer=self.sparseInitializer,
regularizer=self.kernel_regularizer,
constraint=weightFixedAndClippedConstraint(self.sparseInitializer),
dtype=self.dtype,
trainable=True)
self.bias = None
variableShape=(input_shape[-1],self.units)
self.mask = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
self.k1 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
self.k1n = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
self.k2 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
self.k3 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
self.k3n = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
self.k4 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
#only one template starts each cascade:
self.TA0 = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.TI0 = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
#only one inhibition by outputs (units):
self.k5 = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k5n = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k6 = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
#degradation cst for the pseudo-template
self.kdpT = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
#degradation cst for the output
self.kd = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
#degradation cst for the activation cascaded template
self.kdT = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
#degradation cst for the inhibition cascaded template
self.kdT2 = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
self.E0 = tf.Variable(tf.constant(1,dtype=tf.float32),trainable=False,dtype=tf.float32)
self.rescaleFactor = tf.Variable(1,dtype=tf.float32,trainable=False)
self.Xglobal = tf.Variable(tf.constant(1,dtype=tf.float32),trainable=False,dtype=tf.float32)
#other intermediates variable:
self.k1M = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
self.k3M = tf.Variable(tf.zeros(variableShape,dtype=tf.float32),trainable=False)
self.k5M = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k1g = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k1ng = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k2g = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k3g = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k3ng = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k4g = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k1Mg = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.k3Mg = tf.Variable(tf.zeros(variableShape[-1],dtype=tf.float32),trainable=False)
self.built = True
print("Layer successfully built")
LEARNING_RATE_DECAY = 0.9
TRAINING_STEPS = 30000
KEEP_PROB = 0.5
#learning_rate = 0.01
batch_size = 128
trainfile_set = []
num_classes = 95 #the classes of our data set butterfly
refine_layers = [
'fc8'
] #we only refine the last layer, other layer is load from a trained one
train_layers = [
'pool1', 'norm1', 'conv2', 'pool2', 'norm2', 'conv3', 'conv4', 'conv5',
'pool5', 'fc6', 'fc7', 'fc8'
]
global_step = tf.Variable(0, trainable=False)
#Input and output
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
#Initialize model
model = AlexNet(x, keep_prob, num_classes, refine_layers)
# output of the AlexNet
score = model.fc8
var_list = [
v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers
]
#loss function
sign_names = pd.read_csv('signnames.csv')
nb_classes = 43
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
resized = tf.image.resize_images(x, (227, 227))
# NOTE: By setting `feature_extract` to `True` we return
# the second to last layer.
fc7 = AlexNet(resized, feature_extract=True)
# TODO: Define a new fully connected layer followed by a softmax activation to classify
# the traffic signs. Assign the result of the softmax activation to `probs` below.
# HINT: Look at the final layer definition in alexnet.py to get an idea of what this
# should look like.
shape = (fc7.get_shape().as_list()[-1], nb_classes) # use this shape for the weight matrix
fc8W = tf.Variable(tf.truncated_normal(shape, stddev=1e-2))
fc8b = tf.Variable(tf.zeros(nb_classes))
probs = tf.nn.softmax(tf.nn.xw_plus_b(fc7, fc8W, fc8b))
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
# Read Images
im1 = imread("construction.jpg").astype(np.float32)
im1 = im1 - np.mean(im1)
im2 = imread("stop.jpg").astype(np.float32)
im2 = im2 - np.mean(im2)
# Run Inference
data = np.genfromtxt('iris.data', delimiter=",") # iris.data file loading
np.random.shuffle(data) # we shuffle the data
x_data = data[:, 0:4].astype('f4') # the samples are the four first rows of data
y_data = one_hot(data[:, 4].astype(int), 3) # the labels are in the last row. Then we encode them in one hot code
print("\nSome samples...")
for i in range(20):
print(x_data[i], " -> ", y_data[i])
print
x = tf.placeholder("float", [None, 4]) # samples
y_ = tf.placeholder("float", [None, 3]) # labels
W1 = tf.Variable(np.float32(np.random.rand(4, 5)) * 0.1)
b1 = tf.Variable(np.float32(np.random.rand(5)) * 0.1)
W2 = tf.Variable(np.float32(np.random.rand(5, 3)) * 0.1)
b2 = tf.Variable(np.float32(np.random.rand(3)) * 0.1)
h = tf.nn.sigmoid(tf.matmul(x, W1) + b1)
# h = tf.matmul(x, W1) + b1 # Try this!
y = tf.nn.softmax(tf.matmul(h, W2) + b2)
loss = tf.reduce_sum(tf.square(y_ - y))
train = tf.train.GradientDescentOptimizer(0.01).minimize(loss) # learning rate: 0.01
init = tf.initialize_all_variables()
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import tensorflow as tf
import numpy
state = tf.Variable(0.0)
add_op = tf.assign(state, state+tf.constant(1.0))
#assign的结果也是float32
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
#记得initializaer后面加括号
print ('init state:', sess.run(state), 'type:', type(sess.run(state)))
for _ in range(3):
sess.run(add_op)
print(sess.run(state))
print('sess_op:', sess.run(add_op), 'sess_op_type:', type(sess.run(add_op)))
input1 = tf.placeholder(tf.float32)
input2 = tf.placeholder(tf.float32)
output = tf.add(tf.multiply(input1, input2), 1)
with tf.Session() as sess:
print(sess.run([output], feed_dict={input1:[7.0], input2:[2.0]}))
import tensorflow as tf
import numpy as np
# 使用 NumPy 生成假数据(phony data), 总共 100 个点.
x_data = np.float32(np.random.rand(2, 100)) # 随机输入
y_data = np.dot([0.100, 0.200], x_data) + 0.300
# 构造一个线性模型
#
b = tf.Variable(tf.zeros([1]))
W = tf.Variable(tf.random_uniform([1, 2], -1.0, 1.0))
y = tf.matmul(W, x_data) + b
# 最小化方差
loss = tf.reduce_mean(tf.square(y - y_data))
optimizer = tf.train.GradientDescentOptimizer(0.5)
train = optimizer.minimize(loss)
# 初始化变量
#init = tf.initialize_all_variables()
init = tf.global_variables_initializer()
# 启动图 (graph)
sess = tf.Session()
sess.run(init)
# 拟合平面
for step in range(0, 201):
sess.run(train)
if step % 20 == 0:
print(step,sess.run(W),sess.run(b))
#!/usr/bin/env python """ Source: https://www.oreilly.com/learning/hello-tensorflow """ import tensorflow as tf # 1. TensorFlow Graph - first creating the computation graph # Starting off with the default graph graph = tf.get_default_graph() # Setting the Ops x = tf.constant(1.0, name='input') w = tf.Variable(0.8, name='weight') y = tf.mul(w, x, name='output') # Assume that the correct value == 0.0 y_ = tf.constant(0.0) # Loss is the "wrongness" of the model # The model should lessen the square of the diff b/w current output and desired output. loss = (y - y_)**2 # Adding an optimizer to add the "Learning" component optim = tf.train.GradientDescentOptimizer(learning_rate=0.025) # 2. Sessions - computing in Sessions # Initialize all variables and start the session
'''
练习使用滑动平均模型
'''
import tensorflow as tf
v1 = tf.Variable(0, dtype=tf.float32)
step = tf.Variable(0, trainable=False)
ema = tf.train.ExponentialMovingAverage(0.99, step)
maintain_average_op = ema.apply([v1])
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
print(sess.run([v1, ema.average(v1)]))
sess.run(tf.assign(v1, 5))
sess.run(maintain_average_op)
print(sess.run([v1, ema.average(v1)]))
sess.run(tf.assign(step, 1000))
sess.run(tf.assign(v1, 10))
sess.run(maintain_average_op)
print(sess.run([v1, ema.average(v1)]))
sess.run(maintain_average_op)
print(sess.run([v1, ema.average(v1)]))
training_steps = 10000
batch_size = 128
display_step = 200
# Network Parameters
num_input = 28 # MNIST data input (img shape: 28*28)
timesteps = 28 # timesteps
num_hidden = 128 # hidden layer num of features
num_classes = 10 # MNIST total classes (0-9 digits)
# tf Graph input
X = tf.placeholder("float", [None, timesteps, num_input])
Y = tf.placeholder("float", [None, num_classes])
# Define weights
weights = {'out': tf.Variable(tf.random_normal([num_hidden, num_classes]))}
biases = {'out': tf.Variable(tf.random_normal([num_classes]))}
def RNN(x, weights, biases):
# Prepare data shape to match `rnn` function requirements
# Current data input shape: (batch_size, timesteps, n_input)
# Required shape: 'timesteps' tensors list of shape (batch_size, n_input)
# Unstack to get a list of 'timesteps' tensors of shape (batch_size, n_input)
x = tf.unstack(x, timesteps, 1)
# Define a lstm cell with tensorflow
lstm_cell = rnn.BasicLSTMCell(num_hidden, forget_bias=1.0)
def build(self,
n_input_features,
units_per_hidden_layer,
encoder_activation_function='sigmoid',
decoder_activation_function='identity'):
with self.graph.as_default():
self.n_input_features = n_input_features
self.input = self.input = tf.placeholder(tf.float32,
shape=(None,
n_input_features),
name='input')
self.units_per_hidden_layer = units_per_hidden_layer
self.encoder_activation_function = encoder_activation_function
self.decoder_activation_function = decoder_activation_function
self.keep_probability = tf.placeholder(tf.float32,
name='keep_probability')
self.learning_rate = tf.placeholder(tf.float32,
name='learning_rate')
with tf.name_scope('stack'):
n_inputs = self.n_input_features
current_input = self.input
for i, units in enumerate(self.units_per_hidden_layer):
mask = tf.random_uniform(
shape=tf.shape(current_input),
minval=0,
maxval=1,
dtype=tf.float32,
seed=None,
name='weight_initializer_stack_{}'.format(i + 1))
mask = tf.where(mask <= self.keep_probability,
tf.ones_like(current_input,
dtype=tf.float32),
tf.zeros_like(current_input,
dtype=tf.float32),
name='random_mask_stack_{}'.format(i + 1))
self.corrupted_inputs.append(
tf.multiply(current_input,
mask,
name='corruped_input_stack_{}'.format(i +
1)))
with tf.name_scope('encoder_stack_{}'.format(i + 1)):
previous_input = self.corrupted_inputs[-1]
activation_function = self.get_activation_function(
self.encoder_activation_function)
weights = tf.Variable(
tf.truncated_normal([n_inputs, units]),
dtype=tf.float32,
name='weights_stack_{}'.format(i + 1))
self.initial_weights.append(weights)
bias = tf.Variable(tf.zeros([units], dtype=tf.float32),
name='bias_stack_{}'.format(i + 1))
self.initial_biases.append(bias)
self.encoders.append(
activation_function(
tf.add(tf.matmul(previous_input, weights),
bias),
name='encoder_stack_{}'.format(i + 1)))
current_input = self.encoders[-1]
with tf.name_scope('decoder_stack_{}'.format(i + 1)):
weights = tf.Variable(
tf.truncated_normal([units, n_inputs]),
dtype=tf.float32,
name='weights_stack_{}'.format(i + 1))
bias = tf.Variable(tf.zeros([n_inputs],
dtype=tf.float32),
name='bias_stack_{}'.format(i + 1))
activation_function = self.get_activation_function(
self.decoder_activation_function)
self.decoders.append(
activation_function(
tf.add(tf.matmul(self.encoders[-1], weights),
bias),
name='decoder_stack_{}'.format(i + 1)))
n_inputs = units
self.saver = tf.train.Saver()
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def _build_model(self, input_width, input_height, input_channel,
output_size, learning_rate, decay):
# inputs
prev_images = tf.placeholder(dtype=tf.float32,
shape=[None, input_height, input_width, input_channel])
tf.summary.image(name='prev_images', tensor=prev_images)
next_images = tf.placeholder(dtype=tf.float32,
shape=[None, input_height, input_width, input_channel])
tf.summary.image(name='next_images', tensor=next_images)
images = tf.concat([prev_images, next_images], axis=3)
images = tf.image.resize_area(images, [input_height / 4, input_width / 4])
# labels
labels = tf.placeholder(dtype=tf.float32, shape=[None, output_size])
tf.summary.histogram(name='labels', values=labels)
keep_prob = tf.placeholder(dtype=tf.float32, shape=())
# learning rate
self.learning_rate = tf.Variable(learning_rate, name='learning_rate',
trainable=False)
self.decay_lr = tf.assign(self.learning_rate, self.learning_rate * decay)
tf.summary.scalar(name='learning_rate', tensor=self.learning_rate)
# model
with tf.name_scope('conv1'):
h1_size = 32
w = tf.get_variable(name='conv_w1',
shape=[3, 3, input_channel * 2, h1_size],
dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=0.01))
b = tf.get_variable(name='conv_b1', shape=[h1_size],
dtype=tf.float32,
initializer=tf.constant_initializer(value=1e-3))
h = tf.nn.relu(tf.nn.conv2d(images, w, strides=[1, 1, 1, 1],
padding='SAME') + b)
h = tf.nn.max_pool(h, strides=[1, 2, 2, 1], ksize=[1, 2, 2, 1],
padding='SAME')
h = tf.nn.dropout(h, keep_prob)
with tf.name_scope('conv2'):
h2_size = 32
w = tf.get_variable(name='conv_w2',
shape=[3, 3, h1_size, h2_size],
dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=0.01))
b = tf.get_variable(name='conv_b2', shape=[h2_size],
dtype=tf.float32,
initializer=tf.constant_initializer(value=1e-3))
h = tf.nn.relu(tf.nn.conv2d(h, w, strides=[1, 1, 1, 1],
padding='SAME') + b)
h = tf.nn.max_pool(h, strides=[1, 2, 2, 1], ksize=[1, 2, 2, 1],
padding='SAME')
h = tf.nn.dropout(h, keep_prob)
with tf.name_scope('conv3'):
h3_size = 32
w = tf.get_variable(name='conv_w3',
shape=[3, 3, h2_size, h3_size],
dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=0.01))
b = tf.get_variable(name='conv_b3', shape=[h3_size],
dtype=tf.float32,
initializer=tf.constant_initializer(value=1e-3))
h = tf.nn.relu(tf.nn.conv2d(h, w, strides=[1, 1, 1, 1],
padding='SAME') + b)
h = tf.nn.max_pool(h, strides=[1, 2, 2, 1], ksize=[1, 2, 2, 1],
padding='SAME')
h = tf.nn.dropout(h, keep_prob)
with tf.name_scope('fc4'):
h_size = h.get_shape().as_list()
self.logger.info('connect size: %s' % (str(h_size)))
connect_size = h_size[1] * h_size[2] * h_size[3]
h4_size = 1024
w = tf.get_variable(name='w4',
shape=[connect_size, h4_size],
dtype=tf.float32,
initializer=tf.random_normal_initializer(
stddev=np.sqrt(2.0 / connect_size)))
b = tf.get_variable(name='b4', shape=[h4_size],
dtype=tf.float32,
initializer=tf.constant_initializer(value=1e-3))
h = tf.nn.relu(tf.matmul(tf.reshape(h, [-1, connect_size]), w) + b)
with tf.name_scope('fc5'):
h5_size = 256
w = tf.get_variable(name='w5',
shape=[h4_size, h5_size],
dtype=tf.float32,
initializer=tf.random_normal_initializer(
stddev=np.sqrt(2.0 / h4_size)))
b = tf.get_variable(name='b5', shape=[h5_size],
dtype=tf.float32,
initializer=tf.constant_initializer(value=1e-3))
h = tf.nn.relu(tf.matmul(h, w) + b)
with tf.name_scope('output'):
w = tf.get_variable(name='ow', shape=[h5_size, output_size],
dtype=tf.float32,
initializer=tf.random_normal_initializer(
stddev=np.sqrt(2.0 / h5_size)))
b = tf.get_variable(name='ob', shape=[output_size],
dtype=tf.float32,
initializer=tf.constant_initializer(value=1e-3))
logits = tf.matmul(h, w) + b
outputs = logits
tf.summary.histogram(name='outputs', values=outputs)
# loss and optimizer
with tf.name_scope('loss'):
# scale up to cm as unit
loss = tf.reduce_mean(tf.square(logits - labels)) * output_size * (10 **2)
tf.summary.scalar(name='loss', tensor=loss)
error = tf.reduce_mean(tf.abs(logits - labels))
tf.summary.scalar(name='error', tensor=error)
with tf.name_scope('optimizer'):
optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
train_op = optimizer.minimize(loss)
return prev_images, next_images, labels, keep_prob, outputs, \
loss, error, train_op
def train_NN():
x = tf.placeholder(tf.float32,
[None, 256]) # we input 2 vector as our input 128*2
y_ = tf.placeholder(tf.float32,
[None, 5]) # we have four kinds of relationship
#
#w1 = tf.Variable(tf.zeros([256, hiddenunit])) # how many items in each x,unit of hidden layer]
w1 = tf.Variable(tf.truncated_normal([256, hiddenunit], stddev=0.1))
b1 = tf.Variable(tf.zeros([hiddenunit
])) # how many hidden unit in each layer
w2 = tf.Variable(tf.zeros(
[hiddenunit,
5])) # [how many items in each x,unit of hidden layer],初始值是0
b2 = tf.Variable(tf.zeros([5]))
keep_prob = tf.placeholder(tf.float32)
#layer 1
#relu
nnhlyer1 = tf.nn.softmax(tf.matmul(x, w1) + b1)
h_fc1_drop = tf.nn.dropout(nnhlyer1, keep_prob) #avoid overfit
#layer 2
y_nn = tf.nn.softmax(tf.matmul(h_fc1_drop, w2) + b2)
# loss
cross_entropy = tf.reduce_mean(
-tf.reduce_sum(y_ * tf.log(y_nn), reduction_indices=[1])) # loss
#cross_entropy = -tf.reduce_sum(y_*tf.log(y_nn))
correctPred = tf.equal(tf.argmax(y_nn, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correctPred, tf.float32))
# change
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(
logits=(tf.matmul(h_fc1_drop, w2) + b2), labels=y_))
# update
train_step = tf.train.GradientDescentOptimizer(ita).minimize(cross_entropy)
# train_step = tf.train.AdamOptimizer(1e-4).minimize(loss)
#init
init = tf.global_variables_initializer()
saver = tf.train.Saver()
sess = tf.InteractiveSession()
saver = tf.train.Saver()
sess.run(init)
tf.summary.scalar('Loss', cross_entropy)
tf.summary.scalar('Accuracy', accuracy)
merged = tf.summary.merge_all()
logdir = "tensorboard/" + datetime.datetime.now().strftime(
"%Y%m%d-%H%M%S") + "/"
writer = tf.summary.FileWriter(logdir, sess.graph)
# Session
sess = tf.Session()
sess.run(init)
#
for i in range(iterations):
#random
batch_xs, batch_ys = getimageFeaturebynxtbatch(tr_vectors, tr_labels,
batchsize)
sess.run([cross_entropy, train_step],
feed_dict={
x: batch_xs,
y_: batch_ys,
keep_prob: keep_pro
})
# print('>>>batch train, which iteration = %d' % (i))
if i % 20 == 0:
# correct_prediction = tf.equal(tf.argmax(y_nn, 1), tf.argmax(y_, 1))
# accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# print ("Setp: ", i, "Accuracy: ",sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))
save_path = saver.save(sess,
"models/pretrained_lstm.ckpt",
global_step=i)
print("saved to %s" % save_path)
summary = sess.run(merged, {
x: batch_xs,
y_: batch_ys,
keep_prob: 1.0
})
writer.add_summary(summary, i)
writer.close()
def predict(sample):
dataset = pd.read_csv("data.csv")
def normalise(value,values):
m = max(values)
return value/m
index = 2
for i in range(len(sample[0])):
sample[0][i]=normalise(sample[0][i],dataset.iloc[:,index])
index+=1
input = tf.placeholder(tf.float32,[None,12])
W1 = tf.Variable(tf.random.normal([12,20]))
b1 = tf.Variable(tf.random.normal([20]))
layer1 = tf.add(tf.matmul(input,W1),b1)
layer1 = tf.nn.leaky_relu(layer1)
W2 = tf.Variable(tf.random.normal([20,20]))
b2 = tf.Variable(tf.random.normal([20]))
layer2 = tf.add(tf.matmul(layer1,W2),b2)
layer2 = tf.nn.leaky_relu(layer2)
W3 = tf.Variable(tf.random.normal([20,20]))
b3 = tf.Variable(tf.random.normal([20]))
layer3 = tf.add(tf.matmul(layer2,W3),b3)
layer3 = tf.nn.leaky_relu(layer3)
W4 = tf.Variable(tf.random.normal([20,20]))
b4 = tf.Variable(tf.random.normal([20]))
layer4 = tf.add(tf.matmul(layer3,W4),b4)
layer4 = tf.nn.leaky_relu(layer4)
W5 = tf.Variable(tf.random.normal([20,20]))
b5 = tf.Variable(tf.random.normal([20]))
layer5 = tf.add(tf.matmul(layer4,W5),b5)
layer5 = tf.nn.leaky_relu(layer5)
W6 = tf.Variable(tf.random.normal([20,20]))
b6 = tf.Variable(tf.random.normal([20]))
layer6 = tf.add(tf.matmul(layer5,W6),b6)
layer6 = tf.nn.leaky_relu(layer6)
W7 = tf.Variable(tf.random.normal([20,20]))
b7 = tf.Variable(tf.random.normal([20]))
layer7 = tf.add(tf.matmul(layer6,W7),b7)
layer7 = tf.nn.leaky_relu(layer7)
W8 = tf.Variable(tf.random.normal([20,20]))
b8 = tf.Variable(tf.random.normal([20]))
layer8 = tf.add(tf.matmul(layer7,W8),b8)
layer8 = tf.nn.leaky_relu(layer8)
W9 = tf.Variable(tf.random.normal([20,20]))
b9 = tf.Variable(tf.random.normal([20]))
layer9 = tf.add(tf.matmul(layer8,W9),b9)
layer9 = tf.nn.leaky_relu(layer9)
W10= tf.Variable(tf.random.normal([20,1]))
b10 = tf.Variable(tf.random.normal([1]))
output = tf.add(tf.matmul(layer9,W10),b10)
with tf.Session() as sess:
saver = tf.train.Saver()
saver.restore(sess,"housing_price_model/model")
return sess.run(output,feed_dict={input:sample})
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
import tensorflow as tf
import os
# 调整警告等级
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
a = tf.constant([1, 2, 3, 4, 5])
# 变量必须进行显示初始化
var = tf.Variable(tf.random_normal([2, 3], mean=0, stddev=1))
init_op = tf.global_variables_initializer()
print(a, '\n', var)
with tf.Session() as sess:
sess.run(init_op)
print(sess.run([a, var]))
import tensorflow as tf
import numpy as np
weights = tf.Variable(tf.random_normal([784,20],stddev=0.35),name="weights")
heights = tf.Variable(tf.random_normal([20,20]),name="heights")
init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
np.set_printoptions(threshold='nan')
with tf.Session() as sess:
sess.run(init_op)
# saver.restore(sess,"/tmp/tensorflow_data/model.ckpt")
print(weights.eval())
save_path = saver.save(sess,"/tmp/tensorflow_data/model.ckpt")
######################
# CGES Configuration #
######################
tf.app.flags.DEFINE_float('mu', 0.8, 'initialized group sparsity ratio')
tf.app.flags.DEFINE_float('chvar', 0.2, '\'mu\' change per layer')
FLAGS = tf.app.flags.FLAGS
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.memory_usage)
sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options))
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
x = tf.placeholder(tf.float32, shape=[None, 784]) # single flattened 28 * 28 pixel MNIST image
y_ = tf.placeholder(tf.float32, shape=[None, 10]) # 10 classes output
keep_prob = tf.placeholder(tf.float32)
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
batch = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(
FLAGS.base_lr, # Base learning rate.
batch, # Current index.
FLAGS.stepsize, # Decay iteration step.
FLAGS.decay_rate, # Decay rate.
staircase=True)
from mnist_model import mnist_conv
y_conv = mnist_conv(x, 10, keep_prob)
S_vars = [svar for svar in tf.trainable_variables() if 'weight' in svar.name]
ff_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y_))
def map_estimate_BFGS(self,
max_iter=1000,
initialize=True,
batch_size=None):
"""
Runs BFGS on the data
:param max_iter: int | number of max iterations for BFGS
:param initialize: bool | set to True if alpha values should be
initialized
:param batch_size: int | needed for batch update
:return: alpha
"""
sess = self.sess
max_iter = max_iter
if initialize:
self.batch = 0
if batch_size is None:
Inf = genInfectedTensor(self.data, self.numNodes, self.T)
U = genUninfectedTensor(self.data, self.numNodes, self.T)
else:
Inf = genInfectedTensor(
self.data[self.batch:self.batch + batch_size], self.numNodes,
self.T)
U = genUninfectedTensor(
self.data[self.batch:self.batch + batch_size], self.numNodes,
self.T)
self.batch += batch_size
U_ph = tf.placeholder(tf.float32, U.shape)
I_ph = tf.placeholder(tf.float32, Inf.shape)
if initialize:
B = tf.Variable(tf.random_uniform(U.shape[1:]), dtype=tf.float32)
else:
B = tf.Variable(self.a, dtype=tf.float32)
alpha_tensor = tf.nn.relu(B)
psi_1 = tf.map_fn(lambda x: f_psi_1(tf.transpose(alpha_tensor), x),
I_ph,
dtype=tf.float32)
psi_2 = tf.map_fn(lambda x: f_psi_2(tf.transpose(alpha_tensor), x),
U_ph,
dtype=tf.float32)
psi_3 = tf.map_fn(lambda x: f_psi_3(tf.transpose(alpha_tensor), x),
I_ph,
dtype=tf.float32)
prior = gamma_prior(alpha_tensor)
log_p = -(tf.reduce_sum(psi_1) + tf.reduce_sum(psi_2) +
tf.reduce_sum(psi_3) + tf.reduce_sum(prior))
feed_dict = {U_ph: U.eval(session=sess), I_ph: Inf.eval(session=sess)}
optimizer = tf.contrib.opt.\
ScipyOptimizerInterface(log_p,
method='L-BFGS-B',
options={'maxiter': max_iter})
if initialize:
model = tf.global_variables_initializer()
sess.run(model)
optimizer.minimize(sess, feed_dict=feed_dict)
self.a = alpha_tensor.eval(session=sess)
return self.a
L1 = tf.nn.relu(BN1)
L1 = tf.nn.dropout(L1, keep_prob)
tf.summary.histogram('W1', W1)
with tf.name_scope('layer2'):
W2 = weight_init(layer2_shape, 'W2')
z2 = tf.matmul(L1, W2)
BN2 = tf.contrib.layers.batch_norm(z2, center = True, scale = True,
is_training = is_training_holder)
L2 = tf.nn.relu(BN2)
L2 = tf.nn.dropout(L2, keep_prob)
tf.summary.histogram('W2', W2)
with tf.name_scope('output'):
W3 = weight_init(output_shape, 'W3')
b3 = tf.Variable(tf.random_normal([output_shape[1]]))
model = tf.matmul(L2, W3) + b3
tf.summary.histogram('W3', W3)
with tf.name_scope('optimizer'):
base_cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=model, labels=Y))
lossL2 = tf.reduce_mean(tf.nn.l2_loss(W1) +
tf.nn.l2_loss(W2) + tf.nn.l2_loss(W3)) * L2beta
cost = base_cost + lossL2
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
tf.summary.scalar('cost', cost)
with tf.name_scope("accuracy"):
def map_estimate_BFGS_topics(self,
max_iter=1000,
numTopics=2,
initialize=True,
batch_size=None):
"""
Runs BFGS on the data and including the topic information
:param max_iter: int | number of max iterations for BFGS
:param initialize: bool | set to True if alpha values should be
initialized
:param batch_size: int | needed for batch update
:return: alpha
"""
topics = self.topics
sess = self.sess
max_iter = max_iter
if initialize:
self.batch = 0
theta_topics = tf.reshape(
tf.divide(tf.ones((1, numTopics)), numTopics), (numTopics, 1))
if batch_size is None:
Inf = genInfectedTensor(self.data, self.numNodes, self.T)
U = genUninfectedTensor(self.data, self.numNodes, self.T)
else:
Inf = genInfectedTensor(
self.data[self.batch:self.batch + batch_size], self.numNodes,
self.T)
U = genUninfectedTensor(
self.data[self.batch:self.batch + batch_size], self.numNodes,
self.T)
self.batch += batch_size
U_ph = tf.placeholder(tf.float32, U.shape)
I_ph = tf.placeholder(tf.float32, Inf.shape)
rate_intercept = tf.Variable(tf.random_uniform(
(self.numNodes, self.numNodes)),
dtype=tf.float32)
rate_affinity = tf.Variable(tf.random_uniform(
(self.numNodes, self.numNodes, numTopics)),
dtype=tf.float32)
psi_1 = tf.map_fn(lambda x: f_psi_1_t(rate_intercept, rate_affinity, x[
0], x[1], self.numNodes, numTopics), (I_ph, topics),
dtype=tf.float32)
psi_2 = tf.map_fn(lambda x: f_psi_2_t(rate_intercept, rate_affinity, x[
0], x[1], self.numNodes, numTopics), (U_ph, topics),
dtype=tf.float32)
psi_3 = tf.map_fn(lambda x: f_psi_3_t(rate_intercept, rate_affinity, x[
0], x[1], self.numNodes, numTopics), (I_ph, topics),
dtype=tf.float32)
prior = gamma_prior_t(rate_intercept, rate_affinity, theta_topics,
self.numNodes, numTopics)
log_p = -(tf.reduce_sum(psi_1) + tf.reduce_sum(psi_2) +
tf.reduce_sum(psi_3) + tf.reduce_sum(prior))
feed_dict = {U_ph: U.eval(session=sess), I_ph: Inf.eval(session=sess)}
optimizer = tf.contrib.opt.\
ScipyOptimizerInterface(log_p,
method='L-BFGS-B',
options={'maxiter': max_iter})
if initialize:
model = tf.global_variables_initializer()
sess.run(model)
optimizer.minimize(sess, feed_dict=feed_dict)
self.a_t1 = evaluateAlpha(
rate_intercept, rate_affinity, np.array([1, 0]), self.numNodes,
numTopics).eval(session=sess).transpose().round(1)
self.a_t2 = evaluateAlpha(
rate_intercept, rate_affinity, np.array([0, 1]), self.numNodes,
numTopics).eval(session=sess).transpose().round(1)
return self.a_t1, self.a_t2
def run_main(id_data = 0, setting = "dlvm2", link = "nonlinear3",
n = 1000, p = 10, d=3, d_miwae = 3, h_miwae=128, add_mask=True,
num_samples_zmul=200, num_samples_xmul=50, perc_miss = 0.1,
in_folder = "../Data/",
out_folder = "../Data/"):
##########
id = id_data
print(in_folder, out_folder)
print('run_main', id_data, n, p)
#
try:
os.makedirs(in_folder, exist_ok=True)
except FileExistsError:
pass
try:
os.makedirs(out_folder, exist_ok=True)
except FileExistsError:
pass
in_file_text = setting + "_" + link + "_" + str(n) + "_" + str(p) + "_" + str(d) + "_seed" + str(id) + "_propNA"+"{:3.2f}".format(perc_miss)
if add_mask:
out_file_text = setting + "_" + link + "_" + str(n) + "_" + str(p) + "_" + str(d) + "_WY_dmiwae" + str(d_miwae) + "_hmiwae" + str(h_miwae) + "_mask_seed" + str(id) + "_propNA"+"{:3.2f}".format(perc_miss)
else:
out_file_text = setting + "_" + link + "_" + str(n) + "_" + str(p) + "_" + str(d) + "_WY_dmiwae" + str(d_miwae) + "_hmiwae" + str(h_miwae) + "_seed" + str(id) + "_propNA"+"{:3.2f}".format(perc_miss)
print(out_folder+out_file_text)
data = np.array(pd.read_csv(in_folder+in_file_text+"_xcomp.csv", low_memory=False))[:,1:(p+1)]
data_miss = np.array(pd.read_csv(in_folder+in_file_text+"_xmisswy.csv", low_memory=False))[:,1:(p+3)]
print('dim(data)', data.shape)
print('dim([data_miss , w])', data_miss.shape)
# #########
xfull = (data - np.mean(data,0))/np.std(data,0)
n = xfull.shape[0] # number of observations
p = xfull.shape[1] # number of features
print(n)
print(p)
# ##########
np.random.seed(1234)
tf.set_random_seed(1234)
xmiss = np.copy(data_miss)
mask = np.isfinite(xmiss) # binary mask that indicates which values are missing
print('mask', mask.shape)
# ##########
xhat_0 = np.copy(xmiss)
xhat_0[np.isnan(xmiss)] = 0
p_mod = p
pw = 2
if add_mask:
mask_mod = np.copy(mask)
#mask_mod = mask_mod.astype(float)
#mask_mod = (mask_mod - np.mean(mask_mod,0))/np.std(mask_mod,0)
xhat_0 = np.concatenate((xhat_0, mask_mod), axis=1)
xfull = np.concatenate((xfull, mask_mod), axis=1)
mask = np.concatenate((mask, np.ones_like(mask).astype(bool)), axis = 1)
p = p*2
print('[data, w, y, mask, 1, 1]', xhat_0.shape)
pw = 4
# ##########
x = tf.placeholder(tf.float32, shape=[None, p+pw]) # Placeholder for xhat_0
learning_rate = tf.placeholder(tf.float32, shape=[])
batch_size = tf.shape(x)[0]
xmask = tf.placeholder(tf.bool, shape=[None, p+pw])
K= tf.placeholder(tf.int32, shape=[]) # Placeholder for the number of importance weights
# ##########
p_z = tfd.MultivariateNormalDiag(loc=tf.zeros(d_miwae, tf.float32))
# ##########
sigma = "relu"
decoder = tfk.Sequential([
tfkl.InputLayer(input_shape=[d_miwae,]),
tfkl.Dense(h_miwae, activation=sigma,kernel_initializer="orthogonal"),
tfkl.Dense(h_miwae, activation=sigma,kernel_initializer="orthogonal"),
tfkl.Dense(3*(p+pw),kernel_initializer="orthogonal") # the decoder will output both the mean, the scale, and the number of degrees of freedoms (hence the 3*p)
])
# ##########
tiledmask = tf.tile(xmask,[K,1])
tiledmask_float = tf.cast(tiledmask,tf.float32)
mask_not_float = tf.abs(-tf.cast(xmask,tf.float32))
iota = tf.Variable(np.zeros([1,p+pw]),dtype=tf.float32)
tilediota = tf.tile(iota,[batch_size,1])
iotax = x + tf.multiply(tilediota,mask_not_float)
# ##########
encoder = tfk.Sequential([
tfkl.InputLayer(input_shape=[p+pw,]),
tfkl.Dense(h_miwae, activation=sigma,kernel_initializer="orthogonal"),
tfkl.Dense(h_miwae, activation=sigma,kernel_initializer="orthogonal"),
tfkl.Dense(3*d_miwae,kernel_initializer="orthogonal")
])
# ##########
out_encoder = encoder(iotax)
q_zgivenxobs = tfd.Independent(distribution=tfd.StudentT(loc=out_encoder[..., :d_miwae], scale=tf.nn.softplus(out_encoder[..., d_miwae:(2*d_miwae)]), df=3 + tf.nn.softplus(out_encoder[..., (2*d_miwae):(3*d_miwae)])))
zgivenx = q_zgivenxobs.sample(K)
zgivenx_flat = tf.reshape(zgivenx,[K*batch_size,d_miwae])
data_flat = tf.reshape(tf.tile(x,[K,1]),[-1,1])
# ##########
out_decoder = decoder(zgivenx_flat)
all_means_obs_model = out_decoder[..., :(p+pw)]
all_scales_obs_model = tf.nn.softplus(out_decoder[..., (p+pw):(2*(p+pw))]) + 0.001
all_degfreedom_obs_model = tf.nn.softplus(out_decoder[..., (2*(p+pw)):(3*(p+pw))]) + 3
all_log_pxgivenz_flat = tfd.StudentT(loc=tf.reshape(all_means_obs_model,[-1,1]),scale=tf.reshape(all_scales_obs_model,[-1,1]),df=tf.reshape(all_degfreedom_obs_model,[-1,1])).log_prob(data_flat)
all_log_pxgivenz = tf.reshape(all_log_pxgivenz_flat,[K*batch_size,p+pw])
# ##########
logpxobsgivenz = tf.reshape(tf.reduce_sum(tf.multiply(all_log_pxgivenz[:,0:p_mod],tiledmask_float[:,0:p_mod]),1),[K,batch_size])
logpz = p_z.log_prob(zgivenx)
logq = q_zgivenxobs.log_prob(zgivenx)
# ##########
miwae_loss = -tf.reduce_mean(tf.reduce_logsumexp(logpxobsgivenz + logpz - logq,0)) +tf.log(tf.cast(K,tf.float32))
train_miss = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(miwae_loss)
# ##########
xgivenz = tfd.Independent(
distribution=tfd.StudentT(loc=all_means_obs_model, scale=all_scales_obs_model, df=all_degfreedom_obs_model))
# ##########
imp_weights = tf.nn.softmax(logpxobsgivenz + logpz - logq,0) # these are w_1,....,w_L for all observations in the batch
xms = tf.reshape(xgivenz.mean(),[K,batch_size,p+pw])
xm=tf.einsum('ki,kij->ij', imp_weights, xms)
# ##########
z_hat = tf.einsum('ki,kij->ij', imp_weights, zgivenx)
# ##########
sir_logits = tf.transpose(logpxobsgivenz + logpz - logq)
sirx = tfd.Categorical(logits = sir_logits).sample(num_samples_xmul)
xmul = tf.reshape(xgivenz.sample(),[K,batch_size,p+pw])
sirz = tfd.Categorical(logits = sir_logits).sample(num_samples_zmul)
zmul = tf.reshape(zgivenx,[K,batch_size,d_miwae])
# ##########
miwae_loss_train=np.array([])
mse_train=np.array([])
bs = 64 # batch size
n_epochs = 602
xhat = np.copy(xhat_0) # This will be out imputed data matrix
x_mul_imp = np.tile(xhat_0,[num_samples_xmul,1,1])
zhat = np.zeros([n,d_miwae]) # low-dimensional representations
zhat_mul = np.tile(zhat, [num_samples_zmul,1,1])
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for ep in range(1,n_epochs):
perm = np.random.permutation(n) # We use the "random reshuffling" version of SGD
batches_data = np.array_split(xhat_0[perm,], n/bs)
batches_mask = np.array_split(mask[perm,], n/bs)
for it in range(len(batches_data)):
train_miss.run(feed_dict={x: batches_data[it], learning_rate: 0.001, K:20, xmask: batches_mask[it]}) # Gradient step
if ep % 200 == 1:
losstrain = np.array([miwae_loss.eval(feed_dict={x: xhat_0, K:20, xmask: mask})]) # MIWAE bound evaluation
miwae_loss_train = np.append(miwae_loss_train,-losstrain,axis=0)
print('Epoch %g' %ep)
print('MIWAE likelihood bound %g' %-losstrain)
for i in range(n): # We impute the observations one at a time for memory reasons
# # Single imputation:
xhat[i,:][~mask[i,:]]=xm.eval(feed_dict={x: xhat_0[i,:].reshape([1,p+pw]), K:10000, xmask: mask[i,:].reshape([1,p+pw])})[~mask[i,:].reshape([1,p+pw])]
# # Multiple imputation:
# si, xmu = sess.run([sirx, xmul],feed_dict={x: xhat_0[i,:].reshape([1,p+pw]), K:10000, xmask: mask[i,:].reshape([1,p+pw])})
# x_mul_imp[:,i,:][~np.tile(mask[i,:].reshape([1,p+pw]),[num_samples_xmul,1])] = np.squeeze(xmu[si,:,:])[~np.tile(mask[i,:].reshape([1,p+pw]),[num_samples_xmul,1])]
# Dimension reduction:
zhat[i,:] = z_hat.eval(feed_dict={x: xhat_0[i,:].reshape([1,p+pw]), K:10000, xmask: mask[i,:].reshape([1,p+pw])})
# Z|X* sampling:
si, zmu = sess.run([sirz, zmul],feed_dict={x: xhat_0[i,:].reshape([1,p+pw]), K:10000, xmask: mask[i,:].reshape([1,p+pw])})
zhat_mul[:,i,:] = np.squeeze(zmu[si,:,:])
err = np.array([mse(xhat[:,:p_mod],xfull[:,:p_mod],mask[:,:p_mod])])
mse_train = np.append(mse_train,err,axis=0)
print('Imputation MSE %g' %err)
print('-----')
# ##########
print('save files')
if add_mask:
xhat_rescaled = xhat[:,0:int(p/2)]*np.std(data,0) + np.mean(data,0)
else:
xhat_rescaled = xhat[:,:p]*np.std(data,0) + np.mean(data,0)
np.savetxt(out_folder+out_file_text+"_imp.csv", xhat_rescaled, delimiter=";")
# xmiss_rescaled = xmiss*np.std(data,0) + np.mean(data,0)
# np.savetxt(out_folder+"/"+out_file_text+"_xmiss.csv", xmiss_rescaled, delimiter=";")
np.savetxt(out_folder+out_file_text+"_zhat.csv", zhat, delimiter=";")
# if add_mask:
# x_mul_imp_rescaled = x_mul_imp[:,:,0:int(p/2)]
# else:
# x_mul_imp_rescaled = x_mul_imp
# for i in range(num_samples_xmul):
# x_mul_imp_rescaled[i,:,:p] = x_mul_imp_rescaled[i,:,:p]*np.std(data,0) + np.mean(data,0)
# np.savetxt(out_folder+"/"+out_file_text+"_imp_m"+str(i)+".csv", x_mul_imp_rescaled[i,:,:p], delimiter=";")
zhat_mul_rescaled = zhat_mul
for i in range(num_samples_zmul):
np.savetxt(out_folder+out_file_text+"_zhat_m"+str(i)+".csv", zhat_mul_rescaled[i,:,:], delimiter=";")
def __init__(self):
# ---------------------------------------------------------------- #
# PARAMETERS:
# Data Parameters:
batch_size = 100
# Learning Parameters:
LEARNING_RATE = 0.001
LEARNING_THRESHOLD = 0.0001
N_SCREEN_ITERATIONS = 1
# Network Parameters:
nInputs = 2
nNeuronsLayer1 = 100
nOutputs = 1
# Network Construction:
x_ = tf.placeholder(tf.float32, shape=[None, nInputs])
y_ = tf.placeholder(tf.float32, shape=[None, nOutputs])
W1 = tf.Variable(tf.truncated_normal([nInputs,nNeuronsLayer1], stddev=0.1))
W2 = tf.Variable(tf.truncated_normal([nNeuronsLayer1,nNeuronsLayer1], stddev=0.1))
W3 = tf.Variable(tf.truncated_normal([nNeuronsLayer1,nNeuronsLayer1], stddev=0.1))
WOut = tf.Variable(tf.truncated_normal([nNeuronsLayer1,nOutputs], stddev=0.1))
b1 = tf.Variable(tf.truncated_normal([nNeuronsLayer1], stddev=0.1))
b2 = tf.Variable(tf.truncated_normal([nOutputs], stddev=0.1))
b3 = tf.Variable(tf.truncated_normal([nOutputs], stddev=0.1))
bOut = tf.Variable(tf.truncated_normal([nOutputs], stddev=0.1))
y1 = tf.sigmoid(tf.matmul(x_, W1) + b1)
y2 = tf.sigmoid(tf.matmul(y1, W2) + b2)
y3 = tf.sigmoid(tf.matmul(y2, W3) + b3)
yOut = tf.sigmoid(tf.matmul(y3, WOut) + bOut)
# ---------------------------------------------------------------- #
# Define loss, optimizer, accuracy:
#cost = tf.reduce_mean(tf.nn.l2_loss(y_ - yOut))
cost = tf.reduce_mean(tf.square(y_ - yOut))
train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost)
acc, acc_op = tf.metrics.accuracy(labels=tf.round(y_), predictions=tf.round(yOut))
# ---------------------------------------------------------------- #
# Session Initialization:
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()
# ---------------------------------------------------------------- #
# Prepare Data:
datasetFeatures, datasetTargets = self.getTrainingSet()
batchesLeft = int(len(datasetTargets)/batch_size)
# ---------------------------------------------------------------- #
# Train:
i = 0
k = 0
while True:
if batchesLeft > 0:
if i % batch_size == 0:
batch_x, batch_y, batchesLeft = self.getNextBatch(batch_size, i, batchesLeft,datasetFeatures,datasetTargets)
out_batch, _ = sess.run((yOut, train_step), feed_dict={x_: batch_x, y_: batch_y})
else:
batchesLeft = int(len(datasetTargets)/batch_size)
i = -1
# Print Test:
if k % 1000 == 0:
v = sess.run([acc, acc_op], feed_dict={yOut: out_batch, y_: batch_y})
print("Accuracy:")
print(v[0])
# Condition On Exit:
if v[0] > 0.90:
break
k +=1
i +=1
print("Exited")
# ---------------------------------------------------------------- #
# Return Trained Neural Network
self.nnTrained = NeuralNetwork()
self.nnTrained.W1 = sess.run(W1)
self.nnTrained.W2 = sess.run(W2)
self.nnTrained.b1 = sess.run(b1)
self.nnTrained.b2 = sess.run(b2)
self.returnTrainedNN()
def __init__(self,
n_in=784,
n_out=10,
hidden_layers_sizes=[25, 25],
epsilon=0.25,
_batch_size=280,
finetuneLR=0.01):
'''
:param n_in: int, the dimension of input
:param n_out: int, the dimension of output
:param hidden_layers_sizes: list or tuple, the number of hidden neurons, the last item will be the number of hidden neurons in the last hidden layer
:param epsilon: privacy budget epsilon
:param _batch_size: the batch size
:param finetuneLR: fine tunning learning rate
'''
# Number of layers
assert len(hidden_layers_sizes) > 0
self.n_layers = len(hidden_layers_sizes)
self.layers = [] # hidden layers
self.params = [] # keep track of params for training
self.last_n_in = hidden_layers_sizes[
-1] # the number of hidden neurons in the last hidden layer
self.pretrain_ops = []
# list of pretrain objective functions for hidden layers
self.epsilon = epsilon
# privacy budget epsilon epsilon
self.batch_size = _batch_size
# batch size
# Define the input, output, Laplace noise for the output layer
self.x = tf.placeholder(tf.float32, shape=[None, n_in], name='x')
self.y = tf.placeholder(tf.float32, shape=[None, n_out])
self.LaplaceNoise = tf.placeholder(tf.float32, self.last_n_in)
######
#############################
##Construct the Model########
#############################
# Create the 1st auto-encoder layer
Auto_Layer1 = Autoencoder(inpt=self.x,
n_in=784,
n_out=hidden_layers_sizes[0],
activation=tf.nn.sigmoid)
self.layers.append(Auto_Layer1)
self.params.extend(Auto_Layer1.params)
# get the pretrain objective function
self.pretrain_ops.append(
Auto_Layer1.get_dp_train_ops(epsilon=self.epsilon,
data_size=50000,
learning_rate=0.01))
###
# Create the 2nd cauto-encoder layer
Auto_Layer2 = Autoencoder(inpt=self.layers[-1].output,
n_in=self.layers[-1].n_out,
n_out=hidden_layers_sizes[1],
activation=tf.nn.sigmoid)
self.layers.append(Auto_Layer2)
self.params.extend(Auto_Layer2.params)
# get the pretrain objective function
self.pretrain_ops.append(
Auto_Layer2.get_dp_train_ops(epsilon=self.epsilon,
data_size=50000,
learning_rate=0.01))
###
# Create the flat connected hidden layer
flat1 = HiddenLayer(inpt=self.layers[-1].output,
n_in=self.layers[-1].n_out,
n_out=self.last_n_in,
activation=tf.nn.relu)
self.layers.append(flat1)
self.params.extend(flat1.params)
###
# Create the output layer
# We use the differentially private Logistic Regression (dpLogisticRegression) layer as the objective function
self.output_layer = dpLogisticRegression(
inpt=self.layers[-1].output,
n_in=self.last_n_in,
n_out=n_out,
LaplaceNoise=self.LaplaceNoise)
# We can also use the non-differentially private layer: LogisticRegression(inpt=self.layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_out)
self.params.extend(self.output_layer.params)
###
#######################################
##Define Fine Tune Cost and Optimizer##
#######################################
# The finetuning cost
self.cost = self.output_layer.cost(self.y)
# train_op for finetuning with AdamOptimizer
global_step = tf.Variable(0, trainable=False)
#learning_rate = tf.train.exponential_decay(finetuneLR, global_step, 700, 0.96, staircase=True); # learning rate decay can be carefully used
# Fine tune with AdamOptimizer. Note that we do not fine tune the pre-trained parameters at the auto-encoder layers
self.train_op = tf.train.AdamOptimizer(finetuneLR).minimize(
self.cost,
var_list=[flat1.params, self.output_layer.params],
global_step=global_step)
# The accuracy
self.accuracy = self.output_layer.accuarcy(self.y)
def bias_variable(shape):
return tf.Variable(tf.constant(0.05, shape=shape))
