def update_data_flag(
input_flag,
train_dir = "",
test_dir = "",
opt = "",
output_dim = 2,
mode = "train"):
if opt == "mnist" or opt == "MNIST":
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
input_flag.trX, input_flag.trY, input_flag.teX, input_flag.teY = \
mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
else:
if mode == "train":
train_data = np.loadtxt(open(train_dir,"rb"), delimiter=",", dtype=float)
input_flag.trX, input_flag.trY = train_data[:, :-1], train_data[:, -1]
input_flag.trY.astype(int)
temp_tr = np.zeros((len(input_flag.trY), output_dim))
for i in range(len(input_flag.trY)):
temp_tr[i, input_flag.trY[i]] = 1
input_flag.trY = temp_tr
input_flag.teX, input_flag.teY = input_flag.trX, input_flag.trY
elif mode == "test":
test_data = np.loadtxt(open(test_dir,"rb"), delimiter=",", dtype=float)
input_flag.teX, input_flag.teY = test_data[:, :-1], test_data[:, -1]
input_flag.teY.astype(int)
temp_te = np.zeros((len(input_flag.teY), output_dim))
for i in range(len(input_flag.teY)):
temp_te[i, input_flag.teY[i]] = 1
input_flag.teY = temp_te
print input_flag.teY
input_flag.input_dim = np.size(input_flag.teX, 1)
def run_mlp(input_weight = 784, num_output = 10, hidden_weight = [12], lr = 0.001):
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
X = tf.placeholder("float", [None, input_weight])
Y = tf.placeholder("float", [None, num_output])
w_hs = []
w_hs.append(init_weights([input_weight, hidden_weight[0]]))
for i in xrange(len(hidden_weight)-1):
w_hs.append(init_weights([hidden_weight[i], hidden_weight[i+1]]))
w_o = init_weights([hidden_weight[-1], num_output])
py_x = model(X, w_hs, w_o)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(py_x, Y)) # compute costs
# construct an optimizer, choice of learning rate
train_op = tf.train.RMSPropOptimizer(lr, 0.9).minimize(cost)
predict_op = tf.argmax(py_x, 1)
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
for i in range(num_output):
for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):
sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end]})
print i, np.mean(np.argmax(teY, axis=1) ==
sess.run(predict_op, feed_dict={X: teX, Y: teY}))
def main2():
func, costFunc, gradFunc = getCostFunction()
print 'Constructed!'
mnist = input_data.read_data_sets("/home/daiver/Downloads/MNIST_data/", one_hot=True)
data = mnist.test.images
data = normalize(data)
targets = mnist.test.labels
print 'targets shape', targets.shape
targets = targets[:, 5]
weights = np.random.random(data.shape[1] + 1).reshape((1, -1))#, dtype=np.float32)
weights = SGDMomentum(costFunc, gradFunc, weights,
0.1, 0.9, 1000, data, targets)
# for iter in xrange(1000):
#err = costFunc(weights, data, targets)
#grad = gradFunc(weights, data, targets)[0]
#print 'iter', iter, err
##print grad
#weights -= 0.05 * grad
print costFunc(weights, data, targets)
nErr = 0
for i in xrange(data.shape[0]):
res = func(weights, data[i, :])
ans = targets[i]
if i < 10: print i, res, ans
if (res >= 0.5 and ans == 0) or (res <= 0.5 and ans == 1):
nErr += 1
print 'nErr', nErr
def load_data():
mnist = input_data.read_data_sets("/tmp/data/")
print 'Reading data set complete'
def shared_dataset(data_xy, borrow=True):
'''
Function that loads the dataset into shared variables
'''
data_x, data_y = data_xy
shared_x = theano.shared(np.asarray(data_x,
dtype=theano.config.floatX),
borrow=borrow)
shared_y = theano.shared(np.asarray(data_y,
dtype=theano.config.floatX),
borrow=borrow)
return shared_x, T.cast(shared_y, 'int32')
test_set_x, test_set_y = shared_dataset((mnist.test.images,
mnist.test.labels))
train_set_x, train_set_y = shared_dataset((mnist.train.images,
mnist.train.labels))
valid_set_x, valid_set_y = shared_dataset((mnist.validation.images,
mnist.validation.labels))
rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),
(test_set_x, test_set_y)]
return rval
def main(args):
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
with tf.Session() as sess:
x = tf.placeholder("float", shape=[None, 784])
t = tf.placeholder("float", shape=[None, 10])
keep_prob = tf.placeholder("float")
y = mnistnet.inference(tf.reshape(x, [-1, 28, 28, 1]), keep_prob)
loss = mnistnet.loss(y, t)
optimizer = mnistnet.train(loss)
sess.run(tf.initialize_all_variables())
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
batch_size = 50
for i in range(100):
start_time = time.time()
N = 50000
for j in range(0, N, batch_size):
images, labels = mnist.train.next_batch(batch_size)
_ = sess.run(fetches=optimizer, feed_dict={x: images, t: labels, keep_prob: 0.5})
duration = time.time() - start_time
examples_per_sec = N / duration
train_acc = accuracy.eval(feed_dict={x: images, t: labels, keep_prob: 1.0})
test_acc = accuracy.eval(feed_dict={x: mnist.test.images, t: mnist.test.labels, keep_prob: 1.0})
print "[%d]\ttrain-accuracy:%.5f\ttest-accuracy:%.5f(%.1f examples/sec)" % (i, train_acc, test_acc, examples_per_sec)
print "Accuracy: %.3f" % accuracy.eval(feed_dict={x: mnist.test.images, t: mnist.test.labels, keep_prob: 1.0})
def train():
mnist = input_data.read_data_sets("/home/daiver/Downloads/MNIST_data/", one_hot=True)
print (mnist.train.next_batch(100)[0].shape)
print (mnist.train.next_batch(100)[1].shape)
thetaShape = 784 + 1
thetas = []
for labelInd in xrange(10):
theta = np.random.rand(thetaShape) * 0.01
#print mnist.num_examples
batch = mnist.train.next_batch(55000)
X = batch[0]
X = normalize(X)
Y = batch[1][:, labelInd]
J = getJ(X, Y)
gradJ = getGrad(X, Y)
res = nonLinCG(J, gradJ, theta, 200)
#res = gradDescent(J, gradJ, theta, 2000)
theta = res
#res = minimize(J, np.random.rand(thetaShape),
##method='Newton-CG',
##method='bfgs',
#method='nelder-mead',
#options={'disp': True})
#print 'theta', res
print labelInd, J(res)
thetas.append(theta)
np.save("mnist_class4.dump", np.array(thetas))
def train():
mnist = input_data.read_data_sets("Mnist_data/", one_hot=True)
x = tf.placeholder("float", shape=[None, 784]) # 784 = 28*28
y = tf.placeholder("float", shape=[None, 10]) # 正确的label
drop_pro = tf.placeholder("float")
images = tf.reshape(x, shape=[BATCH_SIZE, 28, 28, 1])
logits = net(images, drop_pro)
getAccuracy = get_accuracy(logits, y)
cross_entropy = -tf.reduce_sum(y * tf.log(logits))
global_step = tf.Variable(0, name="global_step")
train_op = tf.train.GradientDescentOptimizer(0.001) \
.minimize(cross_entropy, global_step=global_step)
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
for i in range(3000):
batch = mnist.train.next_batch(BATCH_SIZE)
if i % 100 == 0:
accuracy = sess.run(getAccuracy, feed_dict={x: batch[0]
, y: batch[1], drop_pro: 1})
print("----step:%d-----accuracy:%g" % (i, accuracy))
sess.run(train_op, feed_dict={x: batch[0]
, y: batch[1], drop_pro: 0.5})
def run_train_loops():
#read training dataset
data_sets = input_data.read_data_sets(FLAGS.train_dir,False)
with tf.Graph().as_default():
images_placeholder,labels_placeholder = placeholder_inputs(FLAGS.batch_size)
logits = Inference(images_placeholder)
loss = lossFun(logits,labels_placeholder)
train_op = train_op_Fun(loss)
eval_correct = evaluation(logits,labels_placeholder)
#session & Initialization
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
#train loops
for step in xrange(FLAGS.max_steps):
start_time = time.time()
images_feed,labels_feed = data_sets.train.next_batch(FLAGS.batch_size,False)
_,loss_value = sess.run([train_op,loss],feed_dict={images_placeholder:images_feed,labels_placeholder:labels_feed})
duration = time.time() - start_time
#output loss every 100 steps
if step%100 == 0:
print('Traing Data Eval:')
do_eval(sess,eval_correct,images_placeholder,labels_placeholder,data_sets.train)
print('Test Data Eval:')
do_eval(sess,eval_correct,images_placeholder,labels_placeholder,data_sets.test)
def run_training():
"""Train MNIST for a number of steps."""
data_sets = input_data.read_data_sets(FLAGS.train_dir, FLAGS.fake_data)
with tf.Graph().as_default():
images_placeholder, labels_placeholder = placeholder_inputs(FLAGS.batch_size)
logits = mnist.inference(images_placeholder, FLAGS.hidden1, FLAGS.hidden2)
loss = mnist.calculate_loss(logits, labels_placeholder)
train_op = mnist.training(loss, FLAGS.learning_rate)
eval_correct = mnist.evaluation(logits, labels_placeholder)
summary_op = tf.merge_all_summaries() # Collect all summaries generated by the default graph
saver = tf.train.Saver() # Create a saver for writing training checkpoints.
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir,
graph_def=sess.graph_def)
# Training loop
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(data_sets.train,
images_placeholder,
labels_placeholder)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op.
_, loss_value = sess.run([train_op, loss],
feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
saver.save(sess, FLAGS.train_dir, global_step=step)
print('Training Data Evaluation:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.train)
print('Validation Data Evaluation:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.validation)
print('Test Data Evaluation:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_sets.test)
def run_mlp(input_weight=[10]):
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
X = tf.placeholder("float", [None, 784])
Y = tf.placeholder("float", [None, 10])
w_hs = []
w_hs.append(init_weights([784, input_weight[0]]))
for i in xrange(len(input_weight) - 1):
w_hs.append(init_weights([input_weight[i], input_weight[i + 1]]))
w_o = init_weights([input_weight[-1], 10])
py_x = model(X, w_hs, w_o)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(py_x, Y)) # compute costs
train_op = tf.train.GradientDescentOptimizer(0.05).minimize(cost) # construct an optimizer
predict_op = tf.argmax(py_x, 1)
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
for i in range(100):
for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):
sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end]})
print i, np.mean(np.argmax(teY, axis=1) == sess.run(predict_op, feed_dict={X: teX, Y: teY}))
def main():
# 画像データのダウンロード
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
sess = tf.InteractiveSession()
x = tf.placeholder("float", shape=[None, 784])
y_ = tf.placeholder("float", shape=[None, 10])
# 第１階層
W_conv1 = weight_variable([5, 5, 1, 32])
b_conv1 = bias_variable([32])
# ベクトルの変形
x_image = tf.reshape(x, [-1, 28, 28, 1])
# 第１階層のモデル作成
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
# 第２階層
W_conv2 = weight_variable([5, 5, 32, 64])
b_conv2 = bias_variable([64])
# 第２階層のモデル作成
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)
# densely connected layer
W_fc1 = weight_variable([7 * 7 * 64, 1024])
b_fc1 = bias_variable([1024])
h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
# dropout
keep_prob = tf.placeholder("float")
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# readout
W_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])
y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
cross_entropy = -tf.reduce_sum(y_ * tf.log(y_conv))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
sess.run(tf.initialize_all_variables())
for i in range(20000):
batch = mnist.train.next_batch(50)
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})
print "step %d, training accuracy %g" % (i, train_accuracy)
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
print "test accuracy %g" % accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})
def main(_):
# Import data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True, fake_data=FLAGS.fake_data)
sess = tf.InteractiveSession()
# Create the model
x = tf.placeholder(tf.float32, [None, 784], name="x-input")
W = tf.Variable(tf.zeros([784, 10]), name="weights")
b = tf.Variable(tf.zeros([10], name="bias"))
# Use a name scope to organize nodes in the graph visualizer
with tf.name_scope("Wx_b"):
y = tf.nn.softmax(tf.matmul(x, W) + b)
# Add summary ops to collect data
_ = tf.histogram_summary("weights", W)
_ = tf.histogram_summary("biases", b)
_ = tf.histogram_summary("y", y)
# Define loss and optimizer
y_ = tf.placeholder(tf.float32, [None, 10], name="y-input")
# More name scopes will clean up the graph representation
with tf.name_scope("xent"):
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
_ = tf.scalar_summary("cross entropy", cross_entropy)
with tf.name_scope("train"):
train_step = tf.train.GradientDescentOptimizer(FLAGS.learning_rate).minimize(cross_entropy)
with tf.name_scope("test"):
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
_ = tf.scalar_summary("accuracy", accuracy)
# Merge all the summaries and write them out to /tmp/mnist_logs
merged = tf.merge_all_summaries()
writer = tf.train.SummaryWriter("/tmp/mnist_logs", sess.graph_def)
tf.initialize_all_variables().run()
# Train the model, and feed in test data and record summaries every 10 steps
for i in range(FLAGS.max_steps):
if i % 10 == 0: # Record summary data and the accuracy
if FLAGS.fake_data:
batch_xs, batch_ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data)
feed = {x: batch_xs, y_: batch_ys}
else:
feed = {x: mnist.test.images, y_: mnist.test.labels}
result = sess.run([merged, accuracy], feed_dict=feed)
summary_str = result[0]
acc = result[1]
writer.add_summary(summary_str, i)
print("Accuracy at step %s: %s" % (i, acc))
else:
batch_xs, batch_ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data)
feed = {x: batch_xs, y_: batch_ys}
sess.run(train_step, feed_dict=feed)
def main():
data = input_data.read_data_sets()
num_features = data.train.features.shape[1]
# Graph input
x = tf.placeholder("float", [None,num_features])
y = tf.placeholder("float", [None, 1])
# Model
W = tf.Variable(tf.zeros([num_features,1]))
b = tf.Variable(tf.zeros([1]))
y_pred = tf.nn.sigmoid(tf.matmul(x, W) + b)
# Helper functions
correct_prediction = tf.equal(tf.round(y_pred), y)
accuracy = tf.reduce_mean(tf.cast(correct_prediction,"float"))
# Optimization
cost_func = tf.reduce_sum( tf.nn.sigmoid_cross_entropy_with_logits(y_pred, y) )
train_step = tf.train.MomentumOptimizer(0.003,.5).minimize(cost_func)
# Initialize
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
# Learn
for epoch in range(2501):
batch_xs, batch_ys = data.train.next_batch(100)
sess.run(train_step, feed_dict={x: batch_xs, y: batch_ys})
if epoch % 500 == 0:
# print some outputs
train_cost = sess.run(cost_func,feed_dict={x: data.train.features, y: data.train.targets}) / data.train.num_examples
valid_cost = sess.run(cost_func,feed_dict={x: data.validation.features, y: data.validation.targets}) / data.validation.num_examples
train_acc = sess.run( accuracy, feed_dict={x: data.train.features, y: data.train.targets})
valid_acc = sess.run( accuracy, feed_dict={x: data.validation.features, y: data.validation.targets})
print "Epoch: %s, Train/Valid Cost: %.3f/%.3f, Train/Valid Accuracy: %.3f/%.3f" % (epoch, train_cost, valid_cost, train_acc, valid_acc)
#train_pred = sess.run(y_pred, feed_dict={x: data.train.features, y: data.train.targets})
#valid_pred = sess.run(y_pred, feed_dict={x: data.validation.features, y: data.validation.targets})
test_pred = sess.run(y_pred, feed_dict={x: data.test.features, y: data.test.targets})
print "Weights: %s" % sess.run(W)
print "Intercept: %s" % sess.run(b)
sess.close()
# Write data to file
test_pred = int(np.round(test_pred))
df = pd.DataFrame({'PassengerId' : data.test.ids[:,0], 'Survived' : test_pred[:,0]})
df.to_csv('data/tensorflow_benchmark.csv', index=False)
def get_input_tensor():
mnist = input_data.read_data_sets("MNIST_data/")
images = np.concatenate((mnist.train.images, mnist.validation.images,
mnist.test.images))
images = np.reshape(images, [-1, 28, 28, 1])
dataset = tf.data.Dataset.from_tensor_slices(images)
dataset = dataset.batch(BATCH_SIZE).shuffle(10000).repeat()
one_element = dataset.make_one_shot_iterator().get_next()
return one_element
def main():
# Load MNIST dataset
mnist = input_data.read_data_sets('mnist_data/', one_hot=True)
# Create TensorFlow Session
sess = tf.InteractiveSession()
# Placeholder for data, input and label, None will be defined later by
# the batch size
x = tf.placeholder('float', shape=[None, 784])
y_ = tf.placeholder('float', shape=[None, 10])
# Define variables for the regression
w = tf.Variable(tf.zeros([784, 10]))
w_hist = tf.histogram_summary('weights', w)
b = tf.Variable(tf.zeros([10]))
b_hist = tf.histogram_summary('bias', b)
# Write regression function
y = tf.nn.softmax(tf.matmul(x, w) + b)
# Write cost function
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
# Train Steps
train_step = tf.train.GradientDescentOptimizer(
0.01).minimize(cross_entropy)
# Initialize TensorBoard
merged_summary_op = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter('/Users/royxue/Code/TensorFlow/tensorflow-demos/mnist/logs', sess.graph_def)
total_step = 0
# Intialize the viriables, then they can be used by session
sess.run(tf.initialize_all_variables())
# Train data with batches
for i in range(1000):
total_step += 1
batch_x, batch_y = mnist.train.next_batch(50)
feed = {x: batch_x, y_: batch_y}
sess.run(train_step, feed_dict=feed)
if total_step % 100 == 0:
summary_str = sess.run(merged_summary_op, feed_dict=feed)
summary_writer.add_summary(summary_str, total_step)
# Validation results
correct_predict = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_predict, 'float'))
# Finally output
print accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels})
def main():
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
graph = tf.Graph()
with graph.as_default():
x = tf.placeholder("float", shape=[None, 784])
y_ = tf.placeholder("float", shape=[None, 10])
full_layer = FullLayer(x=x, name="full", nodes=750, debug=True)
model = reduce(
lambda a, b: b(x=a.y),
[full_layer,
partial(DropoutLayer, name="dropout"),
partial(LogReg, name="logreg", debug=True)])
loss, train_op = training(model.y, x, y_)
tf.scalar_summary("loss", loss)
accuracy = evaluate(model.y, y_)
init = tf.initialize_all_variables()
summary_op = tf.merge_all_summaries()
session = tf.Session()
session.run(init)
summary_writer = tf.train.SummaryWriter(
"/tmp/tensorflow/{}".format(datetime.now().isoformat()),
graph_def=session.graph_def)
try:
for i in xrange(20000):
x_batch, y_batch = mnist.train.next_batch(50)
train_op.run(
feed_dict={x: x_batch, y_: y_batch}, session=session)
if i % 250 == 0:
feed_dict = {x: mnist.train.images, y_: mnist.train.labels}
loss_, accuracy_, summary_str, _ = session.run(
[loss, accuracy, summary_op, full_layer.max_W],
feed_dict=feed_dict)
print "Batch:{:4d} loss:{:3.7f}, accuracy:{:1.3f}".format(
i, loss_, accuracy_)
sys.stdout.flush()
summary_writer.add_summary(summary_str, i)
finally:
result = accuracy.eval(
session=session,
feed_dict={x: mnist.test.images, y_:mnist.test.labels})
print ""
print("Model accuracy: {}".format(result))
def main(sigma, sample_path='samples.npy'):
# provide a .npy file where 10k generated samples are saved.
filename = sample_path
print 'loading samples from %s'%filename
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
test_X, test_Y = mnist.test.next_batch(mnist.test.num_examples)
samples = numpy.load(filename)
test_ll = numpy_parzen(test_X, samples, sigma)
print "Mean Log-Likelihood of test set = %.5f" % numpy.mean(test_ll)
print "Std of Mean Log-Likelihood of test set = %.5f" % (numpy.std(test_ll) / 100)
def train_and_save_model(self):
# Our training data
mnist = input_data.read_data_sets('../data/MNIST_digits', one_hot=True)
for i in range(20000):
batch = mnist.train.next_batch(50)
if i%100 == 0:
train_accuracy = self.accuracy.eval(feed_dict={
self.x:batch[0], self.y_: batch[1], self.keep_prob: 1.0
})
print("step %d, training accuracy %g"%(i, train_accuracy))
self.train_step.run(feed_dict={self.x: batch[0], self.y_: batch[1], self.keep_prob: 0.5})
# Saves path
save_path = saver.save(sess, "../trained_models/mnist_digits.ckpt")
print("Model saved in file: ", save_path)
def run_training():
data_sets = input_data.read_data_sets('MNIST_data', FLAGS.fake_data)
with tf.Graph().as_default():
images_placeholder, labels_placeholder = placeholder_inputs(FLAGS.batch_size)
logits = mnist.inference(images_placeholder, FLAGS.hidden1, FLAGS.hidden2)
loss = mnist.loss(logits, labels_placeholder)
train_op = mnist.training(loss, FLAGS.learning_rate)
eval_correct = mnist.evaluation(logits, labels_placeholder)
summary_op = tf.merge_all_summaries()
saver = tf.train.Saver()
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, graph_def=sess.graph_def)
for step in xrange(FLAGS.max_steps):
start_time = time.time()
feed_dict = fill_feed_dict(data_sets.train, images_placeholder, labels_placeholder)
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
duration = time.time() - start_time
if step % 100 == 0:
print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
saver.save(sess, FLAGS.train_dir, global_step=step)
print('Training Data Eval:')
do_eval(sess, eval_correct, images_placeholder, labels_placeholder, data_sets.train)
print('Validation Data Eval:')
do_eval(sess, eval_correct, images_placeholder, labels_placeholder, data_sets.validation)
print('Test Data Eval:')
do_eval(sess, eval_correct, images_placeholder, labels_placeholder, data_sets.test)
def main(_):
mnist = input_data.read_data_sets('MNIST_data', one_hot=True, fake_data=FLAGS.fake_data)
sess = tf.InteractiveSession()
x = tf.placeholder(tf.float32, [None, 784], name='x-input')
W = tf.Variable(tf.zeros([784, 10]), name='weights')
b = tf.Variable(tf.zeros([10]), name='bias')
with tf.name_scope('Wx_b'):
y = tf.nn.softmax(tf.matmul(x, W) + b)
tf.histogram_summary('weights', W)
tf.histogram_summary('biases', b)
tf.histogram_summary('y', y)
y_ = tf.placeholder(tf.float32, [None, 10], name='y-input')
with tf.name_scope('xent'):
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
tf.scalar_summary('cross entropy', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.GradientDescentOptimizer(
FLAGS.learning_rate).minimize(cross_entropy)
with tf.name_scope('test'):
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.scalar_summary('accuracy', accuracy)
merged = tf.merge_all_summaries()
writer = tf.train.SummaryWriter('MNIST_logs', sess.graph_def)
tf.initialize_all_variables().run()
for i in range(FLAGS.max_steps):
if i % 10 == 0:
feed = {x: mnist.test.images, y_: mnist.test.labels}
summary_str, acc = sess.run([merged, accuracy], feed_dict=feed)
writer.add_summary(summary_str, i)
print('Accuracy at step %s: %s' % (i, acc))
else:
batch_xs, batch_ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data)
feed = {x: batch_xs, y_: batch_ys}
sess.run(train_step, feed_dict=feed)
def test():
mnist = input_data.read_data_sets("/home/daiver/Downloads/MNIST_data/", one_hot=True)
X = mnist.test.images
X = normalize(X)
Y = mnist.test.labels
#thetas = np.load("./mnist_class.dump.npy")
thetas = np.load("./mnist_class4.dump.npy")
print thetas.shape
nSamples = 10000
err = 0
for i in xrange(nSamples):
ansInd = np.argmax(Y[i])
x = X[i]
values = np.array([logFN(t, x) for t in thetas])
res = np.argmax(values)
if res != ansInd:
err += 1
print 1.0 - float(err)/nSamples, err, nSamples
def main():
# Load data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
# Start TensorFLow session
sess = tf.InteractiveSession()
# Define input / output
x = tf.placeholder("float", shape=[None, 784])
y_ = tf.placeholder("float", shape=[None, 10])
# Infere
y_conv = inference(x, y_)
# Define cost function
cost_func = loss(y_, y_conv)
# Train
train_step = train(cost_func)
# Evaluate
accuracy = evaluate(y_, y_conv)
# Start session
sess.run(tf.initialize_all_variables())
# Training loop
for i in range(20000):
# Train over 50 training examples per iteration
batch = mnist.train.next_batch(50)
# Display progress
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={x: batch[0],
y_: batch[1],
keep_prob: 1.0})
print("step %d, training accuracy %g" % (i, train_accuracy))
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
# Show final results
print("test accuracy %g" % accuracy.eval(feed_dict={x: mnist.test.images,
y_: mnist.test.labels,
keep_prob: 1.0}))
def main(_):
# Import data
mnist = input_data.read_data_sets(FLAGS.data_dir, fake_data=True, one_hot=True, seed=1)
# Create the model
x = tf.placeholder(tf.float32, [None, 784])
# Define loss and optimizer
y_ = tf.placeholder(tf.float32, [None, 10])
# Build the graph for the deep net
y_conv, keep_prob = deepnn(x)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=y_conv)
cross_entropy = tf.reduce_mean(cross_entropy)
with tf.name_scope('adam_optimizer'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
graph_location = tempfile.mkdtemp()
print('Saving graph to: %s' % graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(200):
batch = mnist.train.next_batch(50, fake_data=True, shuffle=False)
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch[0], y_: batch[1], keep_prob: 1.0})
print('step %d, training accuracy %g' % (i, train_accuracy))
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
print('test accuracy %g' % accuracy.eval(feed_dict={
x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}))
def train(self):
train_loss_d = train_loss_g = train_loss_vae = 0.5
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
n_samples = mnist.train.num_examples
for epoch in range(10):
for idx in range(int(n_samples / self.batchsize)):
# z = np.random.uniform(-1.0,1.0, size=(self.batchsize, self.zdim)).astype(np.float32)
batch = mnist.train.next_batch(self.batchsize)[0].reshape([self.batchsize,self.imgdim,self.imgdim,1])
for i in xrange(4):
train_loss_g, _ = self.sess.run([self.g_loss, self.g_optim],feed_dict={self.images: batch})
train_loss_d, _ = self.sess.run([self.d_loss, self.d_optim],feed_dict={self.images: batch})
train_loss_vae, _ = self.sess.run([self.vae_loss, self.vae_optim],feed_dict={self.images: batch})
print "%d: %f %f %f" % (idx, train_loss_d, train_loss_g, train_loss_vae)
if idx % 15 == 0:
generated_test = self.sess.run(self.G, feed_dict={self.images: batch})
generated_test = generated_test.reshape(self.batchsize,28,28)
ims("results/"+str(idx + epoch*100000)+".jpg",merge(generated_test[:64],[8,8]))
def main():
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
x = tf.placeholder(tf.float32, [None, 784])
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
y = tf.nn.softmax(tf.matmul(x, W)+b)
y_ = tf.placeholder(tf.float32, [None, 10])
cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.GradientDescentOptimizer(
0.01).minimize(cross_entropy)
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
for i in range(1000):
batch_xs, batch_ys = mnist.train.next_batch(100)
sess.run(train_step, feed_dict={x: batch_xs, y_:batch_ys})
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
print(sess.run(accuracy, feed_dict={
x: mnist.test.images, y_: mnist.test.labels}))
def main(epochs=10000, batch_size=150, learn_rate=0.01):
data = input_data.read_data_sets('images')
# Placeholders
x = tf.placeholder("float",[None, 128*128])
y = tf.placeholder("float", [None, 1])
# Variables
W = tf.Variable( tf.zeros([128*128,1]) )
b = tf.Variable( tf.zeros([1]) )
y_pred = tf.nn.sigmoid(tf.matmul(x,W) + b)
# helper functions
correct_prediction = tf.equal(tf.round(y_pred), y)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# Optimization
cost = tf.reduce_sum( tf.nn.sigmoid_cross_entropy_with_logits(y_pred, y))
#train_step = tf.train.GradientDescentOptimizer(learn_rate).minimize(cost)
train_step = tf.train.MomentumOptimizer(.003, .5).minimize(cost)
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
# Initialize
for epoch in range(epochs):
batch_xs, batch_ys = data.train.next_batch(batch_size)
sess.run(train_step, feed_dict={x: batch_xs, y: batch_ys})
if epoch % 100 == 0:
train_cost = sess.run(cost, feed_dict={x: data.train.images, y: data.train.labels}) / data.train.num_examples
valid_cost = sess.run(cost, feed_dict={x: data.validation.images, y: data.validation.labels}) / data.validation.num_examples
train_acc = sess.run(accuracy, feed_dict={x: data.train.images, y: data.train.labels})
valid_acc = sess.run(accuracy, feed_dict={x: data.validation.images, y: data.validation.labels})
print "epoch %s, train/valid cost: %.3f/%.3f, train/valid accuracy %.3f/%.3f" % (epoch, train_cost, valid_cost, train_acc, valid_acc)
sess.close()
# 输入数据
import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
import tensorflow as tf
# 定义网络超参数
learning_rate = 0.001
training_iters = 200000
batch_size = 64
display_step = 20
# 定义网络参数
n_input = 784 # 输入的维度
n_classes = 10 # 标签的维度
dropout = 0.8 # Dropout 的概率
# 占位符输入
x = tf.placeholder(tf.types.float32, [None, n_input])
y = tf.placeholder(tf.types.float32, [None, n_classes])
keep_prob = tf.placeholder(tf.types.float32)
# 卷积操作
def conv2d(name, l_input, w, b):
return tf.nn.relu(tf.nn.bias_add(
tf.nn.conv2d(l_input, w, strides=[1, 1, 1, 1], padding='SAME'), b),
name=name)
# 最大下采样操作
import input_data
import tensorflow as tf
mnist = input_data.read_data_sets("E:\\Python\\tensorflow\\tmp\\mnist", one_hot=True)
x = tf.placeholder("float", [None, 784])
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
y = tf.nn.softmax(tf.matmul(x, W) + b)
y_ = tf.placeholder("float", [None, 10])
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
for i in range(1000):
batch_xs, batch_ys = mnist.train.next_batch(100)
sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}))
def train():
# Import data
# mnist = input_data.read_data_sets(FLAGS.data_dir,
# one_hot=True,
# fake_data=FLAGS.fake_data)
mnist = input_data.read_data_sets(FLAGS.train_images_file,
FLAGS.train_labels_file,
FLAGS.test_images_file,
FLAGS.test_labels_file,
one_hot=True)
sess = tf.InteractiveSession()
# Create a multilayer model.
# Input placeholders
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, 784], name='x-input')
y_ = tf.placeholder(tf.float32, [None, 10], name='y-input')
with tf.name_scope('input_reshape'):
image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])
tf.summary.image('input', image_shaped_input, 10)
# We can't initialize these variables to 0 - the network will get stuck.
def weight_variable(shape):
"""Create a weight variable with appropriate initialization."""
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
"""Create a bias variable with appropriate initialization."""
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def variable_summaries(var):
"""Attach a lot of summaries to a Tensor (for TensorBoard visualization)."""
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
tf.summary.scalar('max', tf.reduce_max(var))
tf.summary.scalar('min', tf.reduce_min(var))
tf.summary.histogram('histogram', var)
def nn_layer(input_tensor,
input_dim,
output_dim,
layer_name,
act=tf.nn.relu):
"""Reusable code for making a simple neural net layer.
It does a matrix multiply, bias add, and then uses relu to nonlinearize.
It also sets up name scoping so that the resultant graph is easy to read,
and adds a number of summary ops.
"""
# Adding a name scope ensures logical grouping of the layers in the graph.
with tf.name_scope(layer_name):
# This Variable will hold the state of the weights for the layer
with tf.name_scope('weights'):
weights = weight_variable([input_dim, output_dim])
variable_summaries(weights)
with tf.name_scope('biases'):
biases = bias_variable([output_dim])
variable_summaries(biases)
with tf.name_scope('Wx_plus_b'):
preactivate = tf.matmul(input_tensor, weights) + biases
tf.summary.histogram('pre_activations', preactivate)
activations = act(preactivate, name='activation')
tf.summary.histogram('activations', activations)
return activations
hidden1 = nn_layer(x, 784, 500, 'layer1')
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
tf.summary.scalar('dropout_keep_probability', keep_prob)
dropped = tf.nn.dropout(hidden1, keep_prob)
# Do not apply softmax activation yet, see below.
y = nn_layer(dropped, 500, 10, 'layer2', act=tf.identity)
with tf.name_scope('cross_entropy'):
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the
# raw outputs of the nn_layer above, and then average across
# the batch.
diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
with tf.name_scope('total'):
cross_entropy = tf.reduce_mean(diff)
tf.summary.scalar('cross_entropy', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(
FLAGS.learning_rate).minimize(cross_entropy)
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# with tf.name_scope('loss'):
# # loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=mnist.test.images, labels=mnist.test.labels))
# loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_, labels=y))
# # loss = 10.5
# tf.summary.scalar('loss', loss)
# Merge all the summaries and write them out to /tmp/tensorflow/mnist/logs/tb (by default)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.tb_log_dir + '/train',
sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.tb_log_dir + '/test')
tf.global_variables_initializer().run()
# Train the model, and also write summaries.
# Every 10th step, measure test-set accuracy, and write test summaries
# All other steps, run train_step on training data, & add training summaries
def feed_dict(train):
"""Make a TensorFlow feed_dict: maps data onto Tensor placeholders."""
if train or FLAGS.fake_data:
xs, ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data)
k = FLAGS.dropout
else:
xs, ys = mnist.test.images, mnist.test.labels
k = 1.0
return {x: xs, y_: ys, keep_prob: k}
model_path = os.environ["RESULT_DIR"] + "/model.ckpt"
print('Calling tf.train.Saver(...)')
saver = tf.train.Saver(max_to_keep=1)
step = restore_from_checkpoint(saver, sess)
if step is None:
step = 1
for i in range(step - 1, FLAGS.max_steps):
if i % 10 == 0: # Record summaries and test-set accuracy
save_checkpoint(step, saver, sess, model_path)
summary, acc = sess.run([merged, accuracy],
feed_dict=feed_dict(False))
test_writer.add_summary(summary, i)
print('Accuracy at step %s: %s' % (i, acc))
else: # Record train set summaries, and train
if i % 100 == 99: # Record execution stats
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
# summary, _, loss_result = sess.run([merged, train_step, loss],
summary, _ = sess.run([merged, train_step],
feed_dict=feed_dict(True),
options=run_options,
run_metadata=run_metadata)
train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
train_writer.add_summary(summary, i)
print('Adding run metadata for', i)
else: # Record a summary
summary, _ = sess.run([merged, train_step],
feed_dict=feed_dict(True))
train_writer.add_summary(summary, i)
save_path = saver.save(sess, model_path)
train_writer.close()
test_writer.close()
def train():
'''
Train CNN_tiny for a number of steps.
'''
with tf.Graph().as_default():
# globalなstep数
global_step = tf.Variable(0, trainable=False)
# 教師データ
#images, labels = data_inputs.distorted_inputs(TF_RECORDS)
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
trX = trX.reshape(-1, 28, 28, 1)
teX = teX.reshape(-1, 28, 28, 1)
# create mini_batch
#datas, targets = trX.(trX, trY, BATCH_SIZE)
images = tf.placeholder(tf.float32, [None, 28, 28, 1])
labels = tf.placeholder(tf.float32, [None, 10])
keep_conv = tf.placeholder("float")
keep_hidden = tf.placeholder("float")
# graphのoutput
logits = model.inference(images, keep_conv, keep_hidden)
# loss graphのoutputとlabelを利用
#loss = model.loss(logits, labels)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits, labels))
predict_op = tf.argmax(logits, 1)
# 学習オペレーション
train_op = op.train(loss, global_step)
# saver
saver = tf.train.Saver(tf.all_variables())
# サマリー
summary_op = tf.merge_all_summaries()
# 初期化オペレーション
init_op = tf.initialize_all_variables()
# Session
sess = tf.Session(config=tf.ConfigProto(
log_device_placement=LOG_DEVICE_PLACEMENT))
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# サマリーのライターを設定
summary_writer = tf.train.SummaryWriter(TRAIN_DIR,
graph_def=sess.graph_def)
# max_stepまで繰り返し学習
for step in xrange(MAX_STEPS):
start_time = time.time()
previous_time = start_time
index = 0
for start, end in zip(range(0, len(trX), BATCH_SIZE),
range(BATCH_SIZE, len(trX), BATCH_SIZE)):
_, loss_value = sess.run(
[train_op, loss],
feed_dict={
images: trX[start:end],
labels: trY[start:end],
keep_conv: 0.8,
keep_hidden: 0.5
})
if index % 10 == 0:
end_time = time.time()
duration = end_time - previous_time
num_examples_per_step = BATCH_SIZE * 10 * (step + 1)
examples_per_sec = num_examples_per_step / duration
print(
"%s: %d[epoch]: %d[iteration]: train loss %f: %d[examples/step]: %f[examples/sec]: %f[sec/iteration]"
% (datetime.now(), step, index, loss_value,
num_examples_per_step, examples_per_sec, duration))
index += 1
assert not np.isnan(
loss_value), 'Model diverged with loss = NaN'
test_indices = np.arange(len(teX)) # Get A Test Batch
np.random.shuffle(test_indices)
test_indices = test_indices[0:5]
print "=" * 20
print teY[test_indices]
predict, cost_value = sess.run(
[predict_op, loss],
feed_dict={
images: teX[test_indices],
labels: teY[test_indices],
keep_conv: 1.0,
keep_hidden: 1.0
})
print predict
print("test loss: %f" % (cost_value))
print "=" * 20
previous_time = end_time
index += 1
assert not np.isnan(
loss_value), 'Model diverged with loss = NaN'
# 1000回ごと
if index % 100 == 0:
pass
summary_str = sess.run(summary_op,
feed_dict={
images: teX[test_indices],
labels: teY[test_indices],
keep_conv: 1.0,
keep_hidden: 1.0
})
# サマリーに書き込む
summary_writer.add_summary(summary_str, step)
if step % 1 == 0 or (step * 1) == MAX_STEPS:
checkpoint_path = TRAIN_DIR + '/model.ckpt'
saver.save(sess, checkpoint_path, global_step=step)
coord.request_stop()
coord.join(threads)
sess.close()
def main():
mnist = input_data.read_data_sets("./data/", one_hot=True)
test(mnist)
from attention import Attention
# train config
DATA_DIR = 'Data' # data directory
CLASS_NUM = 10 # number of classes
dim = 32 # number of dimensions
epochs_num = 20 # number of epochs
segments_N = 5
dict = {}
sess = tf.compat.v1.InteractiveSession()
flags = tf.compat.v1.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_string('data_dir', DATA_DIR, 'Directory for storing data')
mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
# normalization: make the value to [norm_min, norm_max]
def normalization(value, norm_max, norm_min):
value_max = np.max(value)
value_min = np.min(value)
normal_k = (norm_max - norm_min) / (value_max - value_min)
normal_value = np.trunc(norm_min + normal_k * (value - value_min))
return normal_value
# Fill the missing values
def append_equal(first_value, value):
if value.shape(0) < first_value.shape(0):
value = np.append(value, [0])
'''
This code is doing PCA on a purely numerical basis using TensorFlow library.
This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)
Author: Dietrich Klakow based on the logistic regression by Aymeric Damien
'''
#Project: https://repos.lsv.uni-saarland.de/dietrich/Neural_Networks_Implementation_and_Application/tree/master
# Import MINST data
import input_data
import numpy as np
import re
mnist = input_data.read_data_sets("../../data/mnist", one_hot=True)
import tensorflow as tf
# Parameters
learning_rate = 0.02
training_epochs = 20
batch_size = 1000
display_step = 1
# Dimension of hidden space
l = 2
# tf Graph Input
x = tf.placeholder("float", [None, 784]) # mnist data image of shape 28*28=784
y = tf.placeholder("float", [None, 10]) # 0-9 digits recognition => 10 classes
# Create model
def conv2d(x,W):
#stride [1,x_move,y_move,1]
return tf.nn.conv2d(x,W,strides=[1,1,1,1],padding='SAME')
def max_pool_2X2(x):
#stride [1,x_move,y_move,1]
return tf.nn.max_pool(x,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')
def accuracy(v_xs,v_ys):
global pred
y_pred = ses.run(pred,feed_dict={xs:v_xs,ys:v_ys,keep_prob:1})
corect_pred = tf.equal(tf.argmax(y_pred,1),tf.argmax(v_ys,1))
acc = tf.reduce_mean(tf.cast(corect_pred,tf.float32))
return ses.run(acc,feed_dict={xs:v_xs,ys:v_ys,keep_prob:0.5})
if __name__ =="__main__":
mnist = input_data.read_data_sets('data/',one_hot=True)
xs = tf.placeholder(tf.float32,[None,784]) #28*28
ys = tf.placeholder(tf.float32,[None,10])
keep_prob = tf.placeholder(tf.float32)
ximg = tf.reshape(xs,[-1,28,28,1])
#conv1_layer
W_conv1 = weight_init([5,5,1,32])
b_conv1 = bias_init([32])
h_conv1 = tf.nn.relu(conv2d(ximg,W_conv1) + b_conv1) #shape=28*28*32
h_pool1 = max_pool_2X2(h_conv1) #14*14*32
#conv2_layer
W_conv2 = weight_init([5,5,32,64])
b_conv2 = bias_init([64])
h_conv2 = tf.nn.relu(conv2d(h_pool1,W_conv2)+b_conv2)
import tensorflow as tf
import numpy as np
import input_data
mnist = input_data.read_data_sets('../MNIST_data/', one_hot=True)
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(
x, ksize=[
1, 2, 2, 1], strides=[
1, 2, 2, 1], padding='SAME')
from __future__ import print_function
# Import data
import input_data
import tensorflow as tf
import shutil
import os
out_path = '../data/model-deep'
out_fname = 'mnist-deep'
train_dir = './ckpt-deep/'
train_file = train_dir + out_fname + '.ckpt'
# Get data
mnist = input_data.read_data_sets("training_data/", one_hot=True)
def weight_variable(shape, name):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial, name=name)
def bias_variable(shape, name):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial, name=name)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def main(argv):
if len(argv) < 5:
print("Error, Syntax: {0} [train/test] [dataset] [conv dim] [conv width] [fold]".format(argv[0]))
exit()
global conv_shape
conv_shape = int(argv[4])
dataset = argv[2]
conv_dim = argv[3]
test = argv[1]
fold = int(argv[5])
ns.load_settings_raw(dataset, conv_dim, 50, 2, fold)
run_name = "raw-fc1024-lr{0}-adam-{1}conv-{2}-fold{3}".format(ns.LEARNING_RATE, conv_shape, dataset, fold) # +"-"+nowtime
print(run_name)
data_sets = input_data.read_data_sets(ns.TRAINING_FILE, ns.TRAINING_LABEL, ns.IMAGE_SHAPE, test_file=ns.TEST_FILE,
test_label=ns.TEST_LABEL, validation_ratio=0.0, pickle=False, boring=False)
train_data = data_sets.train.images # Returns np.array
train_labels = np.asarray(data_sets.train.labels, dtype=np.int32)
eval_data = data_sets.test.images # Returns np.array
eval_labels = np.asarray(data_sets.test.labels, dtype=np.int32)
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.4
# Create the Estimator
if conv_dim == "1d":
classifier = tf.estimator.Estimator(model_fn=cnn_model_1D, model_dir="models/" + run_name, config=tf.estimator.RunConfig(session_config=config))
else:
classifier = tf.estimator.Estimator(model_fn=cnn_model_2D, model_dir="models/" + run_name, config=tf.estimator.RunConfig(session_config=config))
if test == "train":
# train
# Set up logging for predictions
# Log the values in the "Softmax" tensor with label "probabilities"
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=100)
# Train the model
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": train_data},
y=train_labels,
batch_size=ns.BATCH_SIZE,
num_epochs=None,
shuffle=True)
classifier.train(
input_fn=train_input_fn,
steps=ns.NUM_ITER,
hooks=[logging_hook])
# Evaluate the model and print results
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": eval_data},
y=eval_labels,
num_epochs=1,
shuffle=False)
eval_results = classifier.evaluate(input_fn=eval_input_fn)
print(run_name)
print(eval_results)
else:
# test
# Evaluate the model and print results
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": eval_data},
y=eval_labels,
num_epochs=1,
shuffle=False)
# eval_results = classifier.evaluate(input_fn=eval_input_fn)
labels = eval_labels
predictions = list(classifier.predict(input_fn=eval_input_fn))
predicted_classes = [p["classes"] for p in predictions]
from sklearn.metrics import confusion_matrix, classification_report
print(run_name)
print(confusion_matrix(labels, predicted_classes))
print(classification_report(labels, predicted_classes))
#!/usr/bin/env python # -*- coding: utf-8 -*- """ 使用RNN """ import matplotlib.pyplot as plt import tensorflow as tf import globe import input_data # set random seed for comparing the two result calculations tf.set_random_seed(1) # get data training_data = input_data.read_data_sets() # hyper_parameters lr = 0.001 training_iters = 100000 batch_size = 100 # n_inputs = globe.n_dim # data input size，输入层神经元, 词向量的维度 embeding_size = globe.n_dim # data input size，输入层神经元 n_steps = 1 # time steps n_hidden_units = 200 # neurons in hidden layer，隐藏层神经元个数 n_classes = 2 # classes 二分类 # tf Graph input x = tf.placeholder(tf.float32, [None, n_steps, embeding_size]) y = tf.placeholder(tf.float32, [None, n_classes])
def main(argv=None):
if argv is None:
argv = sys.argv
try:
try:
opts = argv
for item, val in opts:
if (item == '-quantisation_bits'):
q_bits = val
if (item == '-parent_dir'):
parent_dir = val
if (item == '-base_name'):
base_name = val
if (item == '-c_pos'):
c_pos = val
if (item == '-c_neg'):
c_neg = val
if (item == '-central_value'):
central_value = val
except getopt.error, msg:
raise Usage(msg)
# obtain all weight masks
mask_dir = parent_dir + 'masks/' + base_name + '.pkl'
with open(mask_dir, 'rb') as f:
weights_mask = pickle.load(f)
biases_mask = {
'cov1': np.ones([20]),
'cov2': np.ones([50]),
'fc1': np.ones([500]),
'fc2': np.ones([10])
}
mnist = input_data.read_data_sets("MNIST.data/", one_hot=True)
# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_classes])
keys = ['cov1', 'cov2', 'fc1', 'fc2']
x_image = tf.reshape(x, [-1, 28, 28, 1])
weights_dir = parent_dir + 'weights/' + base_name + '.pkl'
weights_tmp, biases, dynamic_range = initialize_variables(
weights_dir, central_value, c_pos, c_neg)
weights = {}
for key in keys:
weights[key] = weights_tmp[key] * weights_mask[key]
new_weights, new_biases = compute_weights_nbits(
weights, weights_mask, biases, q_bits, dynamic_range,
central_value, c_pos, c_neg)
# Construct model
pred, pool = conv_network(x_image, new_weights, new_biases)
# Define loss and optimizer
trainer = tf.train.AdamOptimizer(learning_rate=learning_rate)
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
org_grads = trainer.compute_gradients(cost)
org_grads = [(ClipIfNotNone(grad), var) for grad, var in org_grads]
new_grads = mask_gradients(org_grads, weights_mask, new_weights,
biases_mask, new_biases)
train_step = trainer.apply_gradients(new_grads)
init = tf.initialize_all_variables()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
keys = ['cov1', 'cov2', 'fc1', 'fc2']
training_cnt = 0
train_accuracy = 0
accuracy_list = np.zeros(20)
pre_train_acc = accuracy.eval({
x: mnist.test.images,
y: mnist.test.labels
})
print("Directly before pruning, Test Accuracy:", pre_train_acc)
print(weights['cov1'].eval())
print(70 * '-')
print(new_weights['cov1'].eval())
print(70 * '-')
print('Training starts ...')
# sys.exit()
# return (pre_train_acc,0)
test_acc_save = 0
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(mnist.train.num_examples / batch_size)
# Loop over all batches
for i in range(total_batch):
# execute a pruning
batch_x, batch_y = mnist.train.next_batch(batch_size)
[_, c,
train_accuracy] = sess.run([train_step, cost, accuracy],
feed_dict={
x: batch_x,
y: batch_y
})
training_cnt = training_cnt + 1
accuracy_list = np.concatenate(
(np.array([train_accuracy]), accuracy_list[0:19]))
accuracy_mean = np.mean(accuracy_list)
if (i % 1000 == 0):
print('cost and acc:', c, accuracy_mean)
if (accuracy_mean > 0.99):
test_acc = accuracy.eval({
x: mnist.test.images,
y: mnist.test.labels
})
print('Try quantize {} frac bits, test accuracy is {}'.
format(q_bits, test_acc))
if (test_acc >= 0.9936 or test_acc > test_acc_save):
test_acc_save = test_acc
print('Training ends because accuracy is high')
with open(
parent_dir + 'weights/' + 'quanfp' +
str(q_bits) + '.pkl', 'wb') as f:
pickle.dump((
new_weights['cov1'].eval(),
new_weights['cov2'].eval(),
new_weights['fc1'].eval(),
new_weights['fc2'].eval(),
biases['cov1'].eval(),
biases['cov2'].eval(),
biases['fc1'].eval(),
biases['fc2'].eval(),
), f)
print("saving model ...")
if (test_acc >= 0.9936):
return (pre_train_acc, test_acc)
# Compute average loss
avg_cost += c / total_batch
# Display logs per epoch step
print("Epoch:", '%04d' % (epoch + 1), "cost=",
"{:.9f}".format(avg_cost))
print('Training ends because timeout, but still save the model')
if (test_acc != 0):
return (pre_train_acc, test_acc_save)
else:
with open('weights/quanfp' + str(q_bits) + '.pkl', 'wb') as f:
pickle.dump((
new_weights['cov1'].eval(),
new_weights['cov2'].eval(),
new_weights['fc1'].eval(),
new_weights['fc2'].eval(),
biases['cov1'].eval(),
biases['cov2'].eval(),
biases['fc1'].eval(),
biases['fc2'].eval(),
), f)
test_acc = accuracy.eval({
x: mnist.test.images,
y: mnist.test.labels
})
print("Test Accuracy:", test_acc)
return (pre_train_acc, test_acc)
#coding=utf-8
import input_data
import tensorflow as tf
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
#我们通过操作符号变量来描述这些可交互的操作单元，可以用下面的方式创建一个：
x = tf.placeholder("float", [None, 784])
W = tf.Variable(tf.zeros([784,10]))
b = tf.Variable(tf.zeros([10]))
y = tf.nn.softmax(tf.matmul(x,W) + b)
##保存模型
saver = tf.train.Saver()
path = "/Users/jianying.wcj/github/ml_study/src/main/tensorflow/result/"
y_ = tf.placeholder("float", [None,10])
cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
init = tf.initialize_all_variables()
#现在我们可以在一个Session里面启动我们的模型，并且初始化变量
sess = tf.Session()
sess.run(init)
#然后开始训练模型，这里我们让模型循环训练1000次！
for i in range(1000):
batch_xs, batch_ys = mnist.train.next_batch(100)
#print "x:%s"%batch_xs
#print "y:%s"%batch_ys
sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
saver.save(sess,path+"mnist.model")
import tensorflow as tf
import input_data
learning_rate = 0.01
training_epochs = 15
batch_size = 100
display_step = 1
mnist = input_data.read_data_sets("./MNIST_DATA", one_hot=True)
# tensorflow graph input
X = tf.placeholder('float',
[None, 784]) # mnist data image of shape 28 * 28 = 784
Y = tf.placeholder('float',
[None, 10]) # 0-9 digits recognition = > 10 classes
# set model weights
W1 = tf.Variable(tf.random_normal([784, 256]))
W2 = tf.Variable(tf.random_normal([256, 256]))
W3 = tf.Variable(tf.random_normal([256, 10]))
B1 = tf.Variable(tf.random_normal([256]))
B2 = tf.Variable(tf.random_normal([256]))
B3 = tf.Variable(tf.random_normal([10]))
# Construct model
L1 = tf.nn.relu(tf.add(tf.matmul(X, W1), B1))
L2 = tf.nn.relu(tf.add(tf.matmul(L1, W2),
B2)) # Hidden layer with RELU activation
hypothesis = tf.add(tf.matmul(L2, W3), B3) # No need to use softmax here
# Define loss and optimizer
import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
import tensorflow as tf
sess = tf.InteractiveSession()
x = tf.placeholder("float", shape=[None, 784])
y_ = tf.placeholder("float", shape=[None, 10])
W = tf.Variable(tf.zeros([784,10]))
b = tf.Variable(tf.zeros([10]))
sess.run(tf.global_variables_initializer())
y = tf.nn.softmax(tf.matmul(x,W) + b)
cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
for i in range(1000):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x: batch[0], y_: batch[1]})
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))
label=str(num),
color=colors[num][0],
marker=colors[num][1])
plt.legend()
plt.show()
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Test various optimization strategies')
parser.add_argument('savepath', nargs=1, type=str)
args = parser.parse_args()
print("\nPULLING UP MNIST DATA")
mnist = input_data.read_data_sets("data/", one_hot=False)
print(mnist.test.labels)
# print "\nSTARTING PCA"
# pca = decomposition.PCA(n_components=2)
# pca.fit(mnist.train.images)
#
# print "\nGENERATING PCA CODES AND RECONSTRUCTION"
# pca_codes = pca.transform(mnist.test.images)
# print pca_codes
#
# scatter(pca_codes, mnist.test.labels)
with tf.Graph().as_default():
with tf.variable_scope("autoencoder_model"):
def run_training():
"""Train for a number of steps."""
data_sets = input_data.read_data_sets(seq_file=FLAGS.seq_file,expr_file=FLAGS.expr_file,reg_names_file=FLAGS.reg_names_file)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
seq_placeholder, reg_expr_placeholder, labels_placeholder = placeholder_inputs(FLAGS.batch_size)
# Build a Graph that computes predictions from the inference model.
logits = classification_model.inference(seq_placeholder,reg_expr_placeholder,FLAGS.batch_size)
# Add to the Graph the Ops for loss calculation.
loss = classification_model.loss(logits, labels_placeholder)
# Add to the Graph the Ops that calculate and apply gradients.
train_op = classification_model.training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = classification_model.evaluation(logits, labels_placeholder)
# Build the summary Tensor based on the TF collection of Summaries.
summary = tf.summary.merge_all()
# Add the variable initializer Op.
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(FLAGS.log_dir, sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(data_sets.train,seq_placeholder,reg_expr_placeholder,labels_placeholder)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op.
_, loss_value = sess.run([train_op, loss],feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))
# Update the events file.
summary_str = sess.run(summary, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_file = os.path.join(FLAGS.log_dir, 'model.ckpt')
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
print('Training Data Eval:')
do_eval(sess,eval_correct,seq_placeholder,reg_expr_placeholder,labels_placeholder,data_sets.train)
# Evaluate against the validation set.
print('Validation Data Eval:')
do_eval(sess,eval_correct,seq_placeholder,reg_expr_placeholder,labels_placeholder,data_sets.validation)
# Evaluate against the test set.
print('Test Data Eval:')
do_eval(sess,eval_correct,seq_placeholder,reg_expr_placeholder,labels_placeholder,data_sets.test)
'''
AlexNet implementation example using TensorFlow library.
This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)
AlexNet Paper (http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf)
Author: Aymeric Damien
Project: https://github.com/aymericdamien/TensorFlow-Examples/
'''
# Import MINST data
import input_data
mnist = input_data.read_data_sets("/home/ubuntu/workspace/tmp5/data/",
one_hot=True)
import tensorflow as tf
# Parameters
learning_rate = 0.001
training_iters = 200000
batch_size = 64
display_step = 20
# Network Parameters
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
dropout = 0.8 # Dropout, probability to keep units
# tf Graph input
x = tf.placeholder(tf.types.float32, [None, n_input])
y = tf.placeholder(tf.types.float32, [None, n_classes])
keep_prob = tf.placeholder(tf.types.float32) # dropout (keep probability)
CKPT_FILE = 'inception_v1.ckpt'
init_fn = slim.assign_from_checkpoint_fn(
CKPT_FILE,
inception_except_logits, ignore_missing_vars=True)
y = tf.nn.softmax(logits)
y_ = tf.placeholder(dtype=tf.float32,shape=[None, 5])
output_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='InceptionV1/Logits')
cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy, var_list=output_vars)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
flower_photos = input_data.read_data_sets('flower_photos/')
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
init_fn(sess)
for i in range(3000):
batch_xs, batch_ys = flower_photos.train.next_batch(128)
sess.run(train_step, feed_dict={x:batch_xs,y_:batch_ys,keep_prob:0.8})
if i % 100 == 0:
test_batch_xs, test_batch_ys = flower_photos.test.next_batch(200)
test_accuracy, loss = sess.run([accuracy, cross_entropy] ,feed_dict={x:test_batch_xs,y_:test_batch_ys,keep_prob:1})
print("step %d, test accuracy %g, loss %g" % (i, test_accuracy, loss))
#计算精度
print(sess.run(accuracy,feed_dict={x:flower_photos.test.images,y_:flower_photos.test.labels,keep_prob:1}))
pred_cost = -tf.reduce_mean(tf.reduce_sum(outputs*tf.log(y), 1)) # cost used for prediction
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(outputs, 1)) # no of correct predictions
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) * tf.constant(100.0)
learning_rate = tf.Variable(starter_learning_rate, trainable=False)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
# add the updates of batch normalization statistics to train_step
bn_updates = tf.group(*bn_assigns)
with tf.control_dependencies([train_step]):
train_step = tf.group(bn_updates)
print "=== Loading Data ==="
mnist = input_data.read_data_sets("MNIST_data", n_labeled=num_labeled, one_hot=True)
saver = tf.train.Saver()
print "=== Starting Session ==="
sess = tf.Session()
i_iter = 0
ckpt = tf.train.get_checkpoint_state('checkpoints/') # get latest checkpoint (if any)
if ckpt and ckpt.model_checkpoint_path:
# if checkpoint exists, restore the parameters and set epoch_n and i_iter
saver.restore(sess, ckpt.model_checkpoint_path)
epoch_n = int(ckpt.model_checkpoint_path.split('-')[1])
i_iter = (epoch_n+1) * (num_examples/batch_size)
print "Restored Epoch ", epoch_n
import random
import input_data
import ladder_network
import tensorflow as tf
print "Loading MNIST data"
mnist = input_data.read_data_sets("MNIST_data/",
one_hot=True,
labeled_size=5000,
validation_size=5000)
print
print mnist.train_unlabeled.num_examples, "unlabeled training examples"
print mnist.train_labeled.num_examples, "labeled training examples"
print mnist.validation.num_examples, "validation examples"
print mnist.test.num_examples, "test examples"
hyperparameters = {
"learning_rate":
0.01,
"noise_level":
0.2,
"input_layer_size":
784,
"class_count":
10,
"encoder_layer_definitions": [
(100, tf.nn.relu), # first hidden layer
(50, tf.nn.relu),
(10, tf.nn.softmax) # output layer
],
def run_training():
"""Train MNIST for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
train_dataset, validation_dataset, test_dataset = input_data.read_data_sets(
FLAGS.train_dir)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
images_placeholder, labels_placeholder = placeholder_inputs(
FLAGS.batch_size)
# Build a Graph that computes predictions from the inference model.
logits = my_tensor_model.inference(images_placeholder, FLAGS.hidden1,
FLAGS.hidden2)
# Add to the Graph the Ops for loss calculation.
loss = my_tensor_model.loss(logits, labels_placeholder)
# Add to the Graph the Ops that calculate and apply gradients.
train_op = my_tensor_model.training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = my_tensor_model.evaluation(logits, labels_placeholder)
accuracy = my_tensor_model.accuracy(logits, labels_placeholder)
# accuracy_validation=my_tensor_model.accuracy_validation(logits, labels_placeholder)
# accuracy_test=my_tensor_model.accuracy_test(logits, labels_placeholder)
# Build the summary operation based on the TF collection of Summaries.
# summary_op = tf.summary.merge(loss.op.name)
Entropy_summary = tf.summary.scalar('loss', loss)
training_summary = tf.summary.scalar("training_accuracy", accuracy)
validation_summary = tf.summary.scalar("validation_accuracy", accuracy)
# Add the variable initializer Op.
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop.
for step in range(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(train_dataset, images_placeholder,
labels_placeholder)
feed_dict_validation = fill_feed_dict(validation_dataset,
images_placeholder,
labels_placeholder)
feed_dict_test = fill_feed_dict(test_dataset, images_placeholder,
labels_placeholder)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' %
(step, loss_value, duration))
# Update the events file.
loss_, accu_train, Entropy_summary_str, training_summary_str = sess.run(
[loss, accuracy, Entropy_summary, training_summary],
feed_dict=feed_dict)
summary_writer.add_summary(Entropy_summary_str, step)
summary_writer.add_summary(training_summary_str, step)
accu_validation, validation_summary_str = sess.run(
[accuracy, validation_summary],
feed_dict=feed_dict_validation)
summary_writer.add_summary(validation_summary_str, step)
summary_writer.flush()
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_file = os.path.join(FLAGS.train_dir, 'checkpoint')
saver.save(sess, checkpoint_file, global_step=step)
# Evaluate against the training set.
print('Training Data Eval:')
do_eval(sess, eval_correct, images_placeholder,
labels_placeholder, train_dataset)
# Evaluate against the validation set.
print('Validation Data Eval:')
do_eval(sess, eval_correct, images_placeholder,
labels_placeholder, validation_dataset)
# Evaluate against the test set.
print('Test Data Eval:')
do_eval(sess, eval_correct, images_placeholder,
labels_placeholder, test_dataset)
def main(argv):
if len(argv) < 9:
print(
"Error, Syntax: {0} [train/test] [dataset] [conv dim] [conv len] [input len] [input1 depth] [input2 depth] [input method] [num classes]"
.format(argv[0]))
exit()
global conv_shape
conv_shape = int(argv[4])
dataset = argv[2]
conv_dim = argv[3]
test = argv[1]
input_len = int(argv[5])
input_depth1 = int(argv[6])
input_depth2 = int(argv[7])
input_method = argv[8]
num_classes = int(argv[9])
ns.load_settings_mid(dataset, conv_dim, input_len, input_depth1,
input_depth2, input_method, num_classes)
run_name = "midfusion-fc1024-lr{0}-adam-{1}-{2}conv-{3}-{4}-{5}-all".format(
ns.LEARNING_RATE, conv_dim, conv_shape, dataset, input_method,
input_depth2) # +"-"+nowtime
print(run_name)
data_sets1 = input_data.read_data_sets(ns.TRAINING_FILE1,
ns.TRAINING_LABEL1,
ns.IMAGE_SHAPE1,
test_file=ns.TEST_FILE1,
test_label=ns.TEST_LABEL1,
validation_ratio=0.0,
pickle=False,
boring=False)
train_data1 = data_sets1.train.images # Returns np.array
train_labels = np.asarray(data_sets1.train.labels, dtype=np.int32)
eval_data1 = data_sets1.test.images # Returns np.array
eval_labels = np.asarray(data_sets1.test.labels, dtype=np.int32)
print(np.shape(train_data1))
data_sets2 = input_data.read_data_sets(ns.TRAINING_FILE2,
ns.TRAINING_LABEL2,
ns.IMAGE_SHAPE2,
test_file=ns.TEST_FILE2,
test_label=ns.TEST_LABEL2,
validation_ratio=0.0,
pickle=False,
boring=False)
train_data2 = data_sets2.train.images # Returns np.array
eval_data2 = data_sets2.test.images # Returns np.array
# print(np.reshape(eval_data[0], (50,50))[0,:])
# print(tf.Session().run(tf.reshape(eval_data[0], (50,50))[0,:]))
# print(eval_labels[0])
# exit()
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.4
# Create the Estimator
if conv_dim == "1d":
classifier = tf.estimator.Estimator(
model_fn=cnn_model_1D,
model_dir="models/" + run_name,
config=tf.estimator.RunConfig(session_config=config))
else:
classifier = tf.estimator.Estimator(
model_fn=cnn_model_2D,
model_dir="models/" + run_name,
config=tf.estimator.RunConfig(session_config=config))
if test == "train":
# train
# Set up logging for predictions
# Log the values in the "Softmax" tensor with label "probabilities"
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log,
every_n_iter=100)
# Train the model
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={
"x1": train_data1,
"x2": train_data2
},
y=train_labels,
batch_size=ns.BATCH_SIZE,
num_epochs=None,
shuffle=True)
classifier.train(input_fn=train_input_fn,
steps=ns.NUM_ITER,
hooks=[logging_hook])
# Evaluate the model and print results
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={
"x1": eval_data1,
"x2": eval_data2
},
y=eval_labels,
num_epochs=1,
shuffle=False)
eval_results = classifier.evaluate(input_fn=eval_input_fn)
print(run_name)
print(eval_results)
np.savetxt("output/" + run_name + "-" + str(eval_results["accuracy"]),
[eval_results["accuracy"]])
else:
# test
# Evaluate the model and print results
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={
"x1": eval_data1,
"x2": eval_data2
},
y=eval_labels,
num_epochs=1,
shuffle=False)
# eval_results = classifier.evaluate(input_fn=eval_input_fn)
labels = eval_labels
predictions = list(classifier.predict(input_fn=eval_input_fn))
predicted_classes = [p["classes"] for p in predictions]
from sklearn.metrics import confusion_matrix, classification_report
print(run_name)
print(confusion_matrix(labels, predicted_classes))
print(classification_report(labels, predicted_classes))
def main(_=None):
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
# create the cluster configured by `ps_hosts' and 'worker_hosts'
cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})
# create a server for local task
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
if FLAGS.job_name == "ps":
server.join() # ps hosts only join
elif FLAGS.job_name == "worker":
# workers perform the operation
# ps_strategy = tf.contrib.training.GreedyLoadBalancingStrategy(FLAGS.num_ps)
# Note: tf.train.replica_device_setter automatically place the paramters (Variables)
# on the ps hosts (default placement strategy: round-robin over all ps hosts, and also
# place multi copies of operations to each worker host
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % (FLAGS.task_index),
cluster=cluster)):
# load mnist dataset
mnist = input_data.read_data_sets(
'/storage/emulated/0/tensor-data/', one_hot=True)
# the model
images = tf.placeholder(tf.float32, [None, 784])
labels = tf.placeholder(tf.int32, [None, 10])
logits = model(images)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=labels))
# The StopAtStepHook handles stopping after running given steps.
hooks = [tf.train.StopAtStepHook(last_step=2000)]
global_step = tf.train.get_or_create_global_step()
optimizer = tf.train.AdamOptimizer(learning_rate=1e-04)
if FLAGS.is_sync:
# asynchronous training
# use tf.train.SyncReplicasOptimizer wrap optimizer
# ref: https://www.tensorflow.org/api_docs/python/tf/train/SyncReplicasOptimizer
optimizer = tf.train.SyncReplicasOptimizer(
optimizer,
replicas_to_aggregate=FLAGS.num_workers,
total_num_replicas=FLAGS.num_workers)
# create the hook which handles initialization and queues
hooks.append(
optimizer.make_session_run_hook((FLAGS.task_index == 0)))
train_op = optimizer.minimize(
loss,
global_step=global_step,
aggregation_method=tf.AggregationMethod.ADD_N)
# The MonitoredTrainingSession takes care of session initialization,
# restoring from a checkpoint, saving to a checkpoint, and closing when done
# or an error occurs.
with tf.train.MonitoredTrainingSession(
master=server.target,
is_chief=(FLAGS.task_index == 0),
checkpoint_dir="/storage/emulated/0/checkpoint_dir",
hooks=hooks) as mon_sess:
while not mon_sess.should_stop():
# mon_sess.run handles AbortedError in case of preempted PS.
img_batch, label_batch = mnist.train.next_batch(32)
_, ls, step = mon_sess.run([train_op, loss, global_step],
feed_dict={
images: img_batch,
labels: label_batch
})
if step % 100 == 0:
print("Train step %d, loss: %f" % (step, ls))
def run_training(extra_opts={}):
start = datetime.datetime.now()
start_str = start.strftime('%d-%m-%Y_%H_%M')
train, validation = input_data.read_data_sets()
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
net.build(extra_opts)
#Precision summaries
precision_train = tf.Variable(0.0,
trainable=False,
name='precision_train')
precision_validation = tf.Variable(0.0,
trainable=False,
name='precision_validation')
precision_train_summary = tf.summary.scalar('precision/train',
precision_train)
precision_validation_summary = tf.summary.scalar(
'precision/validation', precision_validation)
graph = tf.get_default_graph()
loss = graph.get_tensor_by_name('loss:0')
train_op = graph.get_operation_by_name('train_op')
correct_count = graph.get_tensor_by_name('correct_count:0')
#Create summary stuff
regular_summaries_names = ['loss', 'learning_rate']
regular_summaries_list = []
for name in regular_summaries_names:
summary = graph.get_tensor_by_name('summary/' + name + ':0')
regular_summaries_list.append(summary)
regular_summaries = tf.summary.merge(regular_summaries_list,
name='summary/regular_summaries')
# Create a saver for writing training checkpoints.
saver = tf.train.Saver(tf.global_variables())
# Run the Op to initialize the variables.
init = tf.global_variables_initializer()
# Create a session for running Ops on the Graph.
sess = tf.Session(graph=graph,
config=tf.ConfigProto(
intra_op_parallelism_threads=3,
inter_op_parallelism_threads=3))
sess.run(init)
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(FLAGS.log_dir,
graph=tf.get_default_graph())
# And then after everything is built, start the training loop.
for step in xrange(1, FLAGS.max_steps + 1):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(train.next_batch(FLAGS.batch_size))
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % FLAGS.overview_steps == 0:
# Print status to stdout.
data_per_sec = FLAGS.batch_size / duration
print('Step %d: loss = %.2f (%.3f sec) [%.2f data/s]' %
(step, loss_value, duration, data_per_sec))
# Update the events file.
summary_str = sess.run(regular_summaries, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
# Save a checkpoint and evaluate the model periodically.
if (step) % FLAGS.evaluation_steps == 0 or step == FLAGS.max_steps:
saver.save(sess, './log' + '/checkpoint', global_step=step)
# Evaluate against the training set.
print('Training Data Eval:')
precision_t, obj_t = do_eval(sess, correct_count, loss, train)
sess.run(precision_train.assign(precision_t))
# Evaluate against the validation set.
print('Validation Data Eval:')
precision_v, obj_v = do_eval(sess, correct_count, loss,
validation)
sess.run(precision_validation.assign(precision_v))
summary_str_0, summary_str_1 = sess.run(
[precision_train_summary, precision_validation_summary])
summary_writer.add_summary(summary_str_0, step)
summary_writer.add_summary(summary_str_1, step)
os.makedirs('./results', exist_ok=True)
res_file = open('./results/res_' + str(start_str), 'w')
res_file.write('Iterations: ' + str(step) + '\n')
now = datetime.datetime.now()
delta = now - start
res_file.write('Learning time: {0:.2f} minutes\n'.format(
delta.total_seconds() / 60.0))
res_file.write(
'Train precision: {0:.5f}\n'.format(precision_t))
res_file.write('Train loss: {0:.5f}\n'.format(obj_t))
res_file.write(
'Validation precision: {0:.5f}\n'.format(precision_v))
res_file.write('Validation loss: {0:.5f}\n'.format(obj_v))
res_file.write('Extra opts: ' + str(extra_opts) + '\n')
res_file.write('Code:\n')
net_file = open('./net.py', 'r')
shutil.copyfileobj(net_file, res_file)
net_file.close()
res_file.close()
from TensorFlowInterface import *
import input_data
from pylab import *
from numpy import *
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
session = tf.InteractiveSession()
# Hacky class to add a null option to the MNIST one hot vector (position 10)
class MNISTModifier(object):
def __init__(self,data):
self.data = data
def next_batch(self,miniBatch):
batch = list(self.data.next_batch(miniBatch))
batch[1] = np.hstack((batch[1],zeros([shape(batch[1])[0]]+[1])))
return batch
class Container(object):
pass
x = tf.placeholder('float',shape=[None,784],name='input') # Input tensor
y_ = tf.placeholder('float', shape=[None,11],name='correctLabels') # Correct labels
xImage = tf.reshape(x,[-1,28,28,1]) # Reshape samples to 28x28x1 images
trainingIterations = 5000
# Standard conv net from Session 3
L1 = Conv2D(xImage,[5,5,1,32],'Conv1')
L2 = MaxPool2x2(L1.output,'MaxPool1')
import numpy as np
import os
import tsne
import numpy as Math
import pylab as Plot
from matplotlib.offsetbox import OffsetImage, AnnotationBbox
tf.flags.DEFINE_string("data_dir", "", "")
tf.flags.DEFINE_boolean("read_attn", True, "enable attention for reader")
tf.flags.DEFINE_boolean("write_attn", True, "enable attention for writer")
FLAGS = tf.flags.FLAGS
## MODEL PARAMETERS ##
data_directory = os.path.join(FLAGS.data_dir, "easy")
if not os.path.exists(data_directory):
os.makedirs(data_directory)
train_data = input_data.read_data_sets(data_directory, one_hot=True).train
A, B = 56, 56 # image width,height
img_size = B * A # the canvas size
enc_size = 500 # number of hidden units / output size in LSTM
dec_size = 500
read_n = 12 # read glimpse grid width/height
write_n = 12 # write glimpse grid width/height
read_size = 2 * read_n * read_n if FLAGS.read_attn else 2 * img_size
rs = np.sqrt(read_size / 2).astype(int)
write_size = write_n * write_n if FLAGS.write_attn else img_size
z_size = 10 # QSampler output size
T = 10 # MNIST generation sequence length
batch_size = train_data._num_examples # training minibatch size
train_iters = 10000
learning_rate = 1e-3 # learning rate for optimizer
eps = 1e-8 # epsilon for numerical stability
"""
A script to demonstrate usage of tf.zeros and tf.ones
"""
import numpy as np
import tensorflow as tf
import input_data
minst = input_data.read_data_sets("MINST_data/", one_hot=True)
presetB = np.float32(np.random.rand(784, 2))
print("presetB:")
print(presetB)
x = tf.placeholder("float", [None, 784])
y_ = tf.placeholder("float", [None, 2])
W = tf.Variable(tf.random_uniform([784, 2], -1.0, 1.0))
b = tf.Variable(tf.zeros([2]))
y = tf.matmul(x, W) + b
# 最小化方差
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
# 初始化变量
init = tf.initialize_all_variables()
# 启动图 (graph)
sess = tf.Session()
sess.run(init)
# this network is the same as the previous one except with an extra
# hidden layer + dropout
def model(capital_x, w_h, w_h2, w_o, p_keep_input, p_keep_hidden):
capital_x = tf.nn.dropout(capital_x, p_keep_input)
h = tf.nn.relu(tf.matmul(capital_x, w_h))
h = tf.nn.dropout(h, p_keep_hidden)
h2 = tf.nn.relu(tf.matmul(h, w_h2))
h2 = tf.nn.dropout(h2, p_keep_hidden)
return tf.matmul(h2, w_o)
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX = mnist.train.images
trY = mnist.train.labels
teX = mnist.test.images
teY = mnist.test.labels
X = tf.placeholder("float", [None, 784])
Y = tf.placeholder("float", [None, 10])
w_h = init_weights([784, 625])
w_h2 = init_weights([625, 625])
w_o = init_weights([625, 10])
p_keep_input = tf.placeholder("float")
p_keep_hidden = tf.placeholder("float")
py_x = model(X, w_h, w_h2, w_o, p_keep_input, p_keep_hidden)
import tensorflow as tf
from numpy.random.mtrand import randint
import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
learning_rate = 0.001
training_epochs = 15
# learning_rate = 0.01
# training_epochs = 25
batch_size = 100
display_step = 1
x = tf.placeholder( "float", [None, 784] )
y = tf.placeholder( "float", [None, 10] )
# x = tf.placeholder( "float", [None, 784] )
# y = tf.placeholder( "float", [None, 10] )
W1 = tf.Variable( tf.random_normal([784, 256]) )
W2 = tf.Variable( tf.random_normal([256, 256]) )
W3 = tf.Variable( tf.random_normal([256, 10]) )
# W = tf.Variable( tf.zeros([784, 10]) )
b1 = tf.Variable( tf.random_normal([256]) )
b2 = tf.Variable( tf.random_normal([256]) )
b3 = tf.Variable( tf.random_normal([10]) )
# b = tf.Variable( tf.zeros([10]) )
# Hypothesis
L1 = tf.nn.relu( tf.add(tf.matmul(x, W1), b1) )
L2 = tf.nn.relu( tf.add(tf.matmul(L1, W2), b2) )
def train():
# Import data
mnist = input_data.read_data_sets(FLAGS.data_dir,
one_hot=True,
fake_data=FLAGS.fake_data)
total_tr_data, total_tr_label = mnist.train.next_batch(
mnist.train._num_examples)
# Gathering a1 Data
tr_data_a1 = total_tr_data[(total_tr_label[:, FLAGS.a1] == 1.0)]
for i in range(len(tr_data_a1)):
for j in range(len(tr_data_a1[0])):
rand_num = np.random.rand()
if (rand_num >= 0.5):
tr_data_a1[i, j] = np.minimum(tr_data_a1[i, j] + rand_num, 1.0)
# Gathering a2 Data
tr_data_a2 = total_tr_data[(total_tr_label[:, FLAGS.a2] == 1.0)]
for i in range(len(tr_data_a2)):
for j in range(len(tr_data_a2[0])):
rand_num = np.random.rand()
if (rand_num >= 0.5):
tr_data_a2[i, j] = np.minimum(tr_data_a2[i, j] + rand_num, 1.0)
# Gathering b1 Data
tr_data_b1 = total_tr_data[(total_tr_label[:, FLAGS.b1] == 1.0)]
for i in range(len(tr_data_b1)):
for j in range(len(tr_data_b1[0])):
rand_num = np.random.rand()
if (rand_num >= 0.5):
tr_data_b1[i, j] = np.minimum(tr_data_b1[i, j] + rand_num, 1.0)
# Gathering b2 Data
tr_data_b2 = total_tr_data[(total_tr_label[:, FLAGS.b2] == 1.0)]
for i in range(len(tr_data_b2)):
for j in range(len(tr_data_b2[0])):
rand_num = np.random.rand()
if (rand_num >= 0.5):
tr_data_b2[i, j] = np.minimum(tr_data_b2[i, j] + rand_num, 1.0)
tr_data1 = np.append(tr_data_a1, tr_data_a2, axis=0)
tr_label1 = np.zeros((len(tr_data1), 2), dtype=float)
for i in range(len(tr_data1)):
if (i < len(tr_data_a1)):
tr_label1[i, 0] = 1.0
else:
tr_label1[i, 1] = 1.0
tr_data2 = np.append(tr_data_b1, tr_data_b2, axis=0)
tr_label2 = np.zeros((len(tr_data2), 2), dtype=float)
for i in range(len(tr_data2)):
if (i < len(tr_data_b1)):
tr_label2[i, 0] = 1.0
else:
tr_label2[i, 1] = 1.0
## TASK 1
sess = tf.InteractiveSession()
# Input placeholders
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, 784], name='x-input')
y_ = tf.placeholder(tf.float32, [None, 2], name='y-input')
with tf.name_scope('input_reshape'):
image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])
tf.summary.image('input', image_shaped_input, 2)
# geopath_examples
geopath = pathnet.geopath_initializer(FLAGS.L, FLAGS.M)
# fixed weights list
fixed_list = np.ones((FLAGS.L, FLAGS.M), dtype=str)
for i in range(FLAGS.L):
for j in range(FLAGS.M):
fixed_list[i, j] = '0'
# Hidden Layers
weights_list = np.zeros(
(FLAGS.L, FLAGS.M),
dtype=object) # weights_list also record conv_kernels
biases_list = np.zeros((FLAGS.L, FLAGS.M), dtype=object)
# change: put init part into pathnet.module
#for i in range(FLAGS.L):
# for j in range(FLAGS.M):
#if(i==0):
#weights_list[i,j]=pathnet.module_weight_variable([784,FLAGS.filt]);
# biases_list[i,j]=pathnet.module_bias_variable([FLAGS.filt]);
#else:
#weights_list[i,j]=pathnet.module_weight_variable([FLAGS.filt,FLAGS.filt]);
#biases_list[i,j]=pathnet.module_bias_variable([FLAGS.filt]);
layer_modules_list = np.zeros(FLAGS.M, dtype=object)
i = 0
for j in range(FLAGS.M):
layer_modules_list[j], weights_list[i, j], biases_list[
i,
j] = pathnet.conv_module(image_shaped_input, FLAGS.filt, [5, 5],
geopath[i, j], 1,
'layer' + str(i + 1) + "_" + str(j + 1))
net = np.sum(layer_modules_list) / FLAGS.M
i = 1
for j in range(FLAGS.M):
layer_modules_list[j], weights_list[i, j], biases_list[
i, j] = pathnet.res_module(net, geopath[i, j],
'layer' + str(i + 1) + "_" + str(j + 1))
net = np.sum(layer_modules_list) / FLAGS.M
net = tf.reshape(net, [-1, 24 * 24 * 20])
i = 2
for j in range(FLAGS.M):
layer_modules_list[j], weights_list[i, j], biases_list[
i, j] = pathnet.module(net, FLAGS.filt, geopath[i, j],
'layer' + str(i + 1) + "_" + str(j + 1))
net = np.sum(layer_modules_list) / FLAGS.M
# need to change: put this values into module
#net=net/FLAGS.M;
# Output Layer
y, output_weights, output_biases = pathnet.nn_layer(
net, 2, 'output_layer')
# Cross Entropy
with tf.name_scope('cross_entropy'):
diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
with tf.name_scope('total'):
cross_entropy = tf.reduce_mean(diff)
tf.summary.scalar('cross_entropy', cross_entropy)
# Need to learn variables
var_list_to_learn = [] + output_weights + output_biases
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '0'):
var_list_to_learn += weights_list[i, j] + biases_list[i, j]
# GradientDescent
with tf.name_scope('train'):
train_step = tf.train.GradientDescentOptimizer(
FLAGS.learning_rate).minimize(cross_entropy,
var_list=var_list_to_learn)
# Accuracy
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Merge all the summaries and write them out to /tmp/tensorflow/mnist/logs/mnist_with_summaries (by default)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train1', sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test1')
tf.global_variables_initializer().run()
# Generating randomly geopath
geopath_set = np.zeros(FLAGS.candi, dtype=object)
for i in range(FLAGS.candi):
geopath_set[i] = pathnet.get_geopath(FLAGS.L, FLAGS.M, FLAGS.N)
# parameters placeholders and ops
var_update_ops = np.zeros(len(var_list_to_learn), dtype=object)
var_update_placeholders = np.zeros(len(var_list_to_learn), dtype=object)
for i in range(len(var_list_to_learn)):
var_update_placeholders[i] = tf.placeholder(
var_list_to_learn[i].dtype, shape=var_list_to_learn[i].get_shape())
var_update_ops[i] = var_list_to_learn[i].assign(
var_update_placeholders[i])
# geopathes placeholders and ops
geopath_update_ops = np.zeros((len(geopath), len(geopath[0])),
dtype=object)
geopath_update_placeholders = np.zeros((len(geopath), len(geopath[0])),
dtype=object)
for i in range(len(geopath)):
for j in range(len(geopath[0])):
geopath_update_placeholders[i, j] = tf.placeholder(
geopath[i, j].dtype, shape=geopath[i, j].get_shape())
geopath_update_ops[i, j] = geopath[i, j].assign(
geopath_update_placeholders[i, j])
acc_geo = np.zeros(FLAGS.B, dtype=float)
summary_geo = np.zeros(FLAGS.B, dtype=object)
for i in range(FLAGS.max_steps):
# Select Candidates to Tournament
compet_idx = range(FLAGS.candi)
np.random.shuffle(compet_idx)
compet_idx = compet_idx[:FLAGS.B]
# Learning & Evaluating
for j in range(len(compet_idx)):
# Shuffle the data
idx = range(len(tr_data1))
np.random.shuffle(idx)
tr_data1 = tr_data1[idx]
tr_label1 = tr_label1[idx]
# Insert Candidate
pathnet.geopath_insert(sess, geopath_update_placeholders,
geopath_update_ops,
geopath_set[compet_idx[j]], FLAGS.L,
FLAGS.M)
acc_geo_tr = 0
for k in range(FLAGS.T):
summary_geo_tr, _, acc_geo_tmp = sess.run(
[merged, train_step, accuracy],
feed_dict={
x:
tr_data1[k * FLAGS.batch_num:(k + 1) *
FLAGS.batch_num, :],
y_:
tr_label1[k * FLAGS.batch_num:(k + 1) *
FLAGS.batch_num, :]
})
acc_geo_tr += acc_geo_tmp
acc_geo[j] = acc_geo_tr / FLAGS.T
summary_geo[j] = summary_geo_tr
# Tournament
winner_idx = np.argmax(acc_geo)
acc = acc_geo[winner_idx]
summary = summary_geo[winner_idx]
# Copy and Mutation
for j in range(len(compet_idx)):
if (j != winner_idx):
geopath_set[compet_idx[j]] = np.copy(
geopath_set[compet_idx[winner_idx]])
geopath_set[compet_idx[j]] = pathnet.mutation(
geopath_set[compet_idx[j]], FLAGS.L, FLAGS.M, FLAGS.N)
train_writer.add_summary(summary, i)
print('Training Accuracy at step %s: %s' % (i, acc))
if (acc >= 0.99):
print('Learning Done!!')
print('Optimal Path is as followed.')
print(geopath_set[compet_idx[winner_idx]])
task1_optimal_path = geopath_set[compet_idx[winner_idx]]
break
"""
geopath_sum=np.zeros((len(geopath),len(geopath[0])),dtype=float);
for j in range(len(geopath_set)):
for k in range(len(geopath)):
for l in range(len(geopath[0])):
geopath_sum[k][l]+=geopath_set[j][k][l];
print(geopath_sum);
"""
# record steps to find optimal path in task1
iter_task1 = i
# Fix task1 Optimal Path
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (task1_optimal_path[i, j] == 1.0):
fixed_list[i, j] = '1'
# Get variables of fixed list
var_list_to_fix = []
#var_list_to_fix=[]+output_weights+output_biases;
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '1'):
var_list_to_fix += weights_list[i, j] + biases_list[i, j]
var_list_fix = pathnet.parameters_backup(var_list_to_fix)
"""
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if(task1_optimal_path[i,j]==1.0):
fixed_list[i,j]='0';
"""
# parameters placeholders and ops
var_fix_ops = np.zeros(len(var_list_to_fix), dtype=object)
var_fix_placeholders = np.zeros(len(var_list_to_fix), dtype=object)
for i in range(len(var_list_to_fix)):
var_fix_placeholders[i] = tf.placeholder(
var_list_to_fix[i].dtype, shape=var_list_to_fix[i].get_shape())
var_fix_ops[i] = var_list_to_fix[i].assign(var_fix_placeholders[i])
## TASK 2
# Need to learn variables
var_list_to_learn = [] + output_weights + output_biases
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '0'):
var_list_to_learn += weights_list[i, j] + biases_list[i, j]
'''
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if(fixed_list[i,j]=='1'):
tmp=biases_list[i,j][0];
break;
break;
'''
# Initialization
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train2', sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test2')
tf.global_variables_initializer().run()
# Update fixed values
pathnet.parameters_update(sess, var_fix_placeholders, var_fix_ops,
var_list_fix)
# GradientDescent
with tf.name_scope('train'):
train_step = tf.train.GradientDescentOptimizer(
FLAGS.learning_rate).minimize(cross_entropy,
var_list=var_list_to_learn)
# Generating randomly geopath
geopath_set = np.zeros(FLAGS.candi, dtype=object)
for i in range(FLAGS.candi):
geopath_set[i] = pathnet.get_geopath(FLAGS.L, FLAGS.M, FLAGS.N)
# parameters placeholders and ops
var_update_ops = np.zeros(len(var_list_to_learn), dtype=object)
var_update_placeholders = np.zeros(len(var_list_to_learn), dtype=object)
for i in range(len(var_list_to_learn)):
var_update_placeholders[i] = tf.placeholder(
var_list_to_learn[i].dtype, shape=var_list_to_learn[i].get_shape())
var_update_ops[i] = var_list_to_learn[i].assign(
var_update_placeholders[i])
acc_geo = np.zeros(FLAGS.B, dtype=float)
summary_geo = np.zeros(FLAGS.B, dtype=object)
for i in range(FLAGS.max_steps):
# Select Candidates to Tournament
compet_idx = range(FLAGS.candi)
np.random.shuffle(compet_idx)
compet_idx = compet_idx[:FLAGS.B]
# Learning & Evaluating
for j in range(len(compet_idx)):
# Shuffle the data
idx = range(len(tr_data2))
np.random.shuffle(idx)
tr_data2 = tr_data2[idx]
tr_label2 = tr_label2[idx]
geopath_insert = np.copy(geopath_set[compet_idx[j]])
for l in range(FLAGS.L):
for m in range(FLAGS.M):
if (fixed_list[l, m] == '1'):
geopath_insert[l, m] = 1.0
# Insert Candidate
pathnet.geopath_insert(sess, geopath_update_placeholders,
geopath_update_ops, geopath_insert, FLAGS.L,
FLAGS.M)
acc_geo_tr = 0
for k in range(FLAGS.T):
summary_geo_tr, _, acc_geo_tmp = sess.run(
[merged, train_step, accuracy],
feed_dict={
x:
tr_data2[k * FLAGS.batch_num:(k + 1) *
FLAGS.batch_num, :],
y_:
tr_label2[k * FLAGS.batch_num:(k + 1) *
FLAGS.batch_num, :]
})
acc_geo_tr += acc_geo_tmp
acc_geo[j] = acc_geo_tr / FLAGS.T
summary_geo[j] = summary_geo_tr
# Tournament
winner_idx = np.argmax(acc_geo)
acc = acc_geo[winner_idx]
summary = summary_geo[winner_idx]
# Copy and Mutation
for j in range(len(compet_idx)):
if (j != winner_idx):
geopath_set[compet_idx[j]] = np.copy(
geopath_set[compet_idx[winner_idx]])
geopath_set[compet_idx[j]] = pathnet.mutation(
geopath_set[compet_idx[j]], FLAGS.L, FLAGS.M, FLAGS.N)
train_writer.add_summary(summary, i)
print('Training Accuracy at step %s: %s' % (i, acc))
if (acc >= 0.99):
print('Learning Done!!')
print('Optimal Path is as followed.')
print(geopath_set[compet_idx[winner_idx]])
task2_optimal_path = geopath_set[compet_idx[winner_idx]]
break
"""
geopath_sum=np.zeros((len(geopath),len(geopath[0])),dtype=float);
for j in range(len(geopath_set)):
for k in range(len(geopath)):
for l in range(len(geopath[0])):
geopath_sum[k][l]+=geopath_set[j][k][l];
print(geopath_sum);
"""
iter_task2 = i
overlap = 0
for i in range(len(task1_optimal_path)):
for j in range(len(task1_optimal_path[0])):
if (task1_optimal_path[i, j] == task2_optimal_path[i, j]) & (
task1_optimal_path[i, j] == 1.0):
overlap += 1
print("Entire Iter:" + str(iter_task1 + iter_task2) + ",TASK1:" +
str(iter_task1) + ",TASK2:" + str(iter_task2) + ",Overlap:" +
str(overlap))
train_writer.close()
test_writer.close()
def main():
mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)
plus_data = load_image_data('csv/plus_data.csv')
data = add_data(mnist_data, [plus_data])
print data
import input_data
import numpy as np
import os
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
dir = os.path.dirname(__file__)
mnist_data = input_data.read_data_sets(os.path.join(dir, 'mnist_data'),
dtype=tf.uint8,
one_hot=True)
mnist_data = {
'train': mnist_data.train,
'validation': mnist_data.validation,
'test': mnist_data.test
}
for d in ['train', 'validation', 'test']:
images = mnist_data[d].images
labels = mnist_data[d].labels
# images.shape is (55000, 784)
pixels = images.shape[1]
num_examples = mnist_data[d].num_examples
# tfrecord文件的保存地址
