# neural network with 5 layers
#
# · · · · · · · · · · (input data, flattened pixels) X [batch, 784] # 784 = 28*28
# \x/x\x/x\x/x\x/x\x/ ✞ -- fully connected layer (relu+dropout) W1 [784, 200] B1[200]
# · · · · · · · · · Y1 [batch, 200]
# \x/x\x/x\x/x\x/ ✞ -- fully connected layer (relu+dropout) W2 [200, 100] B2[100]
# · · · · · · · Y2 [batch, 100]
# \x/x\x/x\x/ ✞ -- fully connected layer (relu+dropout) W3 [100, 60] B3[60]
# · · · · · Y3 [batch, 60]
# \x/x\x/ ✞ -- fully connected layer (relu+dropout) W4 [60, 30] B4[30]
# · · · Y4 [batch, 30]
# \x/ -- fully connected layer (softmax) W5 [30, 10] B5[10]
# · Y5 [batch, 10]
# Download images and labels
mnist = mnist_data.read_data_sets("data")
# input X: 28x28 grayscale images, the first dimension (None) will index the images in the mini-batch
X = tf.placeholder(tf.float32, [None, 28, 28, 1])
# correct answers will go here
Y_ = tf.placeholder(tf.float32, [None, 10])
# variable learning rate
lr = tf.placeholder(tf.float32)
# Probability of keeping a node during dropout = 1.0 at test time (no dropout) and 0.75 at training time
pkeep = tf.placeholder(tf.float32)
# five layers and their number of neurons (tha last layer has 10 softmax neurons)
L = 200
M = 100
N = 60
O = 30
def LoadDataSet(self):
self.mnist = mnist_data.read_data_sets("mydata")
from triplet_clustering import TripletClustering
from clustering_util import run_loss_aug_clustering_on, evaluate_clustering, run_clustering_on
import mnist_data
import tensorflow as tf
import numpy as np
from datetime import datetime
from random import randrange
train_data = mnist_data.read_data_sets('./data/', one_hot=False)
test_im = train_data.test.images
test_l = train_data.test.labels
# HOS: shuffle but move every element. Creates copy.
def sattoloShuffle(x):
items = np.copy(x)
i = len(items)
while i > 1:
i = i - 1
j = randrange(i) # 0 <= j <= i-1
items[j], items[i] = items[i], items[j]
return items
def get_batch():
im, l = train_data.train.next_batch(256)
return im.astype(np.float32), l
def get_triplets(im, l):
pos_idx = np.zeros((256, ), dtype=int)
import mnist_data
import tensorflow as tf
# based on tutorial at http://www.tensorflow.org/tutorials/mnist/pros/index.html
mnist = mnist_data.read_data_sets('MNIST_data', one_hot=True)
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
# Helper method for maximum pooling. strides 2x2, 75% reduction in image size per pooling operation
def max_pool_2x2(x):
return tf.nn.max_pool(x,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
x = tf.placeholder("float", shape=[None, 784],
# predict test data
predict(sess, x, model.keep_prob, model.pred, data.test.images, args.output)
# origiinal test data from 'http://yann.lecun.com/exdb/mnist/'
"""
acc = sess.run(model.accuracy, feed_dict={x: data.test.images, y: data.test.labels, model.keep_prob: 1.0})
print("test accuracy %g"%acc)
"""
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--data_dir', type=str, default='mnist_data/', help='Directory for storing input data')
parser.add_argument('-c', '--ckpt_dir', type=str, default='ckpts/', help='Directory for parameter checkpoints')
parser.add_argument('-l', '--load_params', dest='load_params', action='store_true', help='Restore training from previous model checkpoint?')
parser.add_argument("-o", "--output", type=str, default='prediction.csv', help='Prediction filepath')
parser.add_argument('-e', '--epochs', type=int, default=30, help='How many epochs to run in total?')
parser.add_argument('-b', '--batch_size', type=int, default=50, help='Batch size during training per GPU')
parser.add_argument('-v', '--val_size', type=int, default=5000)
args = parser.parse_args()
# pretty print args
print('input args:\n', json.dumps(vars(args), indent=4, separators=(',',':')))
data = mnist_data.read_data_sets(args.data_dir, one_hot=True, reshape=False, validation_size=args.val_size)
#data = input_data.read_data_sets(args.data_dir, one_hot=True, reshape=False, validation_size=args.val_size)
if not os.path.exists(args.ckpt_dir):
os.makedirs(args.ckpt_dir)
train(args, data)
def train(hold_out, dev_name, sample, go_on, semi):
hold_out_s = int(hold_out)
print hold_out_s
if go_on:
c = pickle.load(open('chosen_ball.bn'))
ch = c['all']
train_data = mnist_data.read_data_sets('./data/',
one_hot=True,
hold_out_size=hold_out_s,
active=True,
choices=ch)
else:
train_data = mnist_data.read_data_sets('./data/',
one_hot=True,
hold_out_size=hold_out_s)
test_im = train_data.test.images[0:1000]
test_l = train_data.test.labels[0:1000]
params, str_v = read_params('settings.json')
print train_data.train.images.shape, train_data.validation.images.shape
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
with tf.Session(config=config) as sesh:
robust_cifar_train = AdverserialTrainer(params.learning_rate,
0.1 * params.learning_rate,
dev_name, 9)
print 'Initial Variable List:'
print[tt.name for tt in tf.trainable_variables()]
init = tf.global_variables_initializer()
sesh.run(init)
saver = tf.train.Saver(max_to_keep=100)
sum_writer = tf.summary.FileWriter(
"./dumps_adverserial__semi_{}_go_on_{}__sample_{}__hold_out_{}__{}/"
.format(semi, go_on, sample, hold_out_s, str_v), sesh.graph)
im, l = train_data.train.next_batch(
params.batch_size) # Sample a boosted set
for batch_id in range(200000):
im, l = train_data.train.next_batch(
params.batch_size) # Sample a boosted set
adv_im, _ = train_data.validation.next_batch(params.batch_size)
flip_f = float(batch_id) / 200000.0
robust_cifar_train.assign_batch({
'images': im,
'labels': l
}, {'images': adv_im})
# first epoch only observe then do staff
create_summ = batch_id % 10 == 0
if semi:
robust_cifar_train.learning_step(sesh, flip_f, 0.0)
else:
robust_cifar_train.learning_step(sesh, flip_f, 1.0)
if create_summ:
loss, summ, a_l, a_s = robust_cifar_train.summary_step(sesh)
sum_writer.add_summary(summ, batch_id)
sum_writer.add_summary(a_s, batch_id)
# save every 100th batch model
if batch_id % 500 == 0:
saver.save(
sesh,
'models/cifar10_go_on_{}_robust_{}_model_hold_out_{}__{}'.
format(go_on, sample, hold_out_s, str_v),
global_step=batch_id)
acc, test_sum = robust_cifar_train.test_step(
test_im, test_l, sesh)
print "{}: step{}, test acc {}".format(datetime.now(),
batch_id, acc)
sum_writer.add_summary(test_sum, batch_id)
if batch_id % 100 == 0:
print "{}: step {}, loss {} adv_loss {}".format(
datetime.now(), batch_id, loss, a_l)
'If true, skip training the model.')
flags.DEFINE_boolean('restore', False,
'If true, restore the model from the latest checkpoint.')
# define artifact directories where results from the session can be saved
model_path = os.environ.get('MODEL_PATH', 'models/')
if not os.path.exists(model_path):
os.mkdir(model_path)
checkpoint_path = os.environ.get('CHECKPOINT_PATH', 'checkpoints/')
if not os.path.exists(checkpoint_path):
os.mkdir(checkpoint_path)
summary_path = os.environ.get('SUMMARY_PATH', 'logs/')
if not os.path.exists(summary_path):
os.mkdir(summary_path)
mnist = mnist.read_data_sets('mnist', one_hot=True)
def weight_bias(W_shape, b_shape, bias_init=0.1):
W = tf.Variable(tf.truncated_normal(shape=W_shape, stddev=0.1),
name='weight')
b = tf.Variable(tf.constant(bias_init, shape=b_shape), name='bias')
return W, b
def dense_layer(x, W_shape, b_shape, activation):
W, b = weight_bias([W_shape, b_shape], [b_shape])
return activation(tf.matmul(x, W) + b)
def highway_layer(x, size, activation, carry_bias=-1.0):
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
#MNIST Data
import mnist_data
mnist = mnist_data.read_data_sets("F:\ASU\pro_auto_encoder", one_hot=True)
#Hyperparameters
learning_rate = 0.01
num_steps = 100
batch_size = 256
display_step = 100
examples_to_show = 10
#Network
num_hidden = 256
latent_dim = 128
num_input = 784
#Graph
X = tf.placeholder("float", [None, num_input])
####weights
weights = {
'encoder_h1': tf.Variable(tf.random_normal([num_input, num_hidden])),
'encoder_h2': tf.Variable(tf.random_normal([num_hidden, latent_dim])),
'decoder_h1': tf.Variable(tf.random_normal([latent_dim, num_hidden])),
import mnist_data
import tensorflow as tf
mnist = mnist_data.read_data_sets("MNIST_data", one_hot=True)
sess = tf.InteractiveSession()
x = tf.placeholder("float", shape=[None, 784]) # input images, 784 == 28**2 flattened
y_ = tf.placeholder("float", shape=[None, 10]) # target output class
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
sess.run(tf.initialize_all_variables())
y = tf.nn.softmax(tf.matmul(x, W) + b)
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
for i in range(1000):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x: batch[0], y_: batch[1]})
# Since evaluating the accuracy is expensive, only run it every 50 steps
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Import data
import mnist_data as input_data
import tensorflow as tf
import tensorflowvisu
import math
tf.set_random_seed(0)
mnist = input_data.read_data_sets("data")
# neural network structure for this sample:
#
# · · · · · · · · · · (input data, 1-deep) X [batch, 28, 28, 1]
# @ @ @ @ @ @ @ @ @ @ -- conv. layer 5x5x1=>24 stride 1 W1 [5, 5, 1, 24] B1 [24]
# ∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶ Y1 [batch, 28, 28, 24]
# @ @ @ @ @ @ @ @ -- conv. layer 5x5x24=>48 stride 2 W2 [5, 5, 24, 48] B2 [48]
# ∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶ Y2 [batch, 14, 14, 48]
# @ @ @ @ @ @ -- conv. layer 5x5x48=>24 stride 1 W3 [5, 5, 48, 24] B3 [24]
# ∶∶∶∶∶∶∶∶∶∶∶ Y3 [batch, 14, 14, 24]
# @ @ @ @ -- conv. layer 5x5x24=>48 stride 2 W4 [5, 5, 24, 48] B4 [48]
# ∶∶∶∶∶∶∶ Y4 [batch, 7, 7, 48] reshaped to Z1 [batch, 7*7*48]
# \x/x\x/ - -- fully connected layer (relu+dropout) Wf1 [7*7*48, 1024] Bf1 [1024]
# · · · T1 [batch, 1024]
# \x/x/ - -- fully connected layer (relu+dropout) Wf2 [1024, 512] Bf2 [512]
import numpy as np
# Import MINST data
import mnist_data
#mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
mnist = mnist_data.read_data_sets("mnist_data/", one_hot=True, reshape=False)
# understand the size (1 , 28 , 28)
batch_xs, batch_ys = mnist.train.next_batch(1)
print(len(batch_xs[0]))
print(len(batch_xs[0][0]))
print(len(batch_xs[0][0][0]))
print(batch_ys)
import tensorflow as tf
# Parameters
learning_rate = 0.001
training_iters = 20000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
dropout = 0.75 # Dropout, probability to keep units
def weight_variable(shape):
initializer = tf.truncated_normal_initializer(dtype=tf.float32,
stddev=1e-1)
return tf.get_variable("weights",
import os
import models
import learning_rates
import mnist_data
import cifar_data
sess = tf.Session()
K.set_session(sess)
USE_CONV = True
MNIST = True # True for MNIST, False for CIFAR-10
if (MNIST):
print("Downloading MNIST if first run")
dataset = mnist_data.read_data_sets('MNIST_data',
one_hot=True,
reshape=(not USE_CONV))
mentor_model = models.build_mentor_model_conv(
) if USE_CONV else models.build_mentor_model()
img_input = models.img_conv if USE_CONV else models.img_dense
else:
dataset = cifar_data.read_data_sets('CIFAR_data')
mentor_model = models.build_mentor_model_conv_cifar10()
img_input = models.img_cifar
# run_name = "mentor" +
run_name = ("_conv" if USE_CONV else "") + ("_mnist" if MNIST else "_cifar10")
summary_name = run_name + "_accuracy"
model_save_name = "mentor" + run_name + ".h5"
mentor_preds = mentor_model.output
import mnist_data
import tensorflow as tf
# based on tutorial at http://www.tensorflow.org/tutorials/mnist/pros/index.html
mnist = mnist_data.read_data_sets('MNIST_data', one_hot=True)
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
# Helper method for maximum pooling. strides 2x2, 75% reduction in image size per pooling operation
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
x = tf.placeholder("float", shape=[None, 784], name="x") # None means the batch size can be any length
y_ = tf.placeholder("float", shape=[None, 10], name="y_") # One-hot 10-dimensional vector indicating which digit class the corresponding MNIST image belongs to
L2_SIZE = 64
L3_SIZE = 128
def distributed_model(config):
tf.app.flags.DEFINE_string("job_name", "", "Enter 'ps' or 'worker' ")
tf.app.flags.DEFINE_integer("task_index", 0,
"Index of the task within the jobs")
tf.app.flags.DEFINE_bool("async", True, "Async or Sync Train")
tf.app.flags.DEFINE_string("ps_hosts", "",
"Comma-separated list of hostname:port pairs")
tf.app.flags.DEFINE_string("worker_hosts", "",
"Comma-separated list of hostname:port pairs")
tf.app.flags.DEFINE_string("data_dir", "./data/", "Data directory")
FLAGS = tf.app.flags.FLAGS
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
mnist = mnist_data.read_data_sets(FLAGS.data_dir, one_hot=True)
cluster = tf.train.ClusterSpec({"worker": worker_hosts, "ps": ps_hosts})
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
if FLAGS.job_name == 'ps':
server.join()
else:
with tf.device(
tf.train.replica_device_setter(
worker_device='/job:worker/task:%d' % FLAGS.task_index,
cluster=cluster)):
global_step = tf.Variable(0, name="global_step", trainable=False)
x = tf.placeholder(tf.float32, [None, 784])
label = tf.placeholder(tf.float32, [None, 10])
pred_y = model(x)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=label, logits=pred_y)
cross_entropy = tf.reduce_mean(cross_entropy)
tf.summary.scalar('loss', cross_entropy)
with tf.name_scope('adam_optimizer'):
optimizer = tf.train.AdamOptimizer(config.lr)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(pred_y, 1),
tf.argmax(label, 1))
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
tf.summary.scalar('accuracy', accuracy)
with tf.name_scope('grads_and_vars'):
grads_and_vars = optimizer.compute_gradients(cross_entropy)
if FLAGS. async:
# 异步模式
train_op = optimizer.apply_gradients(grads_and_vars)
else:
rep_op = tf.train.SyncReplicasOptimizer(
optimizer,
replicas_to_aggregate=len(worker_hosts),
total_num_replicas=len(worker_hosts),
use_locking=True)
train_op = rep_op.apply_gradients(grads_and_vars,
global_step=global_step)
init_token_op = rep_op.get_init_tokens_op()
chief_queue_runner = rep_op.get_chief_queue_runner()
init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
summary_op = tf.summary.merge_all()
sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0),
logdir='./summary_log/',
init_op=init_op,
summary_op=None,
saver=saver,
global_step=global_step,
save_model_secs=60)
summary_writer = tf.summary.FileWriter('./summary_log/')
with sv.prepare_or_wait_for_session(server.target) as sess:
if FLAGS.task_index == 0 and not FLAGS. async:
sv.start_queue_runners(sess, [chief_queue_runner])
sess.run(init_token_op)
for i in xrange(config.epoches):
loss = 0
for j in xrange(train_data_size / config.batch_size):
batch = mnist.train.next_batch(config.batch_size)
sess.run(train_op,
feed_dict={
x: batch[0],
label: batch[1]
})
loss += sess.run(cross_entropy,
feed_dict={
x: batch[0],
label: batch[1]
})
if j % 5 == 0:
train_accuracy = accuracy.eval(session=sess,
feed_dict={
x: batch[0],
label: batch[1]
})
print(
'[Epoch: %d Sample: %d] training accuracy: %g loss: %s'
% (i, (j + 1) * config.batch_size,
train_accuracy, loss / (j + 1)))
summary = sess.run(summary_op)
summary_writer.add_summary(summary, global_step)
train_accuracy = accuracy.eval(session=sess,
feed_dict={
x: mnist.test.images,
label: mnist.test.labels
})
print("[Epoch: %s] Test Accuracy: %s" %
(i + 1, train_accuracy))
sv.stop()
def train(config):
mnist = mnist_data.read_data_sets(config.data_dir, one_hot=True)
x = tf.placeholder(tf.float32, [None, 784])
label = tf.placeholder(tf.float32, [None, 10])
pred_y = model(x)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=label,
logits=pred_y)
cross_entropy = tf.reduce_mean(cross_entropy)
tf.summary.scalar('loss', cross_entropy)
with tf.name_scope('adam_optimizer'):
train_step = tf.train.AdamOptimizer(config.lr).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(pred_y, 1),
tf.argmax(label, 1))
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
tf.summary.scalar('accuracy', accuracy)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# merged = tf.summary.merge_all()
# writer = tf.summary.FileWriter("logs/", sess.graph)
#sess.run(tf.global_variables_initializer())
for i in xrange(config.epoches):
loss = 0
for j in xrange(train_data_size / config.batch_size):
batch = mnist.train.next_batch(config.batch_size)
train_step.run(feed_dict={x: batch[0], label: batch[1]})
loss += sess.run(
cross_entropy, feed_dict={
x: batch[0],
label: batch[1]
}
) #sess.run(tf.reduce_mean(sess.run(cross_entropy, feed_dict={x: batch[0], label: batch[1]})))
if j % 5 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch[0],
label: batch[1]
})
print(
'[Epoch: %d Sample: %d] training accuracy: %g loss: %s'
% (i,
(j + 1) * config.batch_size, train_accuracy, loss /
(j + 1)))
train_accuracy = accuracy.eval(feed_dict={
x: mnist.test.images,
label: mnist.test.labels
})
print("[Epoch: %s] Test Accuracy: %s" % (i + 1, train_accuracy))
