def main(argv=None): # pylint: disable=unused-argument
print("begin")
cifar10.maybe_download_and_extract()
if tf.gfile.Exists(FLAGS.train_dir):
tf.gfile.DeleteRecursively(FLAGS.train_dir)
tf.gfile.MakeDirs(FLAGS.train_dir)
train()
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if tf.gfile.Exists(FLAGS.eval_dir):
tf.gfile.DeleteRecursively(FLAGS.eval_dir)
tf.gfile.MakeDirs(FLAGS.eval_dir)
with tf.device("/cpu:0"):
evaluate()
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if gfile.Exists(FLAGS.train_dir):
gfile.DeleteRecursively(FLAGS.train_dir)
gfile.MakeDirs(FLAGS.train_dir)
train()
f.close()
def backward():
#加载图片及图片batch张量
cifar10.maybe_download_and_extract()
images_train, labels_train = cifar10_input.distorted_inputs(data_dir=DATA_DIR, batch_size=BATCH_SIZE)
#构建反馈
x = tf.placeholder(tf.float32, shape = [None,24,24,3])
y_ = tf.placeholder(tf.float32, shape = [None])
keep_prob = tf.placeholder(tf.float32)
y = cnn_forward.forward(x,keep_prob,REGULARIZER)
global_step = tf.Variable(0, trainable = False)
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
20000/BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase = True)
#计算loss
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits = y, labels = tf.cast(y_, tf.int64))
loss_ce = tf.reduce_mean(ce)
loss_total = loss_ce + tf.add_n(tf.get_collection('losses'))
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss_total, global_step = global_step)
#滑动平均
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
#训练模型
with tf.control_dependencies([train_step, ema_op]) :
train_op = tf.no_op(name = 'train')
saver = tf.train.Saver()
sess = tf.InteractiveSession()
init_op = tf.global_variables_initializer()
sess.run(init_op)
#续训
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
#启用多线程队列加载图片
tf.train.start_queue_runners()
for i in range(STEPS):
image_batch, label_batch = sess.run([images_train, labels_train])
_, loss_value, step = sess.run([train_op, loss_total, global_step], feed_dict = {x: image_batch, y_: label_batch, keep_prob:0.5})
if i % 10 == 0:
print("After %d steps, loss is: %f" %(step, loss_value))
saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step = global_step)
sess.close()
def main(argv=None): # pylint: disable=unused-argument
os.environ["KMP_BLOCKTIME"] = "1"
os.environ["KMP_SETTINGS"] = "1"
os.environ["KMP_AFFINITY"]= "granularity=fine,verbose,compact,1,0"
os.environ["OMP_NUM_THREADS"]= "34"
os.environ["MKL_NUM_THREADS"]= "50"
cifar10.maybe_download_and_extract()
if tf.gfile.Exists(FLAGS.train_dir):
tf.gfile.DeleteRecursively(FLAGS.train_dir)
tf.gfile.MakeDirs(FLAGS.train_dir)
train()
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
# td = '/tmp/cifar10_train' + '-' + datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
# tf.app.flags.DEFINE_string('train_dir', str(td),
# """Directory where to write event logs """
# """and checkpoint.""")
if gfile.Exists(FLAGS.train_dir):
gfile.DeleteRecursively(FLAGS.train_dir)
gfile.MakeDirs(FLAGS.train_dir)
train()
def test():
cifar10.maybe_download_and_extract()
images_test, labels_test = cifar10_input.inputs(eval_data = True, data_dir=DATA_DIR, batch_size=BATCH_SIZE)
x = tf.placeholder(tf.float32, shape = [None,24,24,3])
y_ = tf.placeholder(tf.float32, shape = [None])
keep_prob = tf.placeholder(tf.float32)
y = cnn_forward.forward(x,keep_prob,cnn_backward.REGULARIZER)
ema = tf.train.ExponentialMovingAverage(cnn_backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
correct = tf.equal(tf.argmax(y, 1), tf.cast(y_, tf.int64))
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
while True:
sess = tf.InteractiveSession()
coord = tf.train.Coordinator()
queue_runner = tf.train.start_queue_runners(sess, coord = coord)
image_batch, label_batch = sess.run([images_test, labels_test])
ckpt = tf.train.get_checkpoint_state(cnn_backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
accuracy_score = sess.run(accuracy, feed_dict = {x:image_batch, y_:label_batch, keep_prob:0.98})
print("after %s setps, test accuracy is %g"%(global_step, accuracy_score))
else:
print("no checkpoint")
return
coord.request_stop()
coord.join(queue_runner)
sess.close()
time.sleep(TEST_INTERVAL)
batch_size = 128
data_dir='/tmp/cifar10_data/cifar-10-batches-bin'
def variable_with_weight_loss(shape, stddev, wl=None):
'''创建变量
创建变量，并为变量添加L2约束'''
var = tf.Variable(tf.truncated_normal(shape, stddev=stddev))
if wl is not None:
weight_loss = tf.multiply(tf.nn.l2_loss(var), wl, name='weight_loss')
tf.add_to_collection('losses', weight_loss)
return var
##下载数据集
cifar10.maybe_download_and_extract()
##生成增强(distorted)过的训练数据
images_train, labels_train = cifar10_input.distorted_inputs(data_dir=data_dir, batch_size=batch_size)
##生成测试数据，只裁剪，无增强
images_test, labels_test = cifar10_input.inputs(eval_data=True, data_dir=data_dir, batch_size=batch_size)
##创建输入数据的占位符
images_holder = tf.placeholder(dtype=tf.float32, shape=[batch_size, 24, 24, 3])
labels_holder = tf.placeholder(dtype=tf.int32, shape=[batch_size])
#================================↓开始构建正向网络↓==================================
#======================================
# 创建第一个卷积层
def setup():
cifar10.maybe_download_and_extract()
inception.maybe_download()
import cifar10
cifar10.maybe_download_and_extract()
#放在cifar目录下
import cifar10_input
import tensorflow as tf
import pylab
#取数据
batch_size = 12
data_dir = '/tmp/cifar10_data/cifar-10-batches-bin'
images_test, labels_test = cifar10_input.inputs(eval_data=True,
data_dir=data_dir,
batch_size=batch_size)
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
tf.train.start_queue_runners()
image_batch, label_batch = sess.run([images_test, labels_test])
print("__\n", image_batch[0])
print("__\n", label_batch[0])
pylab.imshow(image_batch[0])
pylab.show()
sess = tf.Session()
tf.global_variables_initializer().run(session=sess)
tf.train.start_queue_runners(sess=sess)
image_batch, label_batch = sess.run([images_test, labels_test])
print("__\n", image_batch[0])
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
train()
def main():
NUM_CLASSES = 10
dropout = 0.5
BATCH_SIZE = 128
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
INITIAL_LEARNING_RATE = 0.1
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average.
DISPLAY_FREQ = 20
TRAIN_OR_TEST = 1
# model_name = 'tmp_20160130.pkl'
# model_name = 'data_sync/test20170203.pkl'
#model_name = '20170205.pkl'
model_name = './data/20170209.pkl'
# model_name = 'data_sync/20170206.pkl'
# model_name = 'test.pkl'
# model_name = '../tf_official_docker/tmp.pkl'
PREV_MODEL_EXIST = 1
# cls_train returns as an integer, labels is the array
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
images_train, cls_train, labels_train = cifar10.load_training_data()
t_data = training_data(images_train, labels_train)
DATA_CNT = len(images_train)
NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE
training_data_list = []
weights, biases = initialize_variables(PREV_MODEL_EXIST, model_name)
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.float32, [None, NUM_CLASSES])
keep_prob = tf.placeholder(tf.float32)
images = pre_process(x, TRAIN_OR_TEST)
pred = cov_network(images, weights, biases, keep_prob)
# print(pred)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(pred, y)
loss_value = tf.reduce_mean(cross_entropy)
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()
# global_step = tf.contrib.framework.get_or_create_global_step()
num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE
decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY)
# Decay the learning rate exponentially based on the number of steps.
# lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
# global_step,
# decay_steps,
# LEARNING_RATE_DECAY_FACTOR,
# staircase=True)
#
# opt = tf.train.GradientDescentOptimizer(lr)
# grads = opt.compute_gradients(loss_value)
#
# Apply gradients.
# train_step = opt.apply_gradients(grads, global_step=global_step)
train_step = tf.train.GradientDescentOptimizer(
INITIAL_LEARNING_RATE).minimize(loss_value)
# variable_averages = tf.train.ExponentialMovingAverage(
# MOVING_AVERAGE_DECAY, global_step)
# variables_averages_op = variable_averages.apply(tf.trainable_variables())
init = tf.global_variables_initializer()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# restore model if exists
# if (os.path.isfile("tmp_20160130/model.meta")):
# op = tf.train.import_meta_graph("tmp_20160130/model.meta")
# op.restore(sess,tf.train.latest_checkpoint('tmp_20160130/'))
# print ("model found and restored")
start = time.time()
if TRAIN_OR_TEST == 1:
for i in range(0, 100000):
(batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE)
train_acc, cross_en = sess.run([accuracy, loss_value],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1.0
})
if (i % DISPLAY_FREQ == 0):
print('This is the {}th iteration, time is {}'.format(
i,
time.time() - start))
print("accuracy is {} and cross entropy is {}".format(
train_acc, cross_en))
if (i % (DISPLAY_FREQ * 50) == 0 and i != 0):
save_pkl_model(weights, biases, model_name)
# saver.save(sess, "tmp_20160130/model")
print("saved the network")
_ = sess.run(train_step,
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: dropout
})
images_test, cls_test, labels_test = cifar10.load_test_data()
test_acc = sess.run(accuracy,
feed_dict={
x: images_test,
y: labels_test,
keep_prob: 1.0
})
# save_pkl_model(weights, biases, model_name)
print("test accuracy is {}".format(test_acc))
def main(_):
cifar10.maybe_download_and_extract()
if tf.gfile.Exists(FLAGS.train_dir):
tf.gfile.DeleteRecursively(FLAGS.train_dir)
tf.gfile.MakeDirs(FLAGS.train_dir)
train()
def load_data(self):
cifar10.maybe_download_and_extract()
self.images_train, self.cls_train, self.labels_train = cifar10.load_training_data(
)
self.images_test, self.cls_test, self.labels_test = cifar10.load_test_data(
)
def train():
# Get SVHN dataset
svhn_maybe_download_and_extract()
file_name = os.path.join(FLAGS.svhn_data_dir, "train_32x32.mat")
train = sio.loadmat(file_name)
tr_data_svhn = np.zeros((len(train['y']), 32 * 32 * 3), dtype=float)
tr_label_svhn = np.zeros((len(train['y']), 10), dtype=float)
for i in range(len(train['y'])):
tr_data_svhn[i] = np.reshape(train['X'][:, :, :, i], [1, 32 * 32 * 3])
tr_label_svhn[i, train['y'][i][0] - 1] = 1.0
tr_data_svhn = tr_data_svhn / 255.0
file_name = os.path.join(FLAGS.svhn_data_dir, "test_32x32.mat")
test = sio.loadmat(file_name)
ts_data_svhn = np.zeros((len(test['y']), 32 * 32 * 3), dtype=float)
ts_label_svhn = np.zeros((len(test['y']), 10), dtype=float)
for i in range(len(test['y'])):
ts_data_svhn[i] = np.reshape(test['X'][:, :, :, i], [1, 32 * 32 * 3])
ts_label_svhn[i, test['y'][i][0] - 1] = 1.0
ts_data_svhn = ts_data_svhn / 255.0
data_num_len_svhn = len(tr_label_svhn)
# Get CIFAR 10 dataset
cifar10.maybe_download_and_extract()
tr_label_cifar10 = np.zeros((50000, 10), dtype=float)
ts_label_cifar10 = np.zeros((10000, 10), dtype=float)
for i in range(1, 6):
file_name = os.path.join(FLAGS.cifar_data_dir,
"data_batch_" + str(i) + ".bin")
f = open(file_name, "rb")
data = np.reshape(bytearray(f.read()), [10000, 3073])
if (i == 1):
tr_data_cifar10 = data[:, 1:] / 255.0
else:
tr_data_cifar10 = np.append(tr_data_cifar10,
data[:, 1:] / 255.0,
axis=0)
for j in range(len(data)):
tr_label_cifar10[(i - 1) * 10000 + j, data[j, 0]] = 1.0
file_name = os.path.join(FLAGS.cifar_data_dir, "test_batch.bin")
f = open(file_name, "rb")
data = np.reshape(bytearray(f.read()), [10000, 3073])
for i in range(len(data)):
ts_label_cifar10[i, data[i, 0]] = 1.0
ts_data_cifar10 = data[:, 1:] / 255.0
data_num_len_cifar10 = len(tr_label_cifar10)
if (FLAGS.cifar_first):
tr_data1 = tr_data_cifar10
tr_label1 = tr_label_cifar10
ts_data1 = ts_data_cifar10
ts_label1 = ts_label_cifar10
data_num_len1 = data_num_len_cifar10
tr_data2 = tr_data_svhn
tr_label2 = tr_label_svhn
ts_data2 = ts_data_svhn
ts_label2 = ts_label_svhn
data_num_len2 = data_num_len_svhn
else:
tr_data1 = tr_data_svhn
tr_label1 = tr_label_svhn
ts_data1 = ts_data_svhn
ts_label1 = ts_label_svhn
data_num_len1 = data_num_len_svhn
tr_data2 = tr_data_cifar10
tr_label2 = tr_label_cifar10
ts_data2 = ts_data_cifar10
ts_label2 = ts_label_cifar10
data_num_len2 = data_num_len_cifar10
## TASK 1
sess = tf.InteractiveSession()
# Input placeholders
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, 32 * 32 * 3], name='x-input')
y_ = tf.placeholder(tf.float32, [None, 10], name='y-input')
with tf.name_scope('input_reshape'):
image_shaped_input = tf.reshape(x, [-1, 32, 32, 3])
tf.summary.image('input', image_shaped_input, 2)
# geopath_examples
geopath = pathnet.geopath_initializer(FLAGS.L, FLAGS.M)
# fixed weights list
fixed_list = np.ones((FLAGS.L, FLAGS.M), dtype=str)
for i in range(FLAGS.L):
for j in range(FLAGS.M):
fixed_list[i, j] = '0'
# Hidden Layers
weights_list = np.zeros((FLAGS.L, FLAGS.M), dtype=object)
biases_list = np.zeros((FLAGS.L, FLAGS.M), dtype=object)
# model define
layer_modules_list = np.zeros(FLAGS.M, dtype=object)
# conv layer
i = 0
for j in range(FLAGS.M):
layer_modules_list[j], weights_list[i, j], biases_list[
i,
j] = pathnet.conv_module(image_shaped_input, FLAGS.filt, [5, 5],
geopath[i, j], 1,
'layer' + str(i + 1) + "_" + str(j + 1))
net = np.sum(layer_modules_list) / FLAGS.M
# res layer
i = 1
for j in range(FLAGS.M):
layer_modules_list[j], weights_list[i, j], biases_list[
i, j] = pathnet.res_module(net, geopath[i, j],
'layer' + str(i + 1) + "_" + str(j + 1))
net = np.sum(layer_modules_list) / FLAGS.M
# dimensionality_reduction layer
i = 2
for j in range(FLAGS.M):
layer_modules_list[j], weights_list[i, j], biases_list[
i, j] = pathnet.Dimensionality_reduction_module(
net, 10, geopath[i, j],
'layer' + str(i + 1) + "_" + str(j + 1))
net = np.sum(layer_modules_list) / FLAGS.M
# conv layer
i = 3
for j in range(FLAGS.M):
layer_modules_list[j], weights_list[i, j], biases_list[
i,
j] = pathnet.conv_module(image_shaped_input, FLAGS.filt, [5, 5],
geopath[i, j], 1,
'layer' + str(i + 1) + "_" + str(j + 1))
net = np.sum(layer_modules_list) / FLAGS.M
# output layer
# reshape
_shape = net.shape[1:]
_length = 1
for _i in _shape:
_length *= int(_i)
net = tf.reshape(net, [-1, _length])
# full connection layer
y, output_weights, output_biases = pathnet.nn_layer(
net, 10, 'output_layer')
# Cross Entropy
with tf.name_scope('cross_entropy'):
diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
with tf.name_scope('total'):
cross_entropy = tf.reduce_mean(diff)
tf.summary.scalar('cross_entropy', cross_entropy)
# Need to learn variables
var_list_to_learn = [] + output_weights + output_biases
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '0'):
var_list_to_learn += weights_list[i, j] + biases_list[i, j]
# GradientDescent
with tf.name_scope('train'):
train_step = tf.train.GradientDescentOptimizer(
FLAGS.learning_rate).minimize(cross_entropy,
var_list=var_list_to_learn)
# Accuracy
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Merge all the summaries and write them out to /tmp/tensorflow/mnist/logs/mnist_with_summaries (by default)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train1', sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test1')
tf.global_variables_initializer().run()
# Generating randomly geopath
geopath_set = np.zeros(FLAGS.candi, dtype=object)
for i in range(FLAGS.candi):
geopath_set[i] = pathnet.get_geopath(FLAGS.L, FLAGS.M, FLAGS.N)
# parameters placeholders and ops
var_update_ops = np.zeros(len(var_list_to_learn), dtype=object)
var_update_placeholders = np.zeros(len(var_list_to_learn), dtype=object)
for i in range(len(var_list_to_learn)):
var_update_placeholders[i] = tf.placeholder(
var_list_to_learn[i].dtype, shape=var_list_to_learn[i].get_shape())
var_update_ops[i] = var_list_to_learn[i].assign(
var_update_placeholders[i])
# geopathes placeholders and ops
geopath_update_ops = np.zeros((len(geopath), len(geopath[0])),
dtype=object)
geopath_update_placeholders = np.zeros((len(geopath), len(geopath[0])),
dtype=object)
for i in range(len(geopath)):
for j in range(len(geopath[0])):
geopath_update_placeholders[i, j] = tf.placeholder(
geopath[i, j].dtype, shape=geopath[i, j].get_shape())
geopath_update_ops[i, j] = geopath[i, j].assign(
geopath_update_placeholders[i, j])
acc_geo = np.zeros(FLAGS.B, dtype=float)
summary_geo = np.zeros(FLAGS.B, dtype=object)
for i in range(FLAGS.max_steps):
# Select Candidates to Tournament
compet_idx = range(FLAGS.candi)
np.random.shuffle(compet_idx)
compet_idx = compet_idx[:FLAGS.B]
# Learning & Evaluating
for j in range(len(compet_idx)):
# Shuffle the data
idx = range(len(tr_data1))
np.random.shuffle(idx)
tr_data1 = tr_data1[idx]
tr_label1 = tr_label1[idx]
# Insert Candidate
pathnet.geopath_insert(sess, geopath_update_placeholders,
geopath_update_ops,
geopath_set[compet_idx[j]], FLAGS.L,
FLAGS.M)
acc_geo_tr = 0
for k in range(FLAGS.T):
'''
print(x.shape)
print(tr_data1[k*FLAGS.batch_num:(k+1)*FLAGS.batch_num,:].shape)
print(y.shape)
print(tr_label1[k*FLAGS.batch_num:(k+1)*FLAGS.batch_num,:].shape)
'''
summary_geo_tr, _, acc_geo_tmp = sess.run(
[merged, train_step, accuracy],
feed_dict={
x:
tr_data1[k * FLAGS.batch_num:(k + 1) *
FLAGS.batch_num, :],
y_:
tr_label1[k * FLAGS.batch_num:(k + 1) *
FLAGS.batch_num, :]
})
acc_geo_tr += acc_geo_tmp
acc_geo[j] = acc_geo_tr / FLAGS.T
summary_geo[j] = summary_geo_tr
# Tournament
winner_idx = np.argmax(acc_geo)
acc = acc_geo[winner_idx]
summary = summary_geo[winner_idx]
# Copy and Mutation
for j in range(len(compet_idx)):
if (j != winner_idx):
geopath_set[compet_idx[j]] = np.copy(
geopath_set[compet_idx[winner_idx]])
geopath_set[compet_idx[j]] = pathnet.mutation(
geopath_set[compet_idx[j]], FLAGS.L, FLAGS.M, FLAGS.N)
train_writer.add_summary(summary, i)
print('Training Accuracy at step %s: %s' % (i, acc))
acc_task1 = acc
task1_optimal_path = geopath_set[compet_idx[winner_idx]]
# Fix task1 Optimal Path
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (task1_optimal_path[i, j] == 1.0):
fixed_list[i, j] = '1'
# Get variables of fixed list
var_list_to_fix = []
#var_list_to_fix=[]+output_weights+output_biases;
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '1'):
var_list_to_fix += weights_list[i, j] + biases_list[i, j]
var_list_fix = pathnet.parameters_backup(var_list_to_fix)
# parameters placeholders and ops
var_fix_ops = np.zeros(len(var_list_to_fix), dtype=object)
var_fix_placeholders = np.zeros(len(var_list_to_fix), dtype=object)
for i in range(len(var_list_to_fix)):
var_fix_placeholders[i] = tf.placeholder(
var_list_to_fix[i].dtype, shape=var_list_to_fix[i].get_shape())
var_fix_ops[i] = var_list_to_fix[i].assign(var_fix_placeholders[i])
## TASK 2
# Need to learn variables
var_list_to_learn = [] + output_weights + output_biases
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '0'):
var_list_to_learn += weights_list[i, j] + biases_list[i, j]
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '1'):
tmp = biases_list[i, j][0]
break
break
# Initialization
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train2', sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test2')
tf.global_variables_initializer().run()
# Update fixed values
pathnet.parameters_update(sess, var_fix_placeholders, var_fix_ops,
var_list_fix)
# GradientDescent
with tf.name_scope('train'):
train_step = tf.train.GradientDescentOptimizer(
FLAGS.learning_rate).minimize(cross_entropy,
var_list=var_list_to_learn)
# Generating randomly geopath
geopath_set = np.zeros(FLAGS.candi, dtype=object)
for i in range(FLAGS.candi):
geopath_set[i] = pathnet.get_geopath(FLAGS.L, FLAGS.M, FLAGS.N)
# parameters placeholders and ops
var_update_ops = np.zeros(len(var_list_to_learn), dtype=object)
var_update_placeholders = np.zeros(len(var_list_to_learn), dtype=object)
for i in range(len(var_list_to_learn)):
var_update_placeholders[i] = tf.placeholder(
var_list_to_learn[i].dtype, shape=var_list_to_learn[i].get_shape())
var_update_ops[i] = var_list_to_learn[i].assign(
var_update_placeholders[i])
acc_geo = np.zeros(FLAGS.B, dtype=float)
summary_geo = np.zeros(FLAGS.B, dtype=object)
for i in range(FLAGS.max_steps):
# Select Candidates to Tournament
compet_idx = range(FLAGS.candi)
np.random.shuffle(compet_idx)
compet_idx = compet_idx[:FLAGS.B]
# Learning & Evaluating
for j in range(len(compet_idx)):
# Shuffle the data
idx = range(len(tr_data2))
np.random.shuffle(idx)
tr_data2 = tr_data2[idx]
tr_label2 = tr_label2[idx]
geopath_insert = np.copy(geopath_set[compet_idx[j]])
for l in range(FLAGS.L):
for m in range(FLAGS.M):
if (fixed_list[l, m] == '1'):
geopath_insert[l, m] = 1.0
# Insert Candidate
pathnet.geopath_insert(sess, geopath_update_placeholders,
geopath_update_ops, geopath_insert, FLAGS.L,
FLAGS.M)
acc_geo_tr = 0
for k in range(FLAGS.T):
summary_geo_tr, _, acc_geo_tmp = sess.run(
[merged, train_step, accuracy],
feed_dict={
x:
tr_data2[k * FLAGS.batch_num:(k + 1) *
FLAGS.batch_num, :],
y_:
tr_label2[k * FLAGS.batch_num:(k + 1) *
FLAGS.batch_num, :]
})
acc_geo_tr += acc_geo_tmp
acc_geo[j] = acc_geo_tr / FLAGS.T
summary_geo[j] = summary_geo_tr
# Tournament
winner_idx = np.argmax(acc_geo)
acc = acc_geo[winner_idx]
summary = summary_geo[winner_idx]
# Copy and Mutation
for j in range(len(compet_idx)):
if (j != winner_idx):
geopath_set[compet_idx[j]] = np.copy(
geopath_set[compet_idx[winner_idx]])
geopath_set[compet_idx[j]] = pathnet.mutation(
geopath_set[compet_idx[j]], FLAGS.L, FLAGS.M, FLAGS.N)
train_writer.add_summary(summary, i)
print('Training Accuracy at step %s: %s' % (i, acc))
acc_task2 = acc
if (FLAGS.cifar_first):
print("CIFAR10_SVHN,TASK1:" + str(acc_task1) + ",TASK2:" +
str(acc_task2) + ",Done")
else:
print("SVHN_CIFAR10,TASK1:" + str(acc_task1) + ",TASK2:" +
str(acc_task2) + ",Done")
train_writer.close()
test_writer.close()
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if not tf.gfile.Exists(FLAGS.train_dir):
tf.gfile.MakeDirs(FLAGS.train_dir)
train()
def simple_nettest():
cifar10.maybe_download_and_extract()
images_test, cls_test, labels_test = cifar10.load_test_data()
images_test.astype(np.float32)
model_params = {
"conv1": [5,5, 64],
"conv2": [3,3,128],
"inception_1":{
"1x1":64,
"3x3":{ "1x1":96,
"3x3":128
},
"5x5":{ "1x1":16,
"5x5":32
},
"s1x1":32
},
"fc3": 128,
"fc4": 10
}
with tf.variable_scope("test"):
x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='x')
net = convolution_layer(x, model_params["conv1"], [1,2,2,1],name="conv1")
net = convolution_layer(net, model_params["conv2"], [1,1,1,1],name="conv2", flatten=False)
net = inception_v1(net, model_params, name= "inception_1", flatten=True)
net = fc_layer(net, model_params["fc3"], name="fc3")
net = fc_layer(net, model_params["fc4"], name="fc4", activat_fn=None)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
netout = sess.run(net, feed_dict={x:images_test[0:5,:,:,:]})
ut.print_operations_in_graph()
return netout
#simpletest = simple_nettest()
def get_train_data(data_dir, batch_size):
cifar10.maybe_download_and_extract(data_dir)
images, labels = cifar10.get_distorted_train_batch(dataset_dir, batch_size)
images = tf.image.resize_images(images=images,
size=[image_size, image_size])
return images, labels
def main(argv=None):
print("cccc"+str(tf.__version__))
cifar10.maybe_download_and_extract()
train()
def backward():
#加载图片及图片batch张量
cifar10.maybe_download_and_extract()
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir=DATA_DIR, batch_size=BATCH_SIZE)
#构建反馈
x = tf.placeholder(tf.float32, shape=[None, 24, 24, 3])
y_ = tf.placeholder(tf.float32, shape=[None])
keep_prob = tf.placeholder(tf.float32)
y = cnn_forward.forward(x, keep_prob, REGULARIZER)
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
20000 / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
#计算loss
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.cast(
y_, tf.int64))
loss_ce = tf.reduce_mean(ce)
loss_total = loss_ce + tf.add_n(tf.get_collection('losses'))
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss_total, global_step=global_step)
#滑动平均
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
#训练模型
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
sess = tf.InteractiveSession()
init_op = tf.global_variables_initializer()
sess.run(init_op)
#续训
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
#启用多线程队列加载图片
tf.train.start_queue_runners()
for i in range(STEPS):
image_batch, label_batch = sess.run([images_train, labels_train])
_, loss_value, step = sess.run([train_op, loss_total, global_step],
feed_dict={
x: image_batch,
y_: label_batch,
keep_prob: 0.5
})
if i % 10 == 0:
print("After %d steps, loss is: %f" % (step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
sess.close()
def main():
cifar10.maybe_download_and_extract()
def main(argv=None):
if (argv is None):
argv = sys.argv
try:
try:
opts = argv
TRAIN = 1
pretrain = 0
for item in opts:
print(item)
opt = item[0]
val = item[1]
if (opt == '-t'):
TRAIN = val
if (opt == '-q_bits'):
q_bits = val
if (opt == '-parent_dir'):
parent_dir = val
if (opt == '-base_model'):
base_model = val
if (opt == '-pretrain'):
pretrain = val
print('pretrain is {}'.format(pretrain))
except getopt.error, msg:
raise Usage(msg)
NUM_CLASSES = 10
dropout = 0.8 # probability of keep
BATCH_SIZE = 128
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
INITIAL_LEARNING_RATE = 0.001
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average.
DISPLAY_FREQ = 20
TEST = 0
TRAIN_OR_TEST = 0
NUM_CHANNELS = 3
DOCKER = 0
mask_dir = parent_dir
model_dir = parent_dir
PREV_MODEL_EXIST = 1
(weights_mask, biases_mask) = initialize_weights_mask(
0, mask_dir + 'masks/' + 'base.pkl')
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
t_data = format_data(images_train, labels_train)
test_data = format_data(images_test, labels_test)
DATA_CNT = len(images_train)
NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE
training_data_list = []
weights, biases, dynamic_range = initialize_variables(
PREV_MODEL_EXIST, parent_dir, q_bits, pretrain)
weights, biases = compute_weights_nbits(weights, biases, q_bits,
dynamic_range)
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.float32, [None, NUM_CLASSES])
if (TRAIN == 1):
TRAIN_OR_TEST = 1
else:
TRAIN_OR_TEST = 0
keep_prob = tf.placeholder(tf.float32)
images = pre_process(x, TRAIN_OR_TEST)
test_images = pre_process(x, 0)
pred = cov_network(images, weights, biases, keep_prob)
test_pred = cov_network(test_images, weights, biases, keep_prob)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=pred,
labels=y)
loss_value = tf.reduce_mean(cross_entropy)
correct_prediction = tf.equal(tf.argmax(test_pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# global_step = tf.contrib.framework.get_or_create_global_step()
#
# num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE
# decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY)
#
# # Decay the learning rate exponentially based on the number of steps.
# lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
# global_step,
# decay_steps,
# LEARNING_RATE_DECAY_FACTOR,
# staircase=True)
#
# opt = tf.train.GradientDescentOptimizer(lr)
# grads = opt.compute_gradients(loss_value)
# org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads]
# new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask)
# # Apply gradients.
# train_step = opt.apply_gradients(new_grads, global_step=global_step)
train_step = tf.train.AdamOptimizer(1e-4).minimize(loss_value)
init = tf.global_variables_initializer()
accuracy_list = np.zeros(30)
accuracy_list = np.zeros(10)
# Launch the graph
print('Graph launching ..')
best_test_acc = 0
with tf.Session() as sess:
sess.run(init)
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
# for key in keys:
# sess.run(weights[key].assign(weights[key].eval()*weights_mask[key]))
# sess.run(biases[key].assign(biases[key].eval()*biases_mask[key]))
print('pre train pruning info')
prune_info(weights, 0)
print(78 * '-')
print('start save these pre trained weights')
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
weights_save = {}
biases_save = {}
for key in keys:
weights_save[key] = weights[key].eval()
biases_save[key] = biases[key].eval()
with open(
parent_dir + 'weights/' + 'weights' + str(q_bits) + '.pkl',
'wb') as f:
pickle.dump((weights_save, biases_save), f)
start = time.time()
if TRAIN == 1:
for i in range(0, 60000):
(batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE)
train_acc, cross_en = sess.run([accuracy, loss_value],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1.0
})
if (i % DISPLAY_FREQ == 0):
# prune_info(weights, 0)
print('This is the {}th iteration, time is {}'.format(
i,
time.time() - start))
print("accuracy is {} and cross entropy is {}".format(
train_acc, cross_en))
# accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:29]))
accuracy_list = np.concatenate(
(np.array([train_acc]), accuracy_list[0:9]))
if (np.mean(accuracy_list) > 0.8):
print(
"training accuracy is large, show the list: {}"
.format(accuracy_list))
NUMBER_OF_BATCH = 10000 / BATCH_SIZE
t_acc = []
for i in range(0, NUMBER_OF_BATCH):
(batch_x, batch_y
) = test_data.feed_next_batch(BATCH_SIZE)
test_acc = sess.run(accuracy,
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1.0
})
t_acc.append(test_acc)
print("test accuracy is {}".format(t_acc))
test_acc = np.mean(t_acc)
accuracy_list = np.zeros(10)
if (q_bits == 2):
threshold = 0.5
if (q_bits == 4):
threshold = 0.8
if (q_bits == 8):
threshold = 0.82
if (q_bits >= 9):
threshold = 0.82
print('test accuracy is {}'.format(test_acc))
if (test_acc > threshold
or test_acc > best_test_acc):
best_test_acc = test_acc
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
weights_save = {}
biases_save = {}
for key in keys:
weights_save[key] = weights[key].eval()
biases_save[key] = biases[key].eval()
with open(
parent_dir + 'weights/' + 'weights' +
str(q_bits) + '.pkl', 'wb') as f:
pickle.dump((weights_save, biases_save), f)
if (best_test_acc > threshold):
print(
'Exiting the training, test accuracy is {}'
.format(test_acc))
return best_test_acc
_ = sess.run(train_step,
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: dropout
})
if (TRAIN == 1):
if (best_test_acc == 0):
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
weights_save = {}
biases_save = {}
for key in keys:
weights_save[key] = weights[key].eval()
biases_save[key] = biases[key].eval()
with open(
parent_dir + 'weights/' + 'weights' + str(q_bits) +
'.pkl', 'wb') as f:
pickle.dump((weights_save, biases_save), f)
if (best_test_acc == 0):
NUMBER_OF_BATCH = 10000 / BATCH_SIZE
t_acc = []
for i in range(0, NUMBER_OF_BATCH):
(batch_x, batch_y) = test_data.feed_next_batch(BATCH_SIZE)
test_acc = sess.run(accuracy,
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1.0
})
t_acc.append(test_acc)
print("test accuracy is {}".format(t_acc))
# save_pkl_model(weights, biases, model_name)
return np.mean(t_acc)
else:
return best_test_acc
# coding=utf-8
# cnn_l2
from __future__ import division
import cifar10, cifar10_input
import tensorflow as tf
import numpy as np
import math
import time
data_dir = 'cifar10_data/cifar-10-batches-bin' # 下载 CIFAR-10 的默认路径
cifar10.maybe_download_and_extract() # 下载数据集，并解压、展开到其默认位置
batch_size = 128
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=batch_size)
images_test, labels_test = cifar10_input.inputs(eval_data=True,
data_dir=data_dir,
batch_size=batch_size)
def weight_variable(shape, stddev, w1):
var = tf.Variable(tf.truncated_normal(shape,
stddev=stddev)) # stddev=stddev！！！
if w1:
weight_loss = tf.multiply(tf.nn.l2_loss(var), w1, name='weight_loss')
tf.add_to_collection('losses', weight_loss)
return var
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if os.path.exists(FLAGS.train_dir):
shutil.rmtree(FLAGS.train_dir)
os.makedirs(FLAGS.train_dir)
train()
batch_size = 128
data_dir = '/tmp/cifar10_data/cifar-10-batches-bin'
def variable_with_weight_loss(shape, stddev, w1):
var = tf.Variable(tf.truncated_normal(shape,
stddev=stddev)) #使用截断的正态分布来初始化权重
if w1 is not None:
weight_loss = tf.multiply(
tf.nn.l2_loss(var), w1,
name='weight_loss') #做l2的正则化，用w1来控制l2 loss的大小
tf.add_to_collection('losses', weight_loss) #把结果保存在一个collection
return var
cifar10.maybe_download_and_extract() #使用cifar10类下载数据集，并解压，展开到默认位置
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=batch_size)
images_test, labels_test = cifar10_input.inputs(eval_data=True,
data_dir=data_dir,
batch_size=batch_size) #生成测试数据
image_holder = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
label_holder = tf.placeholder(tf.int32, [batch_size])
#第一层
weight1 = variable_with_weight_loss(
shape=[5, 5, 3, 64], stddev=5e-2,
w1=0.0) #第一层卷积核使用5乘5的卷积核输入通道为3输出通道为64 不使用l2正则化因此w1设为0
kernel1 = tf.nn.conv2d(image_holder, weight1, [1, 1, 1, 1],
padding='SAME') #卷积
bias1 = tf.Variable(tf.constant(0.0, shape=[64])) #设置偏置
def main(argv=None):
cifar10.maybe_download_and_extract()
if tf.gfile.Exists(FLAGS.eval_dir):
tf.gfile.DeleteRecursively(FLAGS.eval_dir)
tf.gfile.MakeDirs(FLAGS.eval_dir)
evaluate()
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if tf.gfile.Exists('/tmp/cifar10_train'):
tf.gfile.DeleteRecursively('/tmp/cifar10_train')
tf.gfile.MakeDirs('/tmp/cifar10_train')
train()
def main(argv=None):
cifar10.maybe_download_and_extract()
train()
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract(args)
if tf.gfile.Exists(args.train_dir):
tf.gfile.DeleteRecursively(args.train_dir)
tf.gfile.MakeDirs(args.train_dir)
train(args)
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if tf.gfile.Exists(FLAGS.eval_dir):
tf.gfile.DeleteRecursively(FLAGS.eval_dir)
tf.gfile.MakeDirs(FLAGS.eval_dir)
evaluate()
def main(argv=None):
if (argv is None):
argv = sys.argv
try:
try:
opts = argv
for item in opts:
print(item)
opt = item[0]
val = item[1]
if (opt == '-pcov'):
pruning_cov = val
if (opt == '-pfc'):
pruning_fc = val
print('pruning count is {}, {}'.format(pruning_cov, pruning_fc))
except getopt.error, msg:
raise Usage(msg)
NUM_CLASSES = 10
dropout = 0.5
BATCH_SIZE = 128
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
INITIAL_LEARNING_RATE = 0.001
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average.
DISPLAY_FREQ = 20
TRAIN = 0
TEST = 1
TRAIN_OR_TEST = 0
NUM_CHANNELS = 3
DOCKER = 0
# model_name = 'tmp_20160130.pkl'
# model_name = 'data_sync/test20170203.pkl'
#model_name = '20170205.pkl'
# model_name = '20170206.pkl'
if (DOCKER == 1):
# base_model_name = '/root/data/20170206.pkl'
# model_name = '/root/data/pruning.pkl'
# mask_dir = '/root/data/mask.pkl'
base_model_name = '/root/20170206.pkl'
model_name = '/root/pruning'
mask_dir = '/root/mask'
else:
parent_dir = './data/'
parent_dir = '/Users/aaron/Projects/Mphil_project/tmp_Cifar10_quantisation_Han/'
mask_dir = parent_dir + 'mask'
base_model_name = './data/20170206.pkl'
model_name = parent_dir + '/pruning'
# model_name = 'test.pkl'
# model_name = '../tf_official_docker/tmp.pkl'
PREV_MODEL_EXIST = 1
# cls_train returns as an integer, labels is the array
if (pruning_fc == 0 and pruning_cov == 0):
print("It's first time loading!")
first_time_load = 1
else:
first_time_load = 0
print('pruning on cov is {}. on fc is {}'.format(
pruning_cov, pruning_fc))
(weights_mask, biases_mask) = initialize_weights_mask(
first_time_load, mask_dir + 'v' + str(pruning_cov - 10) +
str(pruning_fc - 10) + '.pkl')
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
t_data = training_data(images_train, labels_train)
DATA_CNT = len(images_train)
NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE
training_data_list = []
if (first_time_load == 1):
weights, biases = initialize_variables(PREV_MODEL_EXIST,
base_model_name)
else:
weights, biases = initialize_variables(
PREV_MODEL_EXIST, model_name + 'v' + str(pruning_cov - 10) +
str(pruning_fc - 10) + '.pkl')
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.float32, [None, NUM_CLASSES])
if (first_time_load == 1):
TRAIN = 0
if (TRAIN == 1):
TRAIN_OR_TEST = 1
else:
TRAIN_OR_TEST = 0
keep_prob = tf.placeholder(tf.float32)
images = pre_process(x, TRAIN_OR_TEST)
# images = pre_process(x, 1)
pred = cov_network(images, weights, biases, keep_prob)
# print(pred)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(pred, y)
loss_value = tf.reduce_mean(cross_entropy)
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()
global_step = tf.contrib.framework.get_or_create_global_step()
num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE
decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY)
# Decay the learning rate exponentially based on the number of steps.
lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
global_step,
decay_steps,
LEARNING_RATE_DECAY_FACTOR,
staircase=True)
opt = tf.train.GradientDescentOptimizer(lr)
grads = opt.compute_gradients(loss_value)
org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads]
new_grads = mask_gradients(weights, org_grads, weights_mask, biases,
biases_mask)
#
# Apply gradients.
train_step = opt.apply_gradients(new_grads, global_step=global_step)
# train_step = tf.train.GradientDescentOptimizer(INITIAL_LEARNING_RATE).minimize(loss_value)
# variable_averages = tf.train.ExponentialMovingAverage(
# MOVING_AVERAGE_DECAY, global_step)
# variables_averages_op = variable_averages.apply(tf.trainable_variables())
init = tf.global_variables_initializer()
accuracy_list = np.zeros(30)
accuracy_list = np.zeros(5)
# Launch the graph
print('Graph launching ..')
with tf.Session() as sess:
sess.run(init)
# restore model if exists
# if (os.path.isfile("tmp_20160130/model.meta")):
# op = tf.train.import_meta_graph("tmp_20160130/model.meta")
# op.restore(sess,tf.train.latest_checkpoint('tmp_20160130/'))
# print ("model found and restored")
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
for key in keys:
sess.run(weights[key].assign(weights[key].eval() *
weights_mask[key]))
sess.run(biases[key].assign(biases[key].eval() *
biases_mask[key]))
print('pre train pruning info')
prune_info(weights, 0)
print(78 * '-')
start = time.time()
if TRAIN == 1:
for i in range(0, 60000):
(batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE)
train_acc, cross_en = sess.run([accuracy, loss_value],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1.0
})
if (i % DISPLAY_FREQ == 0):
# prune_info(weights, 0)
print(
'This is the {}th iteration of {},{}pruning, time is {}'
.format(i, pruning_cov, pruning_fc,
time.time() - start))
print("accuracy is {} and cross entropy is {}".format(
train_acc, cross_en))
# accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:29]))
accuracy_list = np.concatenate(
(np.array([train_acc]), accuracy_list[0:4]))
if (i % (DISPLAY_FREQ * 50) == 0 and i != 0):
save_pkl_model(
weights, biases, model_name + 'v' +
str(pruning_cov) + str(pruning_fc) + '.pkl')
print("saved the network")
# if (np.mean(train_acc) > 0.5):
if (np.mean(accuracy_list) > 0.8):
print(
"training accuracy is large, show the list: {}"
.format(accuracy_list))
test_acc = sess.run(accuracy,
feed_dict={
x: images_test,
y: labels_test,
keep_prob: 1.0
})
# accuracy_list = np.zeros(30)
accuracy_list = np.zeros(5)
print('test accuracy is {}'.format(test_acc))
if (test_acc > 0.823):
print(
'Exiting the training, test accuracy is {}'
.format(test_acc))
break
_ = sess.run(train_step,
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: dropout
})
print('hi?')
if (TEST == 1):
test_acc = sess.run(accuracy,
feed_dict={
x: images_test,
y: labels_test,
keep_prob: 1.0
})
print("test accuracy is {}".format(test_acc))
# save_pkl_model(weights, biases, model_name)
print('saving pruned model ...')
prune_weights(
pruning_cov, pruning_fc, weights, weights_mask,
mask_dir + 'v' + str(pruning_cov) + str(pruning_fc) + '.pkl',
biases, biases_mask)
save_pkl_model(
weights, biases,
model_name + 'v' + str(pruning_cov) + str(pruning_fc) + '.pkl')
return test_acc
def main(argv=None):
cifar10.maybe_download_and_extract()
if gfile.Exists(FLAGS.train_dir):
gfile.DeleteRecursively(FLAGS.train_dir)
gfile.MakeDirs(FLAGS.train_dir)
train()
def main():
args = get_arguments()
allvars = get_all_model_variables(args)
# Load Test Dataset
if (allvars['model2load'] == 'fcnn') or (allvars['model2load'] == 'lenet'):
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets(allvars['data_dir'], one_hot=True)
X = mnist.test.images
y = mnist.test.labels
labels_dict = [
'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven',
'eight', 'nine'
]
# Free Memory
mnist = None
if (allvars['model2load'] == 'nin') or (allvars['model2load']
== 'densenet'):
import cifar10
cifar10.data_path = allvars['data_dir']
cifar10.maybe_download_and_extract()
X, _, y = cifar10.load_test_data()
labels_dict = [
'airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog',
'horse', 'ship', 'truck'
]
# Free Memory
cifar = None
X, y = pre_process_data(X, y, allvars['model2load'])
X, y = collect_correctly_predicted_images(X, y, allvars)
eps_rescale = np.max(np.abs(np.max(X.flatten()) - np.min(X.flatten())))
N = get_adversarial_noise(X,
y,
allvars['epsilon'] * eps_rescale,
allvars,
method=allvars['method'])
y_adv, _ = predict_CNN(X + N, allvars)
import scipy
# scipy.misc.imsave('outfile.jpg', image_array*255.)
t = 0
eps = allvars['epsilon']
for i in range(X.shape[0]):
Ximage = (X / eps_rescale + np.min(X.flatten()))
Nimage = (N / eps_rescale / eps / 2 + 0.5)
Xadv = (1 - 2 * eps) * Ximage + eps + 2 * eps * (Nimage - 0.5)
if y_adv[i] != y[i]:
scipy.misc.imsave(
allvars['out_dir'] + str(int(t)) + '_Original_' +
labels_dict[y[i]] + '.eps',
np.squeeze(Ximage[i, :, :, :]) * 255.)
scipy.misc.imsave(
allvars['out_dir'] + str(int(t)) + '_Noise_' +
labels_dict[y[i]] + '.eps',
np.squeeze(Nimage[i, :, :, :]) * 255.)
scipy.misc.imsave(
allvars['out_dir'] + str(int(t)) + '_Adversarial_' +
labels_dict[y_adv[i]] + '.eps',
np.squeeze(Xadv[i, :, :, :]) * 255.)
t = t + 1
print(t)
if t >= allvars['n_images']:
break
def main(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if gfile.Exists(FLAGS.train_dir):
gfile.DeleteRecursively(FLAGS.train_dir)
gfile.MakeDirs(FLAGS.train_dir)
train()
def __init__(self, batch_size=100,testProp=0.3, validation_proportion=0.3,seed = 1, normalize=True):
self.cifarFolder = "data/cifar10"
cifar10.data_path = self.cifarFolder
cifar10.maybe_download_and_extract()
Dataset.__init__(self,self.cifarFolder,batch_size=batch_size,testProp=testProp, validation_proportion=validation_proportion,seed = seed, normalize=normalize)
def train():
# Get SVHN dataset
svhn_maybe_download_and_extract()
file_name = os.path.join(FLAGS.svhn_data_dir, "train_32x32.mat")
train = sio.loadmat(file_name)
tr_data_svhn = np.zeros((len(train['y']), 32 * 32 * 3), dtype=float)
tr_label_svhn = np.zeros((len(train['y']), 10), dtype=float)
for i in range(len(train['y'])):
tr_data_svhn[i] = np.reshape(train['X'][:, :, :, i], [1, 32 * 32 * 3])
tr_label_svhn[i, train['y'][i][0] - 1] = 1.0
tr_data_svhn = tr_data_svhn / 255.0
tr_label_svhn = np.zeros((len(train['y']), 10), dtype=float)
file_name = os.path.join(FLAGS.svhn_data_dir, "test_32x32.mat")
test = sio.loadmat(file_name)
ts_data_svhn = np.zeros((len(test['y']), 32 * 32 * 3), dtype=float)
ts_label_svhn = np.zeros((len(test['y']), 10), dtype=float)
for i in range(len(test['y'])):
ts_data_svhn[i] = np.reshape(test['X'][:, :, :, i], [1, 32 * 32 * 3])
ts_label_svhn[i, test['y'][i][0] - 1] = 1.0
ts_data_svhn = ts_data_svhn / 255.0
data_num_len_svhn = len(tr_label_svhn)
# Get CIFAR 10 dataset
cifar10.maybe_download_and_extract()
tr_label_cifar10 = np.zeros((50000, 10), dtype=float)
ts_label_cifar10 = np.zeros((10000, 10), dtype=float)
for i in range(1, 6):
file_name = os.path.join(FLAGS.cifar_data_dir,
"data_batch_" + str(i) + ".bin")
f = open(file_name, "rb")
data = np.reshape(bytearray(f.read()), [10000, 3073])
if (i == 1):
tr_data_cifar10 = data[:, 1:] / 255.0
else:
tr_data_cifar10 = np.append(tr_data_cifar10,
data[:, 1:] / 255.0,
axis=0)
for j in range(len(data)):
tr_label_cifar10[(i - 1) * 10000 + j, data[j, 0]] = 1.0
file_name = os.path.join(FLAGS.cifar_data_dir, "test_batch.bin")
f = open(file_name, "rb")
data = np.reshape(bytearray(f.read()), [10000, 3073])
for i in range(len(data)):
ts_label_cifar10[i, data[i, 0]] = 1.0
ts_data_cifar10 = data[:, 1:] / 255.0
data_num_len_cifar10 = len(tr_label_cifar10)
tr_data1 = tr_data_svhn
tr_label1 = tr_label_svhn
ts_data1 = ts_data_svhn
ts_label1 = ts_label_svhn
data_num_len1 = data_num_len_svhn
tr_data2 = tr_data_cifar10
tr_label2 = tr_label_cifar10
ts_data2 = ts_data_cifar10
ts_label2 = ts_label_cifar10
data_num_len2 = data_num_len_cifar10
## TASK 1 (SVHN)
sess = tf.InteractiveSession()
# Create a multilayer model.
# Input placeholders
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, 32 * 32 * 3], name='x-input')
y_ = tf.placeholder(tf.float32, [None, 10], name='y-input')
with tf.name_scope('input_reshape'):
image_shaped_input = tf.reshape(x, [-1, 32, 32, 3])
tf.summary.image('input', image_shaped_input, 10)
# geopath_examples
geopath = pathnet.geopath_initializer(FLAGS.L, FLAGS.M)
# fixed weights list
fixed_list = np.ones((FLAGS.L, FLAGS.M), dtype=str)
for i in range(FLAGS.L):
for j in range(FLAGS.M):
fixed_list[i, j] = '0'
# reinitializing weights list
rein_list = np.ones((FLAGS.L, FLAGS.M), dtype=str)
for i in range(FLAGS.L):
for j in range(FLAGS.M):
rein_list[i, j] = '0'
# Input Layer
"""
input_weights=pathnet.module_weight_variable([784,FLAGS.filt]);
input_biases=pathnet.module_bias_variable([FLAGS.filt]);
net = pathnet.nn_layer(x,input_weights,input_biases,'input_layer');
"""
# Hidden Layers
weights_list = np.zeros((FLAGS.L, FLAGS.M), dtype=object)
biases_list = np.zeros((FLAGS.L, FLAGS.M), dtype=object)
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (i == 0):
weights_list[i, j] = pathnet.module_weight_variable(
[32 * 32 * 3, FLAGS.filt])
biases_list[i, j] = pathnet.module_bias_variable([FLAGS.filt])
else:
weights_list[i, j] = pathnet.module_weight_variable(
[FLAGS.filt, FLAGS.filt])
biases_list[i, j] = pathnet.module_bias_variable([FLAGS.filt])
for i in range(FLAGS.L):
layer_modules_list = np.zeros(FLAGS.M, dtype=object)
for j in range(FLAGS.M):
if (i == 0):
layer_modules_list[j] = pathnet.module(
x, weights_list[i, j], biases_list[i, j],
'layer' + str(i + 1) + "_" + str(j + 1)) * geopath[i, j]
else:
layer_modules_list[j] = pathnet.module(
net, weights_list[i, j], biases_list[i, j],
'layer' + str(i + 1) + "_" + str(j + 1)) * geopath[i, j]
net = np.sum(layer_modules_list)
"""
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
tf.summary.scalar('dropout_keep_probability', keep_prob)
dropped = tf.nn.dropout(hidden1, keep_prob)
"""
# Do not apply softmax activation yet, see below.
output_weights = pathnet.module_weight_variable([FLAGS.filt, 10])
output_biases = pathnet.module_bias_variable([10])
y = pathnet.nn_layer(net,
output_weights,
output_biases,
'output_layer',
act=tf.identity)
with tf.name_scope('cross_entropy'):
diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
with tf.name_scope('total'):
cross_entropy = tf.reduce_mean(diff)
tf.summary.scalar('cross_entropy', cross_entropy)
# Need to learn variables
#var_list_to_learn=[]+input_weights+input_biases+output_weights+output_biases;
var_list_to_learn = [] + output_weights + output_biases
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '0'):
var_list_to_learn += weights_list[i, j] + biases_list[i, j]
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(
cross_entropy, var_list=var_list_to_learn)
def feed_dict(train, tr_flag=0):
#Make a TensorFlow feed_dict: maps data onto Tensor placeholders.
if train or FLAGS.fake_data:
xs = tr_data1[tr_flag:tr_flag + 16, :]
ys = tr_label1[tr_flag:tr_flag + 16, :]
k = FLAGS.dropout
else:
xs = ts_data1
ys = ts_label1
k = 1.0
return {x: xs, y_: ys}
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Merge all the summaries and write them out to /tmp/tensorflow/mnist/logs/mnist_with_summaries (by default)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test')
tf.global_variables_initializer().run()
# Generating randomly geopath
geopath_set = np.zeros(FLAGS.candi, dtype=object)
for i in range(FLAGS.candi):
geopath_set[i] = pathnet.get_geopath(FLAGS.L, FLAGS.M, FLAGS.N)
# parameters placeholders and ops
var_update_ops = np.zeros(len(var_list_to_learn), dtype=object)
var_update_placeholders = np.zeros(len(var_list_to_learn), dtype=object)
for i in range(len(var_list_to_learn)):
var_update_placeholders[i] = tf.placeholder(
var_list_to_learn[i].dtype, shape=var_list_to_learn[i].get_shape())
var_update_ops[i] = var_list_to_learn[i].assign(
var_update_placeholders[i])
# geopathes placeholders and ops
geopath_update_ops = np.zeros((len(geopath), len(geopath[0])),
dtype=object)
geopath_update_placeholders = np.zeros((len(geopath), len(geopath[0])),
dtype=object)
for i in range(len(geopath)):
for j in range(len(geopath[0])):
geopath_update_placeholders[i, j] = tf.placeholder(
geopath[i, j].dtype, shape=geopath[i, j].get_shape())
geopath_update_ops[i, j] = geopath[i, j].assign(
geopath_update_placeholders[i, j])
tr_flag = 0
for i in range(FLAGS.max_steps):
# Select Two Candidate to Tournament
first, second = pathnet.select_two_candi(FLAGS.candi)
# First Candidate
pathnet.geopath_insert(sess, geopath_update_placeholders,
geopath_update_ops, geopath_set[first], FLAGS.L,
FLAGS.M)
var_list_backup = pathnet.parameters_backup(var_list_to_learn)
tr_flag_bak = tr_flag
for j in range(FLAGS.T):
summary_geo1_tr, _ = sess.run([merged, train_step],
feed_dict=feed_dict(train=True,
tr_flag=tr_flag))
tr_flag = (tr_flag + 16) % data_num_len1
summary_geo1_ts, acc_geo1 = sess.run([merged, accuracy],
feed_dict=feed_dict(train=False))
var_list_task1 = pathnet.parameters_backup(var_list_to_learn)
tr_flag = tr_flag_bak
# Second Candidate
pathnet.geopath_insert(sess, geopath_update_placeholders,
geopath_update_ops, geopath_set[second],
FLAGS.L, FLAGS.M)
pathnet.parameters_update(sess, var_update_placeholders,
var_update_ops, var_list_backup)
for j in range(FLAGS.T):
summary_geo2_tr, _ = sess.run([merged, train_step],
feed_dict=feed_dict(train=True,
tr_flag=tr_flag))
tr_flag = (tr_flag + 16) % data_num_len1
summary_geo2_ts, acc_geo2 = sess.run([merged, accuracy],
feed_dict=feed_dict(train=False))
var_list_task2 = pathnet.parameters_backup(var_list_to_learn)
# Compatition between two cases
if (acc_geo1 > acc_geo2):
geopath_set[second] = np.copy(geopath_set[first])
pathnet.mutation(geopath_set[second], FLAGS.L, FLAGS.M, FLAGS.N)
pathnet.parameters_update(sess, var_update_placeholders,
var_update_ops, var_list_task1)
train_writer.add_summary(summary_geo1_tr, i)
test_writer.add_summary(summary_geo1_ts, i)
print('Accuracy at step %s: %s' % (i + 1, acc_geo1))
else:
geopath_set[first] = np.copy(geopath_set[second])
pathnet.mutation(geopath_set[first], FLAGS.L, FLAGS.M, FLAGS.N)
pathnet.parameters_update(sess, var_update_placeholders,
var_update_ops, var_list_task2)
train_writer.add_summary(summary_geo2_tr, i)
test_writer.add_summary(summary_geo2_ts, i)
print('Accuracy at step %s: %s' % (i + 1, acc_geo2))
if (acc_geo1 > acc_geo2):
task1_acc = acc_geo1
task1_optimal_path = geopath_set[first]
else:
task1_acc = acc_geo2
task1_optimal_path = geopath_set[second]
# Fix task1 Optimal Path
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (task1_optimal_path[i, j] == 1.0):
fixed_list[i, j] = '1'
# Get variables of fixed list
var_list_to_fix = [] + output_weights + output_biases
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '1'):
var_list_to_fix += weights_list[i, j] + biases_list[i, j]
var_list_fix = pathnet.parameters_backup(var_list_to_fix)
# parameters placeholders and ops
var_fix_ops = np.zeros(len(var_list_to_fix), dtype=object)
var_fix_placeholders = np.zeros(len(var_list_to_fix), dtype=object)
for i in range(len(var_list_to_fix)):
var_fix_placeholders[i] = tf.placeholder(
var_list_to_fix[i].dtype, shape=var_list_to_fix[i].get_shape())
var_fix_ops[i] = var_list_to_fix[i].assign(var_fix_placeholders[i])
## TASK 2 (CIFAR 10)
# Output Layer for Task2
output_weights2 = pathnet.module_weight_variable([FLAGS.filt, 10])
output_biases2 = pathnet.module_bias_variable([10])
y2 = pathnet.nn_layer(net,
output_weights2,
output_biases2,
'output_layer2',
act=tf.identity)
with tf.name_scope('cross_entropy2'):
diff2 = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y2)
with tf.name_scope('total2'):
cross_entropy2 = tf.reduce_mean(diff2)
tf.summary.scalar('cross_entropy2', cross_entropy2)
# Need to learn variables
#var_list_to_learn=[]+input_weights+input_biases+output_weights2+output_biases2;
var_list_to_learn = [] + output_weights2 + output_biases2
for i in range(FLAGS.L):
for j in range(FLAGS.M):
if (fixed_list[i, j] == '0'):
var_list_to_learn += weights_list[i, j] + biases_list[i, j]
with tf.name_scope('train2'):
train_step2 = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(
cross_entropy2, var_list=var_list_to_learn)
#train_step2 = tf.train.GradientDescentOptimizer(FLAGS.learning_rate).minimize(
# cross_entropy2,var_list=var_list_to_learn)
with tf.name_scope('accuracy2'):
with tf.name_scope('correct_prediction2'):
correct_prediction2 = tf.equal(tf.argmax(y2, 1), tf.argmax(y_, 1))
with tf.name_scope('accuracy2'):
accuracy2 = tf.reduce_mean(tf.cast(correct_prediction2,
tf.float32))
tf.summary.scalar('accuracy2', accuracy2)
# Merge all the summaries and write them out to /tmp/tensorflow/mnist/logs/mnist_with_summaries (by default)
merged2 = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train2', sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test2')
tf.global_variables_initializer().run()
# Update fixed values
pathnet.parameters_update(sess, var_fix_placeholders, var_fix_ops,
var_list_fix)
def feed_dict2(train, tr_flag=0):
#Make a TensorFlow feed_dict: maps data onto Tensor placeholders.
if train or FLAGS.fake_data:
xs = tr_data2[tr_flag:tr_flag + 16, :]
ys = tr_label2[tr_flag:tr_flag + 16, :]
k = FLAGS.dropout
else:
xs = ts_data2
ys = ts_label2
k = 1.0
return {x: xs, y_: ys}
# Generating randomly geopath
geopath_set = np.zeros(FLAGS.candi, dtype=object)
for i in range(FLAGS.candi):
geopath_set[i] = pathnet.get_geopath(FLAGS.L, FLAGS.M, FLAGS.N)
# parameters placeholders and ops
var_update_ops = np.zeros(len(var_list_to_learn), dtype=object)
var_update_placeholders = np.zeros(len(var_list_to_learn), dtype=object)
for i in range(len(var_list_to_learn)):
var_update_placeholders[i] = tf.placeholder(
var_list_to_learn[i].dtype, shape=var_list_to_learn[i].get_shape())
var_update_ops[i] = var_list_to_learn[i].assign(
var_update_placeholders[i])
tr_flag = 0
for i in range(FLAGS.max_steps):
# Select Two Candidate to Tournament
first, second = pathnet.select_two_candi(FLAGS.candi)
# First Candidate
pathnet.geopath_insert(sess, geopath_update_placeholders,
geopath_update_ops, geopath_set[first], FLAGS.L,
FLAGS.M)
tr_flag_bak = tr_flag
var_list_backup = pathnet.parameters_backup(var_list_to_learn)
for j in range(FLAGS.T):
summary_geo1_tr, _ = sess.run([merged2, train_step2],
feed_dict=feed_dict2(True, tr_flag))
tr_flag = (tr_flag + 16) % data_num_len2
summary_geo1_ts, acc_geo1 = sess.run([merged2, accuracy2],
feed_dict=feed_dict2(False))
var_list_task1 = pathnet.parameters_backup(var_list_to_learn)
# Second Candidate
pathnet.geopath_insert(sess, geopath_update_placeholders,
geopath_update_ops, geopath_set[first], FLAGS.L,
FLAGS.M)
tr_flag = tr_flag_bak
pathnet.parameters_update(sess, var_update_placeholders,
var_update_ops, var_list_backup)
for j in range(FLAGS.T - 1):
summary_geo2_tr, _, acc_geo2_tmp = sess.run(
[merged2, train_step2, accuracy2],
feed_dict=feed_dict2(True, tr_flag))
tr_flag = (tr_flag + 16) % data_num_len2
summary_geo2_ts, acc_geo2 = sess.run([merged2, accuracy2],
feed_dict=feed_dict2(False))
var_list_task2 = pathnet.parameters_backup(var_list_to_learn)
# Compatition between two cases
if (acc_geo1 > acc_geo2):
geopath_set[second] = np.copy(geopath_set[first])
pathnet.mutation(geopath_set[second], FLAGS.L, FLAGS.M, FLAGS.N)
pathnet.parameters_update(sess, var_update_placeholders,
var_update_ops, var_list_task1)
train_writer.add_summary(summary_geo1_tr, i)
test_writer.add_summary(summary_geo1_ts, i)
print('Accuracy at step %s: %s' % (i + 1, acc_geo1))
else:
geopath_set[first] = np.copy(geopath_set[second])
pathnet.mutation(geopath_set[first], FLAGS.L, FLAGS.M, FLAGS.N)
pathnet.parameters_update(sess, var_update_placeholders,
var_update_ops, var_list_task2)
train_writer.add_summary(summary_geo2_tr, i)
test_writer.add_summary(summary_geo2_ts, i)
print('Accuracy at step %s: %s' % (i + 1, acc_geo2))
if (acc_geo1 > acc_geo2):
task2_acc = acc_geo1
else:
task2_acc = acc_geo2
print("SVHN Acc:" + str(task1_acc) + ",CIFAR10:" + str(task2_acc))
train_writer.close()
test_writer.close()
def main():
args = get_arguments()
allvars = get_all_model_variables(args)
# Load Test Dataset
if (allvars['model2load'] == 'fcnn') or (allvars['model2load'] == 'lenet'):
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets(allvars['data_dir'], one_hot=True)
X = mnist.test.images
y = mnist.test.labels
# Free Memory
mnist = None
if (allvars['model2load'] == 'nin') or (allvars['model2load'] == 'densenet'):
import cifar10
cifar10.data_path = allvars['data_dir']
cifar10.maybe_download_and_extract()
X, _, y = cifar10.load_test_data()
# Free Memory
cifar = None
X, y = pre_process_data(X, y, allvars['model2load'])
X, y = collect_correctly_predicted_images(X, y, allvars)
eps_rescale = np.max(np.abs( np.max(X.flatten()) - np.min(X.flatten()) ))
deepfool_norms, deepfooled_vec = get_deepfool_ellenorms(X, y, allvars)
print('DeepFool fooling ratio: '+str(np.mean(deepfooled_vec)*100)+' %')
print('DeepFool mean epsilon: '+str(np.mean(deepfool_norms[deepfooled_vec==1])/eps_rescale))
if args.rho_mode:
print('Computing performance metrics...')
print()
rho_1 = get_robustness_metrics(X, y, deepfool_norms, deepfooled_vec, allvars, method='rho1')
rho_2 = get_robustness_metrics(X, y, deepfool_norms, deepfooled_vec, allvars, method='rho2')/eps_rescale
eps99 = get_robustness_metrics(X, y, deepfool_norms, deepfooled_vec, allvars, method='eps99')/eps_rescale
print('rho_1 = ' + str(rho_1))
print('rho_2 = ' + str(rho_2))
print('eps99 = ' + str(eps99))
np.savetxt(allvars['output_dir'] + 'robustness_' + allvars['model2load'] + '_' + str(allvars['n_images']) + \
'_' + str(int(allvars['max_epsilon'] * 1000)) + '.csv',
np.array([rho_1, rho_2, eps99]),
delimiter=";")
else:
print('Computing fooling ratios...')
print()
epsilon = np.array(np.linspace(0.001, allvars['max_epsilon'], 10))
fool_dict = {'epsilon': 0, 'FGS': 1, 'Alg1': 2, 'Alg2': 3, 'rand': 4, 'DeepFool': 5}
fool_mtx = np.zeros([len(epsilon), len(fool_dict)])
for i in range(len(epsilon)):
eps = epsilon[i]*eps_rescale
print(allvars['model2load'] + ': realization '+str(i+1)+'/'+str(len(epsilon))+'...' )
fool_mtx[i, fool_dict['epsilon']] = epsilon[i]
fool_mtx[i, fool_dict['FGS']] = get_foolratio(X, y, eps, allvars, method='FGS')
fool_mtx[i, fool_dict['Alg1']] = get_foolratio(X, y, eps, allvars, method='Alg1')
fool_mtx[i, fool_dict['Alg2']] = get_foolratio(X, y, eps, allvars, method='Alg2')
fool_mtx[i, fool_dict['rand']] = get_foolratio(X, y, eps, allvars, method='rand')
fool_mtx[i, fool_dict['DeepFool']] = np.mean(np.array((deepfool_norms<eps) * (deepfooled_vec==1)))
np.savetxt(allvars['output_dir']+'fool_summary_' + allvars['model2load'] +'_' +str(allvars['n_images'])+\
'_'+str(int(allvars['max_epsilon']*1000))+'.csv', fool_mtx, delimiter=";")
save_foolratio_fig(fool_mtx,
allvars['figs_dir'] + 'fig_'+ allvars['model2load'] +'_' +str(allvars['n_images'])+\
'_'+str(int(allvars['max_epsilon']*1000))+'.eps',
fool_dict, legend=True)
def main(argv=None):
cifar10.maybe_download_and_extract()
if tf.gfile.Exists(FLAGS.train_dir):
tf.gfile.DeleteRecursively(FLAGS.train_dir)
tf.gfile.MakeDirs(FLAGS.train_dir)
train()
def cifar10(path, # pylint: disable=invalid-name
conv_channels=None,
linear_layers=None,
batch_norm=True,
batch_size=128,
mode="train"):
"""Cifar10 classification with a convolutional network."""
import cifar10
cifar10.data_path = "CIFAR-10-data/"
cifar10.maybe_download_and_extract()
images_train, cls_train, labels_train = cifar10.load_training_data()
images = tf.constant(images_train, dtype=tf.float32, name="CIFAR_images")
labels = tf.constant(cls_train, dtype=tf.int64, name="CIFAR_labels")
# Network.
def _conv_activation(x): # pylint: disable=invalid-name
return tf.nn.max_pool(tf.nn.relu(x),
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="SAME")
conv = snt.nets.ConvNet2D(output_channels=conv_channels,
kernel_shapes=[5],
strides=[1],
paddings=[snt.SAME],
activation=_conv_activation,
activate_final=True,
initializers=_nn_initializers,
use_batch_norm=batch_norm,
batch_norm_config={'update_ops_collection': None})
if batch_norm:
conv1 = lambda x: conv(x, is_training=True)
linear_activation = lambda x: tf.nn.relu(snt.BatchNorm(update_ops_collection = None)(x, is_training=True))
else:
conv1 = conv
linear_activation = tf.nn.relu
mlp = snt.nets.MLP(list(linear_layers) + [10],
activation=linear_activation,
initializers=_nn_initializers)
network = snt.Sequential([conv1, snt.BatchFlatten(), mlp])
def build():
indices = tf.random_uniform([batch_size], 0, len(images_train)-1, tf.int64)
image_batch = tf.gather(images, indices)
label_batch = tf.gather(labels, indices)
output = network(image_batch)
return _xent_loss(output, label_batch)
def convex_loss():
v = tf.get_variable("v", shape=[1, 10], dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=0.01))
# Non-trainable variables.
target = tf.get_variable("target", shape=[1, 10], dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=0.01), trainable=False)
return tf.reduce_mean(tf.clip_by_value(tf.square(v - target), 0, 10))
return collections.OrderedDict([('Opt_loss', build), ('Aux_loss', convex_loss)])
def mainWithMcnemar(argv=None): # pylint: disable=unused-argument
cifar10.maybe_download_and_extract()
if tf.gfile.Exists(tfFLAGS.eval_dir):
tf.gfile.DeleteRecursively(tfFLAGS.eval_dir)
tf.gfile.MakeDirs(tfFLAGS.eval_dir)
evaluateWithMcnemar()
def main():
"""
主函数
"""
# 下载cifar10数据集并解压
cifar10.maybe_download_and_extract()
# distorted_inputs产生训练数据
images_train, labels_train = cifar10_input.distorted_inputs(data_dir=dataset_dir,
batch_size=batch_size)
# 产生测试数据
images_test, labels_test = cifar10_input.inputs(eval_data=True,
data_dir=dataset_dir,
batch_size=batch_size)
# 为特征和label创建placeholder
image_holder = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
label_holder = tf.placeholder(tf.int32, [batch_size])
# 创建CNN网络，并得到输出
logitis = build_cnn_network(image_holder)
# 计算loss
total_loss = get_total_loss(logitis, label_holder)
# 设置优化算法
train_op = tf.train.AdamOptimizer(1e-3).minimize(total_loss)
top_k_op = tf.nn.in_top_k(logitis, label_holder, 1)
# 创建session
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
# 启动线程操作，因为cifar10_input.distorted_inputs需要线程操作
tf.train.start_queue_runners()
# 训练模型
for step in range(max_steps):
start_time = time.time()
image_batch, label_batch = sess.run([images_train, labels_train])
_, loss_value = sess.run([train_op, total_loss],
feed_dict={image_holder: image_batch,
label_holder: label_batch})
duration = time.time() - start_time
if step % disp_step == 0:
sample_per_sec = batch_size / duration
sec_per_batch = float(duration)
print('step %d, loss=%.2f (%.1f sample/sec; %.3f sec/batch)' % (
step, loss_value, sample_per_sec, sec_per_batch
))
# 测试模型
n_test_samples = 10000
n_iter = int(math.ceil(n_test_samples / batch_size))
true_count = 0
total_sample_count = n_iter * batch_size
step = 0
while step < n_iter:
image_batch, label_batch = sess.run([images_test, labels_test])
predictions = sess.run([top_k_op], feed_dict={image_holder: image_batch,
label_holder: label_batch})
true_count += np.sum(predictions)
step += 1
precision = true_count / total_sample_count
print('top 1 precision: %.3f' % precision)
