def getImageBatchAndOneHotLabels(dataset_dir, dataset_name, num_readers,
num_preprocessing_threads, batch_size):
'''
:param dataset_dir: directory where the tfrecord files are stored
:param dataset_name: name of the dataset e.g. train / validation
:return:
'''
dataset = imagenet.get_split(dataset_name, dataset_dir)
# DataSetProvider on CPU
with tf.device('/device:CPU:0'):
# ------- Dataset Provider ---------
provider_train = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=num_readers,
common_queue_capacity=2 * batch_size,
common_queue_min=batch_size)
[image, label] = provider_train.get(['image', 'label'])
# Preprocessing of Dataset
train_image_size = alexnet.alexnet_v2.default_image_size
image = alexnet_preprocessing.preprocess_image(image, train_image_size,
train_image_size)
# Generate Batches
images, labels = tf.train.batch([image, label],
batch_size=batch_size,
num_threads=num_preprocessing_threads,
capacity=5 * batch_size)
labels = slim.one_hot_encoding(labels, dataset.num_classes)
return dataset, images, labels
def get_alpha_from_compression_rate(compression_rate):
def bin_search_k(shape_lst, cr_target):
import math
high = 1.0
low = 0.001
alpha = (high + low) / 2
while True:
nparams_orig = 0
nparams_comp = 0
for shape in shape_lst:
n, c, w = shape[3], shape[2], shape[1]
nparams_orig += n * c * w * w * 32
k = int(alpha * n * c * w)
nparams_comp += (n * c * w * int(math.log(k, 2) + 1) +
k * w * 32)
cr = nparams_orig / nparams_comp
delta = cr_target - cr
print('delta: {}'.format(delta))
print('alpha: {}'.format(alpha))
print('###################')
if math.fabs(delta) < 0.02:
break
if delta < 0.0:
low = (high + low) / 2
alpha = (high + low) / 2
else:
high = (high + low) / 2
alpha = (high + low) / 2
return alpha
from functools import reduce
model_name = 'inception_v1'
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
dataset = imagenet.get_split('validation', FLAGS.dataset_dir)
istraining_placeholder = tf.placeholder(tf.bool)
network_fn = GoogLeNet.GoogLeNet(
model_name,
num_classes=(dataset.num_classes - 1),
weight_decay=FLAGS.weight_decay,
is_training=istraining_placeholder)
logits_lst = []
images_placeholder = tf.placeholder(
tf.float32,
shape=(FLAGS.batch_size * FLAGS.gpu_num,
network_fn.default_image_size,
network_fn.default_image_size, 3))
with tf.variable_scope(tf.get_variable_scope()) as scope:
for gpu_index in range(0, 1):
with tf.device('/gpu:%d' % gpu_index):
print('/gpu:%d' % gpu_index)
with tf.name_scope('%s_%d' %
('gpu', gpu_index)) as scope:
logits, end_points, end_points_Ofmap, end_points_Ifmap = (
images_placeholder[gpu_index *
FLAGS.batch_size:
(gpu_index + 1) *
FLAGS.batch_size])
logits_lst.append(logits)
#end_points_lst.append(end_points)
# Reuse variables for the next tower.
tf.get_variable_scope().reuse_variables()
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
sess.run(init_op)
shape_lst = []
for v in tf.trainable_variables():
if 'weights' in v.name:
#print(sess.run(v).shape)
print("v.name: {}".format(v.name))
print("v.shape: {}".format(sess.run(v).shape))
print('=====================================')
shape_lst.append(sess.run(v).shape)
if 'beta' in v.name:
print("v.name: {}".format(v.name))
print("v.shape: {}".format(sess.run(v).shape))
print('+++++++++++++++++++++++++++++++++++++')
alpha = bin_search_k(shape_lst, compression_rate)
print("compression rate: {:.4f}, alpha: {:.4f}".format(
compression_rate, alpha))
def run_gpu_eval(use_compression=False,
use_quantization=False,
compute_energy=False,
use_pretrained_model=True,
epoch_num=0):
from functools import reduce
module_name = 'inception_v1'
checkpoint_dir = 'checkpoint/{}_{}_{}'.format(module_name, epoch_num,
FLAGS.alpha)
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
dataset = imagenet.get_split('validation', FLAGS.dataset_dir)
istraining_placeholder = tf.placeholder(tf.bool)
network_fn = GoogLeNet.GoogLeNet(
num_classes=(dataset.num_classes - FLAGS.labels_offset),
weight_decay=FLAGS.weight_decay,
is_training=istraining_placeholder)
logits_lst = []
images_placeholder = tf.placeholder(
tf.float32,
shape=(FLAGS.batch_size * FLAGS.gpu_num,
network_fn.default_image_size,
network_fn.default_image_size, 3))
labels_placeholder = tf.placeholder(
tf.int64,
shape=(FLAGS.batch_size * FLAGS.gpu_num, dataset.num_classes))
with tf.variable_scope(tf.get_variable_scope()) as scope:
for gpu_index in range(0, FLAGS.gpu_num):
with tf.device('/gpu:%d' % gpu_index):
print('/gpu:%d' % gpu_index)
with tf.name_scope('%s_%d' %
('gpu', gpu_index)) as scope:
logits, end_points, end_points_Ofmap, end_points_Ifmap = network_fn(
images_placeholder[gpu_index *
FLAGS.batch_size:
(gpu_index + 1) *
FLAGS.batch_size])
logits_lst.append(logits)
# Reuse variables for the next tower.
tf.get_variable_scope().reuse_variables()
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
is_training=False)
logits_op = tf.concat(logits_lst, 0)
right_count_top1_op = tf.reduce_sum(
tf.cast(
tf.equal(tf.argmax(tf.nn.softmax(logits_op), axis=1),
tf.argmax(labels_placeholder, axis=1)), tf.int32))
right_count_topk_op = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(tf.nn.softmax(logits_op),
tf.argmax(labels_placeholder, axis=1), 5),
tf.int32))
images_op, labels_op = input_data.inputs(
dataset=dataset,
image_preprocessing_fn=image_preprocessing_fn,
network_fn=network_fn,
num_epochs=1,
batch_size=FLAGS.batch_size * FLAGS.gpu_num)
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
gvar_list = tf.global_variables()
bn_moving_vars = [g for g in gvar_list if 'moving_mean' in g.name]
bn_moving_vars += [
g for g in gvar_list if 'moving_variance' in g.name
]
print([var.name for var in bn_moving_vars])
if use_pretrained_model:
varlist = tf.trainable_variables()
varlist += bn_moving_vars
print(varlist)
saver = tf.train.Saver(varlist)
# saver = tf.train.Saver(vardict)
if os.path.isfile(FLAGS.checkpoint_path):
saver.restore(sess, FLAGS.checkpoint_path)
print(
'#############################Session restored from pretrained model at {}!###############################'
.format(FLAGS.checkpoint_path))
else:
ckpt = tf.train.get_checkpoint_state(
checkpoint_dir=FLAGS.checkpoint_path)
if ckpt and ckpt.model_checkpoint_path:
saver = tf.train.Saver(varlist)
saver.restore(sess, ckpt.model_checkpoint_path)
print(
'Session restored from pretrained degradation model at {}!'
.format(ckpt.model_checkpoint_path))
else:
varlist = tf.trainable_variables()
varlist += bn_moving_vars
saver = tf.train.Saver(varlist)
ckpt = tf.train.get_checkpoint_state(
checkpoint_dir=checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print(
'#############################Session restored from trained model at {}!###############################'
.format(ckpt.model_checkpoint_path))
else:
raise FileNotFoundError(errno.ENOENT,
os.strerror(errno.ENOENT),
checkpoint_dir)
mat_eng = matlab.engine.start_matlab()
seed = 500
alpha = FLAGS.alpha
memory = 0
for v in tf.trainable_variables() + bn_moving_vars:
if 'weights' in v.name:
memory += np.prod(sess.run(v).shape)
print("weights.name: {}".format(v.name))
print("weights.shape: {}".format(sess.run(v).shape))
if use_compression:
weights = np.transpose(sess.run(v), (3, 2, 1, 0))
shape = weights.shape
n, c, w = shape[0], shape[1], shape[2]
k = int(alpha * n * c * w)
weight_clustered, mse = cluster_conv(weights, k, seed)
weight_clustered = np.transpose(
weight_clustered, (3, 2, 1, 0))
sess.run(v.assign(weight_clustered))
print("weight_clustered shape: {}".format(
weight_clustered.shape))
print("mse: {}".format(mse))
seed += 1
if use_quantization:
weights = np.transpose(sess.run(v), (3, 2, 1, 0))
shape = weights.shape
weight_quantized = mat_eng.get_fi(
matlab.double(weights.tolist()), FLAGS.bitwidth,
FLAGS.bitwidth -
FLAGS.bitwidth_minus_fraction_length)
weight_quantized = np.asarray(
weight_quantized).reshape(shape).astype('float32')
weight_quantized = np.transpose(
weight_quantized, (3, 2, 1, 0))
sess.run(v.assign(weight_quantized))
print("weight_quantized shape: {}".format(
weight_quantized.shape))
print('=====================================')
if any(x in v.name for x in ['beta']):
memory += np.prod(sess.run(v).shape)
print("beta.name: {}".format(v.name))
print("beta.shape: {}".format(sess.run(v).shape))
if use_quantization:
weights = sess.run(v)
shape = weights.shape
weight_quantized = mat_eng.get_fi(
matlab.double(weights.tolist()), FLAGS.bn_bitwidth,
FLAGS.bn_bitwidth -
FLAGS.bitwidth_minus_fraction_length)
weight_quantized = np.asarray(
weight_quantized).reshape(shape).astype('float32')
sess.run(v.assign(weight_quantized))
print("beta_quantized shape: {}".format(
weight_quantized.shape))
print('+++++++++++++++++++++++++++++++++++++')
checkpoint_path = os.path.join(checkpoint_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=0)
print(
"############################################### MEMORY IS {} ###############################################"
.format(memory))
if compute_energy:
weights_dict = {}
for v in tf.trainable_variables():
if 'weights' in v.name:
vname = "_".join(v.name.split('/')[1:-1])
print("v.name: {}".format(vname))
print("v.shape: {}".format(sess.run(v).shape))
#weights = np.transpose(sess.run(v), (3, 2, 1, 0))
weights = sess.run(v)
print("v.nzeros: {}".format(
np.count_nonzero(weights == 0)))
weights_dict[vname] = [
reduce(lambda x, y: x * y, weights.shape) *
(1 - FLAGS.alpha), weights.shape
]
print('=====================================')
total_v = 0.0
test_correct_num_top1 = 0.0
test_correct_num_topk = 0.0
from tqdm import tqdm
pbar = tqdm(total=dataset.num_samples //
(FLAGS.gpu_num * FLAGS.batch_size), )
i = 1
model_params_dict = {}
try:
while not coord.should_stop():
pbar.update(1)
images, labels = sess.run([images_op, labels_op])
right_count_top1, right_count_topk = sess.run(
[right_count_top1_op, right_count_topk_op],
feed_dict={
images_placeholder: images,
labels_placeholder: labels,
istraining_placeholder: False
})
end_points_Ofmap_dict, end_points_Ifmap_dict = sess.run(
[end_points_Ofmap, end_points_Ifmap],
feed_dict={
images_placeholder: images,
labels_placeholder: labels,
istraining_placeholder: False
})
test_correct_num_top1 += right_count_top1
test_correct_num_topk += right_count_topk
total_v += labels.shape[0]
if compute_energy:
keys = list(end_points_Ifmap_dict.keys())
if i == 1:
for k in keys:
model_params_dict[k] = {}
model_params_dict[k][
"IfMap_Shape"] = end_points_Ifmap_dict[
k].shape
model_params_dict[k][
"IfMap_nZeros"] = np.count_nonzero(
end_points_Ifmap_dict[k] == 0)
model_params_dict[k][
"Filter_Shape"] = weights_dict[k][1]
model_params_dict[k]["Filter_nZeros"] = int(
weights_dict[k][0])
model_params_dict[k][
"OfMap_Shape"] = end_points_Ofmap_dict[
k].shape
model_params_dict[k][
"OfMap_nZeros"] = np.count_nonzero(
end_points_Ofmap_dict[k] == 0)
print("Layer Name: {}".format(k))
print("IfMap Shape: {}".format(
end_points_Ifmap_dict[k].shape))
print("IfMap nZeros: {:.4e}".format(
np.count_nonzero(
end_points_Ifmap_dict[k] == 0)))
print("IfMap nZeros Avg: {:.4e}".format(
model_params_dict[k]["IfMap_nZeros"]))
print("Filter Shape: {}".format(
weights_dict[k][1]))
print("Filter nZeros: {:.4e}".format(
int(weights_dict[k][0])))
print("OfMap Shape: {}".format(
end_points_Ofmap_dict[k].shape))
print("OfMap nZeros: {:.4e}".format(
np.count_nonzero(
end_points_Ofmap_dict[k] == 0)))
print("OfMap nZeros Avg: {:.4e}".format(
model_params_dict[k]["OfMap_nZeros"]))
print(
'=========================================================================='
)
else:
for k in keys:
model_params_dict[k]["IfMap_nZeros"] = (
model_params_dict[k]["IfMap_nZeros"] +
np.count_nonzero(
end_points_Ifmap_dict[k] == 0) /
(i - 1)) * (i - 1) / i
model_params_dict[k]["OfMap_nZeros"] = (
model_params_dict[k]["OfMap_nZeros"] +
np.count_nonzero(
end_points_Ofmap_dict[k] == 0) /
(i - 1)) * (i - 1) / i
i += 1
except tf.errors.OutOfRangeError:
print('Done testing on all the examples')
finally:
coord.request_stop()
if compute_energy:
import pickle
with open('model_params_dict.pkl', 'wb') as f:
pickle.dump(model_params_dict, f,
pickle.HIGHEST_PROTOCOL)
with open('GoogLeNet_Pruned_{}.txt'.format(FLAGS.alpha),
'w') as wf:
for k in keys:
wf.write("Layer Name: {}\n".format(k))
wf.write("IfMap Shape: {}\n".format(
model_params_dict[k]["IfMap_Shape"]))
wf.write("IfMap nZeros: {:.4e}\n".format(
model_params_dict[k]["IfMap_nZeros"]))
wf.write("Filter Shape: {}\n".format(
model_params_dict[k]["Filter_Shape"]))
wf.write("Filter nZeros: {:.4e}\n".format(
model_params_dict[k]["Filter_nZeros"]))
wf.write("OfMap Shape: {}\n".format(
model_params_dict[k]["OfMap_Shape"]))
wf.write("OfMap nZeros: {:.4e}\n".format(
model_params_dict[k]["OfMap_nZeros"]))
wf.write(
'==========================================================================\n'
)
coord.join(threads)
print('Test acc top1:', test_correct_num_top1 / total_v,
'Test_correct_num top1:', test_correct_num_top1, 'Total_v:',
total_v)
print('Test acc topk:', test_correct_num_topk / total_v,
'Test_correct_num topk:', test_correct_num_topk, 'Total_v:',
total_v)
isCompression = lambda bool: "Compression_" if bool else "NoCompression_"
isQuantization = lambda bool: "Quantization_" if bool else "NoQuantization_"
with open(
'{}_{}_{}_{}_evaluation.txt'.format(
isCompression(use_compression),
isQuantization(use_quantization), epoch_num,
FLAGS.alpha), 'w') as wf:
wf.write(
'test acc top1:{}\ttest_correct_num top1:{}\ttotal_v:{}\n'.
format(test_correct_num_top1 / total_v,
test_correct_num_top1, total_v))
wf.write(
'test acc topk:{}\ttest_correct_num topk:{}\ttotal_v:{}\n'.
format(test_correct_num_topk / total_v,
test_correct_num_topk, total_v))
print("done")
def run_gpu_train(use_pretrained_model, epoch_num):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
module_name = 'inception_v1'
checkpoint_dir = 'checkpoint/{}_{}_{}'.format(module_name, epoch_num - 1,
FLAGS.alpha)
saved_checkpoint_dir = 'checkpoint/{}_{}_{}'.format(
module_name, epoch_num, FLAGS.alpha)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
if not os.path.exists(saved_checkpoint_dir):
os.makedirs(saved_checkpoint_dir)
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
dataset = imagenet.get_split('train', FLAGS.dataset_dir)
dataset_val = imagenet.get_split('validation', FLAGS.dataset_dir)
global_step = slim.create_global_step()
learning_rate = _configure_learning_rate(dataset.num_samples,
global_step)
istraining_placeholder = tf.placeholder(tf.bool)
network_fn = GoogLeNet.GoogLeNet(
num_classes=(dataset.num_classes - FLAGS.labels_offset),
weight_decay=FLAGS.weight_decay,
is_training=istraining_placeholder)
tower_grads = []
logits_lst = []
losses_lst = []
opt = _configure_optimizer(learning_rate)
images_placeholder = tf.placeholder(
tf.float32,
shape=(FLAGS.batch_size * FLAGS.gpu_num,
network_fn.default_image_size,
network_fn.default_image_size, 3))
labels_placeholder = tf.placeholder(
tf.int64,
shape=(FLAGS.batch_size * FLAGS.gpu_num, dataset.num_classes))
with tf.variable_scope(tf.get_variable_scope()) as scope:
for gpu_index in range(0, FLAGS.gpu_num):
with tf.device('/gpu:%d' % gpu_index):
print('/gpu:%d' % gpu_index)
with tf.name_scope('%s_%d' %
('gpu', gpu_index)) as scope:
logits, _, _, _ = network_fn(
images_placeholder[gpu_index *
FLAGS.batch_size:
(gpu_index + 1) *
FLAGS.batch_size])
logits_lst.append(logits)
loss = tower_loss_xentropy_dense(
logits, labels_placeholder[gpu_index *
FLAGS.batch_size:
(gpu_index + 1) *
FLAGS.batch_size])
losses_lst.append(loss)
# varlist = [v for v in tf.trainable_variables() if any(x in v.name for x in ["logits"])]
varlist = tf.trainable_variables()
#print([v.name for v in varlist])
grads = opt.compute_gradients(loss, varlist)
tower_grads.append(grads)
# Reuse variables for the next tower.
tf.get_variable_scope().reuse_variables()
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
is_training=True)
val_image_preprocessing_fn = preprocessing_factory.get_preprocessing(
is_training=False)
loss_op = tf.reduce_mean(losses_lst, name='softmax')
logits_op = tf.concat(logits_lst, 0)
acc_op = accuracy(logits_op, labels_placeholder)
grads = average_gradients(tower_grads)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
print(update_ops)
with tf.control_dependencies([tf.group(*update_ops)]):
apply_gradient_op = opt.apply_gradients(
grads, global_step=global_step)
images_op, labels_op = input_data.inputs(
dataset=dataset,
image_preprocessing_fn=image_preprocessing_fn,
network_fn=network_fn,
num_epochs=1,
batch_size=FLAGS.batch_size * FLAGS.gpu_num)
val_images_op, val_labels_op = input_data.inputs(
dataset=dataset_val,
image_preprocessing_fn=val_image_preprocessing_fn,
network_fn=network_fn,
num_epochs=None,
batch_size=FLAGS.batch_size * FLAGS.gpu_num)
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
gvar_list = tf.global_variables()
bn_moving_vars = [g for g in gvar_list if 'moving_mean' in g.name]
bn_moving_vars += [
g for g in gvar_list if 'moving_variance' in g.name
]
print([var.name for var in bn_moving_vars])
if use_pretrained_model:
varlist = tf.trainable_variables()
varlist += bn_moving_vars
print(varlist)
# vardict = {v.name[:-2].replace('MobileNet', 'MobilenetV1'): v for v in varlist}
saver = tf.train.Saver(varlist)
# saver = tf.train.Saver(vardict)
if os.path.isfile(FLAGS.checkpoint_path):
saver.restore(sess, FLAGS.checkpoint_path)
print(
'#############################Session restored from pretrained model at {}!###############################'
.format(FLAGS.checkpoint_path))
else:
ckpt = tf.train.get_checkpoint_state(
checkpoint_dir=FLAGS.checkpoint_path)
if ckpt and ckpt.model_checkpoint_path:
saver = tf.train.Saver(varlist)
saver.restore(sess, ckpt.model_checkpoint_path)
print(
'Session restored from pretrained degradation model at {}!'
.format(ckpt.model_checkpoint_path))
else:
varlist = tf.trainable_variables()
varlist += bn_moving_vars
saver = tf.train.Saver(varlist)
ckpt = tf.train.get_checkpoint_state(
checkpoint_dir=checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print(
'#############################Session restored from trained model at {}!###############################'
.format(ckpt.model_checkpoint_path))
else:
raise FileNotFoundError(errno.ENOENT,
os.strerror(errno.ENOENT),
checkpoint_dir)
saver = tf.train.Saver(tf.trainable_variables() + bn_moving_vars)
step = 0
try:
while not coord.should_stop():
start_time = time.time()
images, labels = sess.run([images_op, labels_op])
_, loss_value = sess.run(
[apply_gradient_op, loss_op],
feed_dict={
images_placeholder: images,
labels_placeholder: labels,
istraining_placeholder: True
})
assert not np.isnan(
loss_value), 'Model diverged with loss = NaN'
duration = time.time() - start_time
print('Step: {:4d} time: {:.4f} loss: {:.8f}'.format(
step, duration, loss_value))
if step % FLAGS.val_step == 0:
start_time = time.time()
images, labels = sess.run([images_op, labels_op])
acc, loss_value = sess.run(
[acc_op, loss_op],
feed_dict={
images_placeholder: images,
labels_placeholder: labels,
istraining_placeholder: False
})
print(
"Step: {:4d} time: {:.4f}, training accuracy: {:.5f}, loss: {:.8f}"
.format(step,
time.time() - start_time, acc, loss_value))
start_time = time.time()
images, labels = sess.run(
[val_images_op, val_labels_op])
acc, loss_value = sess.run(
[acc_op, loss_op],
feed_dict={
images_placeholder: images,
labels_placeholder: labels,
istraining_placeholder: False
})
print(
"Step: {:4d} time: {:.4f}, validation accuracy: {:.5f}, loss: {:.8f}"
.format(step,
time.time() - start_time, acc, loss_value))
# Save a checkpoint and evaluate the model periodically.
if step % FLAGS.save_step == 0 or (step +
1) == FLAGS.max_steps:
checkpoint_path = os.path.join(saved_checkpoint_dir,
'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
step += 1
except tf.errors.OutOfRangeError:
print('Done training on all the examples')
finally:
coord.request_stop()
coord.request_stop()
coord.join(threads)
checkpoint_path = os.path.join(saved_checkpoint_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
print("done")
def get_k_list(alpha):
model_name = 'inception_v1'
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
dataset = imagenet.get_split('validation', FLAGS.dataset_dir)
istraining_placeholder = tf.placeholder(tf.bool)
network_fn = GoogLeNet.GoogLeNet(
num_classes=(dataset.num_classes - FLAGS.labels_offset),
weight_decay=FLAGS.weight_decay,
is_training=istraining_placeholder)
logits_lst = []
images_placeholder = tf.placeholder(
tf.float32,
shape=(FLAGS.batch_size * FLAGS.gpu_num,
network_fn.default_image_size,
network_fn.default_image_size, 3))
labels_placeholder = tf.placeholder(
tf.int64,
shape=(FLAGS.batch_size * FLAGS.gpu_num, dataset.num_classes))
with tf.variable_scope(tf.get_variable_scope()) as scope:
for gpu_index in range(0, FLAGS.gpu_num):
with tf.device('/gpu:%d' % gpu_index):
print('/gpu:%d' % gpu_index)
with tf.name_scope('%s_%d' %
('gpu', gpu_index)) as scope:
logits, end_points, end_points_Ofmap, end_points_Ifmap = network_fn(
images_placeholder[gpu_index *
FLAGS.batch_size:
(gpu_index + 1) *
FLAGS.batch_size])
logits_lst.append(logits)
# Reuse variables for the next tower.
tf.get_variable_scope().reuse_variables()
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
sess.run(init_op)
k_list = {}
memory_all = 0
for v in tf.trainable_variables():
if 'weights' in v.name:
#print(sess.run(v).shape)
print("v.name: {}".format(v.name))
print("v.shape: {}".format(sess.run(v).shape))
print('=====================================')
shape = sess.run(v).shape
n, c, w = shape[3], shape[2], shape[1]
memory_all += n * c * w * w * 32
k = int(alpha * n * c * w)
k_list[v.name] = k
pass
if 'beta' in v.name:
print("v.name: {}".format(v.name))
print("v.shape: {}".format(sess.run(v).shape))
print('+++++++++++++++++++++++++++++++++++++')
print("memory_all(bits): {:.1f}".format(memory_all))
return k_list, memory_all
graph = load_graph(args.frozen_model_filename)
# We can verify that we can access the list of operations in the graph
for op in graph.get_operations():
print(op.name)
# We access the input and output nodes in the graph
images_placeholder = graph.get_tensor_by_name('prefix/Images_Placeholder:0')
labels_placeholder = graph.get_tensor_by_name('prefix/Labels_Placeholder:0')
predicted_labels = graph.get_tensor_by_name('prefix/Predicted_Labels:0')
right_count_top1_op = graph.get_tensor_by_name('prefix/Right_Count_Top1:0')
right_count_topk_op = graph.get_tensor_by_name('prefix/Right_Count_Topk:0')
dataset = imagenet.get_split(
'validation', '../datasets/imagenet-data/tfrecords')
with tf.Session(graph=graph) as sess:
images_op, labels_op = inputs(dataset=dataset, image_preprocessing_fn=preprocessing_factory.get_preprocessing(is_training=False),
num_epochs=1, batch_size=48)
init_op = tf.group(tf.local_variables_initializer(), tf.global_variables_initializer())
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
total_v = 0.0
test_correct_num_top1 = 0.0
test_correct_num_topk = 0.0
from tqdm import tqdm
def evaluate():
"""Eval CIFAR-10 for a number of steps."""
with tf.Graph().as_default() as g:
# Get images and labels for CIFAR-10.
# eval_data = FLAGS.eval_data == 'test'
# images, labels = vgg.inputs(eval_data=eval_data)
dataset = imagenet.get_split(
'validation', '/data/ramyadML/TF-slim-data/imageNet/processed')
# Creates a TF-Slim DataProvider which reads the dataset in the background
# during both training and testing.
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=4,
common_queue_capacity=20 * 32,
common_queue_min=10 * 32,
shuffle=True)
preprocessing_name = 'vgg_16'
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
[image, label] = provider.get(['image', 'label'])
image = image_preprocessing_fn(image, 224, 224)
label -= 1
# batch up some training data
images, labels = tf.train.batch([image, label],
batch_size=32,
num_threads=4,
capacity=5 * 32)
print(images.shape)
images = tf.cast(images, tf.float32)
# Build a Graph that computes the logits predictions from the
# inference model.
logits = vgg.inference(images)
predictions = tf.argmax(logits, 1)
labels = tf.squeeze(labels)
top_k_op = slim.metrics.streaming_accuracy(predictions, labels)
# Calculate predictions.
# top_k_op = tf.nn.in_top_k(logits, labels, 1)
# # Restore the moving average version of the learned variables for eval.
# variable_averages = tf.train.ExponentialMovingAverage(
# vgg.MOVING_AVERAGE_DECAY)
# variables_to_restore = variable_averages.variables_to_restore()
# saver = tf.train.Saver(variables_to_restore)
# Save
list_var_names = [
'vgg_16/conv1/conv1_1/biases', 'vgg_16/conv1/conv1_1/weights',
'vgg_16/conv1/conv1_2/biases', 'vgg_16/conv1/conv1_2/weights',
'vgg_16/conv2/conv2_1/biases', 'vgg_16/conv2/conv2_1/weights',
'vgg_16/conv2/conv2_2/biases', 'vgg_16/conv2/conv2_2/weights',
'vgg_16/conv3/conv3_1/biases', 'vgg_16/conv3/conv3_1/weights',
'vgg_16/conv3/conv3_2/biases', 'vgg_16/conv3/conv3_2/weights',
'vgg_16/conv3/conv3_3/biases', 'vgg_16/conv3/conv3_3/weights',
'vgg_16/conv4/conv4_1/biases', 'vgg_16/conv4/conv4_1/weights',
'vgg_16/conv4/conv4_2/biases', 'vgg_16/conv4/conv4_2/weights',
'vgg_16/conv4/conv4_3/biases', 'vgg_16/conv4/conv4_3/weights',
'vgg_16/conv5/conv5_1/biases', 'vgg_16/conv5/conv5_1/weights',
'vgg_16/conv5/conv5_2/biases', 'vgg_16/conv5/conv5_2/weights',
'vgg_16/conv5/conv5_3/biases', 'vgg_16/conv5/conv5_3/weights',
'vgg_16/fc6/biases', 'vgg_16/fc6/weights', 'vgg_16/fc7/biases',
'vgg_16/fc7/weights', 'vgg_16/fc8/biases', 'vgg_16/fc8/weights',
'vgg_16/conv1/conv1_1/mask', 'vgg_16/conv1/conv1_2/mask',
'vgg_16/conv2/conv2_1/mask', 'vgg_16/conv2/conv2_2/mask',
'vgg_16/conv3/conv3_1/mask', 'vgg_16/conv3/conv3_2/mask',
'vgg_16/conv3/conv3_3/mask', 'vgg_16/conv4/conv4_1/mask',
'vgg_16/conv4/conv4_2/mask', 'vgg_16/conv4/conv4_3/mask',
'vgg_16/conv5/conv5_1/mask', 'vgg_16/conv5/conv5_2/mask',
'vgg_16/conv5/conv5_3/mask', 'vgg_16/fc6/mask', 'vgg_16/fc7/mask',
'vgg_16/fc8/mask'
]
var_list_to_restore = []
for name in list_var_names:
var_list_to_restore = var_list_to_restore + tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, name)
saver = tf.train.Saver(var_list_to_restore)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, g)
while True:
eval_once(saver, summary_writer, top_k_op, summary_op)
if FLAGS.run_once:
break
time.sleep(FLAGS.eval_interval_secs)
def main(config, RANDOM_SEED, LOG_DIR, TRAIN_NUM, BATCH_SIZE, LEARNING_RATE,
DECAY_VAL, DECAY_STEPS, DECAY_STAIRCASE, BETA, K, D, SAVE_PERIOD,
SUMMARY_PERIOD, **kwargs):
np.random.seed(RANDOM_SEED)
tf.set_random_seed(RANDOM_SEED)
# >>>>>>> DATASET
train_dataset = imagenet.get_split('train', 'datasets/ILSVRC2012')
valid_dataset = imagenet.get_split('validation', 'datasets/ILSVRC2012')
train_ims, _ = _build_batch(train_dataset, BATCH_SIZE, 4)
valid_ims, _ = _build_batch(valid_dataset, 4, 1)
# >>>>>>> MODEL
with tf.variable_scope('train'):
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(LEARNING_RATE,
global_step,
DECAY_STEPS,
DECAY_VAL,
staircase=DECAY_STAIRCASE)
tf.summary.scalar('lr', learning_rate)
with tf.variable_scope('params') as params:
pass
net = VQVAE(learning_rate, global_step, BETA, train_ims, K, D,
_imagenet_arch, params, True)
with tf.variable_scope('valid'):
params.reuse_variables()
valid_net = VQVAE(None, None, BETA, valid_ims, K, D, _imagenet_arch,
params, False)
with tf.variable_scope('misc'):
# Summary Operations
tf.summary.scalar('loss', net.loss)
tf.summary.scalar('recon', net.recon)
tf.summary.scalar('vq', net.vq)
tf.summary.scalar('commit', BETA * net.commit)
tf.summary.scalar('nll', tf.reduce_mean(net.nll))
tf.summary.image('origin', train_ims, max_outputs=4)
tf.summary.image('recon', net.p_x_z, max_outputs=4)
summary_op = tf.summary.merge_all()
# Initialize op
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
config_summary = tf.summary.text('TrainConfig',
tf.convert_to_tensor(
config.as_matrix()),
collections=[])
extended_summary_op = tf.summary.merge([
tf.summary.scalar('valid_loss', valid_net.loss),
tf.summary.scalar('valid_recon', valid_net.recon),
tf.summary.scalar('valid_vq', valid_net.vq),
tf.summary.scalar('valid_commit', BETA * valid_net.commit),
tf.summary.scalar('valid_nll', tf.reduce_mean(valid_net.nll)),
tf.summary.image('valid_origin', valid_ims, max_outputs=4),
tf.summary.image('valid_recon', valid_net.p_x_z, max_outputs=4),
])
# <<<<<<<<<<
# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Run!
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.graph.finalize()
sess.run(init_op)
summary_writer = tf.summary.FileWriter(LOG_DIR, sess.graph)
summary_writer.add_summary(config_summary.eval(session=sess))
try:
# Start Queueing
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
for step in tqdm(xrange(TRAIN_NUM), dynamic_ncols=True):
it, loss, _ = sess.run([global_step, net.loss, net.train_op])
if (it % SAVE_PERIOD == 0):
net.save(sess, LOG_DIR, step=it)
if (it % SUMMARY_PERIOD == 0):
tqdm.write('[%5d] Loss: %1.3f' % (it, loss))
summary = sess.run(summary_op)
summary_writer.add_summary(summary, it)
if (it % (SUMMARY_PERIOD * 2) == 0): #Extended Summary
summary = sess.run(extended_summary_op)
summary_writer.add_summary(summary, it)
except Exception as e:
coord.request_stop(e)
finally:
net.save(sess, LOG_DIR)
coord.request_stop()
coord.join(threads)
def train_prior(config, RANDOM_SEED, MODEL, TRAIN_NUM, BATCH_SIZE,
LEARNING_RATE, DECAY_VAL, DECAY_STEPS, DECAY_STAIRCASE,
GRAD_CLIP, K, D, BETA, NUM_LAYERS, NUM_FEATURE_MAPS,
SUMMARY_PERIOD, SAVE_PERIOD, **kwargs):
np.random.seed(RANDOM_SEED)
tf.set_random_seed(RANDOM_SEED)
LOG_DIR = os.path.join(os.path.dirname(MODEL), 'pixelcnn')
# >>>>>>> DATASET
train_dataset = imagenet.get_split('train', 'datasets/ILSVRC2012')
ims, labels = _build_batch(train_dataset, BATCH_SIZE, 4)
# <<<<<<<
# >>>>>>> MODEL for Generate Images
with tf.variable_scope('net'):
with tf.variable_scope('params') as params:
pass
vq_net = VQVAE(None, None, BETA, ims, K, D, _imagenet_arch, params,
False)
# <<<<<<<
# >>>>>> MODEL for Training Prior
with tf.variable_scope('pixelcnn'):
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(LEARNING_RATE,
global_step,
DECAY_STEPS,
DECAY_VAL,
staircase=DECAY_STAIRCASE)
tf.summary.scalar('lr', learning_rate)
net = PixelCNN(learning_rate, global_step, GRAD_CLIP,
vq_net.k.get_shape()[1], vq_net.embeds, K, D, 1000,
NUM_LAYERS, NUM_FEATURE_MAPS)
# <<<<<<
with tf.variable_scope('misc'):
# Summary Operations
tf.summary.scalar('loss', net.loss)
summary_op = tf.summary.merge_all()
# Initialize op
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
config_summary = tf.summary.text('TrainConfig',
tf.convert_to_tensor(
config.as_matrix()),
collections=[])
sample_images = tf.placeholder(tf.float32, [None, 128, 128, 3])
sample_summary_op = tf.summary.image('samples',
sample_images,
max_outputs=20)
# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Run!
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.graph.finalize()
sess.run(init_op)
vq_net.load(sess, MODEL)
summary_writer = tf.summary.FileWriter(LOG_DIR, sess.graph)
summary_writer.add_summary(config_summary.eval(session=sess))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
try:
for step in tqdm(xrange(TRAIN_NUM), dynamic_ncols=True):
batch_xs, batch_ys = sess.run([vq_net.k, labels])
it, loss, _ = sess.run([global_step, net.loss, net.train_op],
feed_dict={
net.X: batch_xs,
net.h: batch_ys
})
if (it % SAVE_PERIOD == 0):
net.save(sess, LOG_DIR, step=it)
sampled_zs, log_probs = net.sample_from_prior(
sess, np.random.randint(0, 1000, size=(10, )), 2)
sampled_ims = sess.run(vq_net.gen,
feed_dict={vq_net.latent: sampled_zs})
summary_writer.add_summary(
sess.run(sample_summary_op,
feed_dict={sample_images: sampled_ims}), it)
if (it % SUMMARY_PERIOD == 0):
tqdm.write('[%5d] Loss: %1.3f' % (it, loss))
summary = sess.run(summary_op,
feed_dict={
net.X: batch_xs,
net.h: batch_ys
})
summary_writer.add_summary(summary, it)
except Exception as e:
coord.request_stop(e)
finally:
net.save(sess, LOG_DIR)
coord.request_stop()
coord.join(threads)
return images, images_raw, labels
flowers_data_dir = "/home/robin/Dataset/flowers"
mnist_data_dir = "/home/robin/Dataset/mnist"
cifar10_data_dir = "/home/robin/Dataset/cifar10"
imagenet_data_dir = "/home/robin/Dataset/imaget/output_tfrecord"
if __name__ == "__main__":
with tf.Graph().as_default():
# dataset = flowers.get_split('train', flowers_data_dir) #load_batch(dataset,height=224, width=224)
# dataset = mnist.get_split("train",mnist_data_dir) #load_batch(dataset,height=28, width=28)
# dataset = cifar10.get_split("train",cifar10_data_dir) #load_batch(dataset,height=32, width=32)
dataset = imagenet.get_split(
'validation',
imagenet_data_dir) #load_batch(dataset,height=224, width=224)
batch_image, batch_raw_image, batch_labels = load_batch(
dataset, height=224, width=224, is_training=True)
one_hot_batch_labels = slim.one_hot_encoding(batch_labels,
dataset.num_classes)
print(batch_image, batch_raw_image, batch_labels)
data_provider = slim.dataset_data_provider.DatasetDataProvider(
dataset, common_queue_capacity=32, common_queue_min=1)
image, label = data_provider.get(['image', 'label'])
one_hot_labels = slim.one_hot_encoding(label, dataset.num_classes)
with tf.Session() as sess:
with slim.queues.QueueRunners(sess):
def train():
"""Train CIFAR-10 for a number of steps."""
with tf.Graph().as_default():
global_step = tf.contrib.framework.get_or_create_global_step()
# images, labels = vgg.distorted_inputs()
dataset = imagenet.get_split('train', '/data/ramyadML/TF-slim-data/imageNet/processed')
# Creates a TF-Slim DataProvider which reads the dataset in the background
# during both training and testing.
provider = slim.dataset_data_provider.DatasetDataProvider(dataset,
num_readers=4,
common_queue_capacity=20*32,
common_queue_min=10*32,
shuffle=True)
preprocessing_name = 'vgg_16'
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name,
is_training=True)
[image, label] = provider.get(['image', 'label'])
image = image_preprocessing_fn(image, 224, 224)
label -= 1
# batch up some training data
images, labels = tf.train.batch([image, label],
batch_size=32,
num_threads=4,
capacity=5*32)
print (images.shape)
images = tf.cast(images, tf.float32)
# Build a Graph that computes the logits predictions from the
# inference model.
logits = vgg.inference(images)
print ("logits shape:", logits.shape)
# Calculate loss.
print ("label shape", labels.shape)
# Calculate loss.
loss = vgg.loss(logits, labels)
# Save
list_var_names = [ 'vgg_16/conv1/conv1_1/biases',
'vgg_16/conv1/conv1_1/weights',
'vgg_16/conv1/conv1_2/biases',
'vgg_16/conv1/conv1_2/weights',
'vgg_16/conv2/conv2_1/biases',
'vgg_16/conv2/conv2_1/weights',
'vgg_16/conv2/conv2_2/biases',
'vgg_16/conv2/conv2_2/weights',
'vgg_16/conv3/conv3_1/biases',
'vgg_16/conv3/conv3_1/weights',
'vgg_16/conv3/conv3_2/biases',
'vgg_16/conv3/conv3_2/weights',
'vgg_16/conv3/conv3_3/biases',
'vgg_16/conv3/conv3_3/weights',
'vgg_16/conv4/conv4_1/biases',
'vgg_16/conv4/conv4_1/weights',
'vgg_16/conv4/conv4_2/biases',
'vgg_16/conv4/conv4_2/weights',
'vgg_16/conv4/conv4_3/biases',
'vgg_16/conv4/conv4_3/weights',
'vgg_16/conv5/conv5_1/biases',
'vgg_16/conv5/conv5_1/weights',
'vgg_16/conv5/conv5_2/biases',
'vgg_16/conv5/conv5_2/weights',
'vgg_16/conv5/conv5_3/biases',
'vgg_16/conv5/conv5_3/weights',
'vgg_16/fc6/biases',
'vgg_16/fc6/weights',
'vgg_16/fc7/biases',
'vgg_16/fc7/weights',
'vgg_16/fc8/biases',
'vgg_16/fc8/weights']
var_list_to_restore = []
for name in list_var_names:
var_list_to_restore = var_list_to_restore + tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, name)
saver = tf.train.Saver(var_list_to_restore)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = vgg.train(loss, global_step)
# Parse pruning hyperparameters
pruning_hparams = pruning.get_pruning_hparams().parse(FLAGS.pruning_hparams)
# Create a pruning object using the pruning hyperparameters
pruning_obj = pruning.Pruning(pruning_hparams, global_step=global_step)
# Use the pruning_obj to add ops to the training graph to update the masks
# The conditional_mask_update_op will update the masks only when the
# training step is in [begin_pruning_step, end_pruning_step] specified in
# the pruning spec proto
mask_update_op = pruning_obj.conditional_mask_update_op()
# Use the pruning_obj to add summaries to the graph to track the sparsity
# of each of the layers
pruning_obj.add_pruning_summaries()
class _LoggerHook(tf.train.SessionRunHook):
"""Logs loss and runtime."""
def begin(self):
self._step = -1
def before_run(self, run_context):
self._step += 1
self._start_time = time.time()
return tf.train.SessionRunArgs(loss) # Asks for loss value.
def after_run(self, run_context, run_values):
duration = time.time() - self._start_time
loss_value = run_values.results
if self._step % 10 == 0:
num_examples_per_step = 128
examples_per_sec = num_examples_per_step / duration
sec_per_batch = float(duration)
format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print(format_str % (datetime.datetime.now(), self._step, loss_value,
examples_per_sec, sec_per_batch))
with tf.train.MonitoredTrainingSession(
checkpoint_dir=FLAGS.train_dir,
hooks=[tf.train.StopAtStepHook(last_step=FLAGS.max_steps),
tf.train.NanTensorHook(loss),
_LoggerHook()],
config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement)) as mon_sess:
saver.restore(mon_sess,"trained_weights/vgg_16.ckpt")
while not mon_sess.should_stop():
mon_sess.run(train_op)
# Update the masks
mon_sess.run(mask_update_op)