def replace_read_ops(loss_or_losses, var_list):
"""
Replaces read ops of each variable in `vars` with new read ops obtained
from `read_value()`, thus forcing to read the most up-to-date values of
the variables (which might incur copies across devices).
The graph is seeded from the tensor(s) `loss_or_losses`.
"""
# ops between var ops and the loss
ops = set(ge.get_walks_intersection_ops([var.op for var in var_list], loss_or_losses))
if not ops: # loss_or_losses doesn't depend on any var in var_list, so there is nothiing to replace
return
# filter out variables that are not involved in computing the loss
var_list = [var for var in var_list if var.op in ops]
# assume that for each variable, the only op required to compute the loss
# is a read op, and there is exactly one per variable
read_ops = []
for var in var_list:
output, = var.op.outputs
read_op, = set(output.consumers()) & ops
read_ops.append(read_op)
for var, read_op in zip(var_list, read_ops):
with tf.name_scope('/'.join(read_op.name.split('/')[:-1])):
with tf.device(read_op.device):
read_t, = read_op.outputs
consumer_ops = set(read_t.consumers()) & ops
# consumer_sgv might have multiple inputs, but we only care
# about replacing the input that is read_t
consumer_sgv = ge.sgv(consumer_ops)
consumer_sgv = consumer_sgv.remap_inputs([list(consumer_sgv.inputs).index(read_t)])
ge.connect(ge.sgv(var.read_value().op), consumer_sgv)
def tensor_swapin_and_out(g, origin_op, swapin_op):
global added_control
all_ops = g.get_operations()
#find the origin_op's output tensor name
origin_op_name = origin_op.values()[0].name
added_control = False
#search the to_swapin_op which use
for op in all_ops:
for i in range(len(op.inputs)):
if ((op.inputs[i].name == origin_op_name) and
("_grad" in op.name)):
print("gradient op.name:", op.name)
"""
('op.name:', u'layer1/L1_SwapOut')
('op.name:', u'layer2/MatMul')
('op.name:', u'optimizer/gradients/layer1/Sigmoid_grad/SigmoidGrad')
"""
#Use connect and remap function to reconnect
ge.connect(ge.sgv(swapin_op), ge.sgv(op).remap_inputs([i]))
# FIXME:
# obviously we cannot add more than 1 control dependency for swap_in op
if added_control is False:
print("Control Dependency==> swapin_op:", swapin_op, "op:", op)
add_control_dependency(all_ops, swapin_op, op)
def _connect_ops(self,
src_op,
dest_op,
remap_inputs=False,
remap_outputs=False,
idx=None,
disconnect_first=False):
"""A wrapper of `tensorflow.contrib.graph_editor.connect`.
This method does an in-place modification to the graph.
Args:
src_op: a `tf.Operation`.
dest_op: a `tf.Operation`.
remap_inputs: remap the input of `dest_op` or not.
remap_outputs: remap the output of `src_op` or not.
idx: index of input or output tensor.
disconnect_first: True means the current outputs of sgv0 are
disconnected.
"""
src_sgv = ge.sgv(src_op, graph=self._graph)
dest_sgv = ge.sgv(dest_op, graph=self._graph)
if remap_outputs:
src_sgv = src_sgv.remap_outputs([idx])
if remap_inputs:
dest_sgv = dest_sgv.remap_inputs([idx])
ge.connect(src_sgv, dest_sgv, disconnect_first)
def build_pb_fact(pb_location, main_location, breath, quant, lvl):
graph = load_graph(pb_location)
W1 = graph.get_tensor_by_name('prefix/w_in:0')
matmul = graph.get_tensor_by_name('prefix/MatMul:0')
bias = graph.get_tensor_by_name('prefix/b_in:0')
add = graph.get_tensor_by_name('prefix/add:0')
reshape = graph.get_tensor_by_name('prefix/Reshape:0')
# #remove all conncetions from matmul
ge.detach(ge.sgv(matmul.op))
with tf.Session(graph=graph) as sess:
# os.system("mkdir " + main_location + breath + "/" + quant + "/fact_" + str(lvl))
# for op in sess.graph.get_operations():
# print(op.name)
W = W1.eval()
u, s, v, ss = svd_compress_gs(W, lvl)
logEntry("structural_similarity == > " + str(ss))
u1 = tf.matmul(reshape, u, name="prefix/u1")
s1 = tf.matmul(u1, s, name="prefix/s1")
v1 = tf.matmul(s1, v, name="prefix/v1")
ge.connect(ge.sgv(v1.op), ge.sgv(add.op).remap_inputs([0]))
sess.run(tf.variables_initializer([tf.Variable(5, name="dummy" + str(lvl))]))
saver = tf.train.Saver()
# save log for tensorboad
LOGDIR = main_location + '/LOG'
train_writer = tf.summary.FileWriter(LOGDIR)
train_writer.add_graph(sess.graph)
train_writer.close()
# save the freezed model
os.system("mkdir " + main_location + "pb")
tf.train.write_graph(sess.graph_def, main_location + 'pb/', "RNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt")
saver.save(sess, save_path=main_location + "model.ckpt")
input_graph_path = main_location + '/pb/' + "RNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt"
checkpoint_path = main_location + "model.ckpt"
restore_op_name = "save/restore_all"
filename_tensor_name = "save/Const:0"
output_frozen_graph_name = main_location + 'pb/' + "RNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pb"
logEntry("Start Freezing the graph")
freeze_graph.freeze_graph(input_graph_path, input_saver="",
input_binary=False, input_checkpoint=checkpoint_path,
output_node_names="prefix/y_", restore_op_name="save/restore_all",
filename_tensor_name="save/Const:0",
output_graph=output_frozen_graph_name, clear_devices=True, initializer_nodes="")
logEntry("End Freezing the graph")
sess.close()
def _connect_ops(src_op, dest_op, remap_inputs=False, remap_outputs=True):
src_sgv = ge.sgv(src_op, graph=tf.get_default_graph())
dest_sgv = ge.sgv(dest_op, graph=tf.get_default_graph())
if remap_outputs:
src_sgv = src_sgv.remap_outputs([0])
if remap_inputs:
dest_sgv = dest_sgv.remap_inputs([0])
ge.connect(src_sgv, dest_sgv)
def test_connect(self):
"""Test for ge.connect."""
with self.graph.as_default():
x = tf.constant([1., 1.], shape=[2], name="x")
y = tf.constant([2., 2.], shape=[2], name="y")
z = tf.add(x, y, name="z")
sgv = ge.sgv(x.op, y.op, z.op)
ge.connect(sgv, ge.sgv(self.e.op).remap_inputs([0]))
self.assertTrue(ge.matcher("^foo/bar/e$").input_ops("^z$", "foo/d$")
(self.e.op))
def remove_dropout_node(graph, output_op_name, input_op_name,
dst_node_name_list):
output_op = graph.get_operation_by_name(output_op_name)
input_op = graph.get_operation_by_name(input_op_name)
ge.connect(output_op, input_op)
graph_def = graph.as_graph_def()
# display_node(graph_def);
new_graph_def = tf.graph_util.extract_sub_graph(graph_def,
dst_node_name_list)
return new_graph_def
def build_pb_fact(pb_location, main_location, breath, quant, lvl):
graph = load_graph(pb_location)
with tf.Session(graph=graph) as sess:
count = 0
pre_layer = [
'Placeholder', 'hidden_layer1', 'hidden_layer2', 'hidden_layer3',
'hidden_layer4', 'hidden_layer5', 'hidden_layer6'
]
svd_num = [2048, 1024, 512, 256, 128, 64, 32]
for layer in [
'hidden_layer1', 'hidden_layer2', 'hidden_layer3',
'hidden_layer4', 'hidden_layer5', 'hidden_layer6',
'hidden_layer7'
]:
if (count == 0):
layer_input = graph.get_tensor_by_name('prefix/' +
pre_layer[count] + ':0')
else:
layer_input = graph.get_tensor_by_name('prefix/' +
pre_layer[count] +
'/Tanh:0')
W = graph.get_tensor_by_name('prefix/' + layer + '/kernel:0')
matmul = graph.get_tensor_by_name('prefix/' + layer + '/MatMul:0')
add = graph.get_tensor_by_name('prefix/' + layer + '/BiasAdd:0')
ge.detach(ge.sgv(matmul.op))
W1 = W.eval()
print(W1.shape)
u_1, s_1, v_1, ss = svd_compress_gs(
W1, math.ceil(svd_num[count] * lvl / 100))
logEntry(
str(lvl) + " ==> layer ==> " + str(count) + " ssim ==> " +
str(ss))
u1 = tf.matmul(layer_input, u_1, name="prefix/" + layer + "/u1")
s1 = tf.matmul(u1, s_1, name="prefix/" + layer + "/s1")
v1 = tf.matmul(s1, v_1, name="prefix/" + layer + "/v1")
ge.connect(ge.sgv(v1.op), ge.sgv(add.op).remap_inputs([0]))
count += 1
sess.run(
tf.variables_initializer([tf.Variable(5,
name="dummy" + str(lvl))]))
saver = tf.train.Saver()
# save log for tensorboad
LOGDIR = main_location + '/LOG'
train_writer = tf.summary.FileWriter(LOGDIR)
train_writer.add_graph(sess.graph)
train_writer.close()
# save the freezed model
os.system("mkdir " + main_location + "pb")
tf.train.write_graph(
sess.graph_def, main_location + 'pb/',
"DNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt")
saver.save(sess, save_path=main_location + "model.ckpt")
input_graph_path = main_location + '/pb/' + "DNN_" + breath + "_" + quant + "_fact_" + str(
lvl) + ".pbtxt"
checkpoint_path = main_location + "model.ckpt"
restore_op_name = "save/restore_all"
filename_tensor_name = "save/Const:0"
output_frozen_graph_name = main_location + 'pb/' + "DNN_" + breath + "_" + quant + "_fact_" + str(
lvl) + ".pb"
logEntry("Start Freezing the graph")
freeze_graph.freeze_graph(input_graph_path,
input_saver="",
input_binary=False,
input_checkpoint=checkpoint_path,
output_node_names="prefix/softmax",
restore_op_name="save/restore_all",
filename_tensor_name="save/Const:0",
output_graph=output_frozen_graph_name,
clear_devices=True,
initializer_nodes="")
logEntry("End Freezing the graph")
sess.close()
def build_pb(file, lvl, svd_num):
tf.reset_default_graph()
graph = load_graph(file)
logEntry(svd_num)
with tf.Session(graph=graph) as sess:
count = 0
pre_layer = [
'Placeholder', 'hidden_layer1', 'hidden_layer2', 'hidden_layer3',
'hidden_layer4', 'hidden_layer5', 'hidden_layer6'
]
# svd_num = [40, 8, 8, 8, 8, 8, 8]
for layer in [
'hidden_layer1', 'hidden_layer2', 'hidden_layer3',
'hidden_layer4', 'hidden_layer5', 'hidden_layer6',
'hidden_layer7'
]:
if (count == 0):
layer_input = graph.get_tensor_by_name('prefix/' +
pre_layer[count] + ':0')
else:
layer_input = graph.get_tensor_by_name('prefix/' +
pre_layer[count] +
'/Tanh:0')
W = graph.get_tensor_by_name('prefix/' + layer + '/kernel:0')
matmul = graph.get_tensor_by_name('prefix/' + layer + '/MatMul:0')
add = graph.get_tensor_by_name('prefix/' + layer + '/BiasAdd:0')
W1 = W.eval()
print(W1.shape)
u_1, s_1, v_1, ss = svd_compress_gs(W1, svd_num[count])
logEntry(
str(lvl) + " ==> layer ==> " + str(count) + " ssim ==> " +
str(ss))
u1 = tf.matmul(layer_input, u_1, name="prefix/" + layer + "/u1")
s1 = tf.matmul(u1, s_1, name="prefix/" + layer + "/s1")
v1 = tf.matmul(s1, v_1, name="prefix/" + layer + "/v1")
ge.connect(ge.sgv(v1.op), ge.sgv(add.op).remap_inputs([0]))
count += 1
a = tf.Variable(5, name="dummy")
sess.run(tf.variables_initializer([a]))
saver = tf.train.Saver()
os.system("mkdir /flush1/raj034/DNN/model/test_cases/" + breath + "/" +
quant + "/fact_" + str(lvl))
os.system("mkdir /flush1/raj034/DNN/model/test_cases/" + breath + "/" +
quant + "/fact_" + str(lvl) + "/pb")
LOGDIR = "/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + "/fact_" + str(
lvl) + "/LOG"
train_writer = tf.summary.FileWriter(LOGDIR)
train_writer.add_graph(sess.graph)
train_writer.close()
tf.train.write_graph(
sess.graph_def, "/flush1/raj034/DNN/model/test_cases/" + breath +
"/" + quant + '/fact_' + str(lvl) + '/pb/',
"DNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt")
saver.save(sess,
save_path="/flush1/raj034/DNN/model/test_cases/" + breath +
"/" + quant + '/fact_' + str(lvl) + "/model.ckpt")
input_graph_path = "/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + '/fact_' + \
str(lvl) + '/pb/' + "DNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt"
checkpoint_path = "/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + '/fact_' + str(
lvl) + "/model.ckpt"
restore_op_name = "save/restore_all"
filename_tensor_name = "save/Const:0"
output_frozen_graph_name = "/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + '/fact_' + \
str(lvl) + '/pb/' + "DNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pb"
logEntry("Start Freezing the graph")
freeze_graph.freeze_graph(input_graph_path,
input_saver="",
input_binary=False,
input_checkpoint=checkpoint_path,
output_node_names="prefix/softmax",
restore_op_name="save/restore_all",
filename_tensor_name="save/Const:0",
output_graph=output_frozen_graph_name,
clear_devices=True,
initializer_nodes="")
logEntry("End Freezing the graph")
sess.close()
# print(op.control_inputs, ge.get_ops_ios(op, False))
for i, relu_op in enumerate(relu_ops):
relu_node = g.get_tensor_by_name(relu_op.name + ":0")
new_name = '/'.join(relu_op.name.split('/')[:-1]) + "/wrapper_to_relu_" + str(i)
wrapper = tf.identity(relu_node, name=new_name)
# ge.detach_inputs(wrapper)
# ge.swap_outputs(relu_node, wrapper)
# ge.connect(relu_node, wrapper)
print("here", relu_node, wrapper)
post_relu_ops = ge.get_consuming_ops([relu_node])
print(post_relu_ops, ge.get_consuming_ops([wrapper]))
wrapper_op = tf.get_default_graph().get_operation_by_name("tower0/linear3/output")
print(wrapper_op, post_relu_ops[0])
ge.connect(wrapper_op, post_relu_ops[0])
print(post_relu_ops, ge.get_consuming_ops([wrapper]))
print("after")
for op in g.get_operations():
if 'wrapper' in op.name:
print(op.name, op.type, op.values())
# node = g.get_tensor_by_name(op.name + ":0")
# print(ge.sgv(node))
if 'Relu' in op.type and 'tower0' in op.name and 'gradients' not in op.name:
print(op.name, op.type, op.values())
# node = g.get_tensor_by_name(op.name + ":0")
# print(ge.sgv(node))
# saver = tf.train.Saver()
a = tf.train.export_meta_graph('wrapperrelutest/model.meta', as_text=True)
def main():
with tf.name_scope('inputs'):
x_update_ = tf.placeholder(dtype=tf.float32, shape=(None, None, 1))
x_meta_ = tf.placeholder(dtype=tf.float32, shape=(None, None, 1))
y_update_ = tf.placeholder(dtype=tf.float32, shape=(None, None, 1))
y_meta_ = tf.placeholder(dtype=tf.float32, shape=(None, None, 1))
updated_parameters = list()
update_losses = list()
# ______________________________________________________________________________________________
# trainable update learning rate
lr = tf.Variable(1., dtype=tf.float32)
lr = tf.nn.sigmoid(lr) * 1e-2
non_model_trainable_variables = tf.trainable_variables()
model_variables = []
x_update_tasks = tf.unstack(x_update_, num=FLAGS.meta_batch_size)
y_update_tasks = tf.unstack(y_update_, num=FLAGS.meta_batch_size)
for j in range(FLAGS.meta_batch_size):
updated_parameters.append([])
update_losses.append([])
for i in range(FLAGS.n_update_steps):
with tf.name_scope('update_{}_task_{}'.format(i, j)):
with tf.variable_scope('model', reuse=(i != 0 or j != 0)):
y_update = model(x_update_tasks[j])
if i == 0 and j == 0:
# __________________________________________________________________________________
# determine the models variables
model_variables = \
list(item for item in tf.trainable_variables() if item not in non_model_trainable_variables)
if i == 0:
updated_parameters[j].append(model_variables)
update_loss = tf.reduce_mean((y_update - y_update_tasks[j])**2)
# __________________________________________________________________________________
# ATTENTION
# need to differentiate wrt the original parameters because in the graph the updated steps
# are until now only connected to the original parameters
grads = tf.gradients(update_loss, model_variables)
updated_parameters[j].append(
list(v - lr * g
for v, g in zip(updated_parameters[j][-1], grads)))
update_losses[j].append(update_loss)
logger.info('Graph built')
y_task_meta = []
for j in range(FLAGS.meta_batch_size):
with tf.name_scope('meta_task_{}'.format(j)):
with tf.variable_scope('model', reuse=True):
y_meta = model(x_meta_)
y_task_meta.append(y_meta)
with tf.name_scope('meta_loss'):
meta_loss = tf.reduce_mean((tf.stack(y_task_meta) - y_meta_)**2)
# ______________________________________________________________________________________________
# connect updated parameters to each update step
for i in range(1, FLAGS.n_update_steps):
for j in range(FLAGS.meta_batch_size):
pv = ge.sgv(*updated_parameters[j][i])
s = ge.sgv_scope('update_{}_task_{}'.format(i, j),
tf.get_default_graph())
ind = list(
s.input_index(tf.convert_to_tensor(a))
for a in model_variables)
s = s.remap_inputs(ind)
ge.connect(pv, s)
# ______________________________________________________________________________________________
# connect final updated parameters
for j in range(FLAGS.meta_batch_size):
pv = ge.sgv(*updated_parameters[j][-1])
s = ge.sgv_scope('meta_task_{}'.format(j), tf.get_default_graph())
ind = list(
s.input_index(tf.convert_to_tensor(a)) for a in model_variables)
s = s.remap_inputs(ind)
ge.connect(pv, s)
logger.info('Graph connected')
with tf.name_scope('optimize'):
optimizer = tf.train.AdamOptimizer(FLAGS.meta_lr)
update_step = optimizer.minimize(meta_loss)
def gen_data(phase):
x_batch_update = np.random.uniform(-1,
1,
size=(
FLAGS.meta_batch_size,
n_batch,
))
x_batch_meta = np.random.uniform(-1,
1,
size=(
FLAGS.meta_batch_size,
n_batch,
))
y_batch_update = np.sin(x_batch_update + phase)
y_batch_meta = np.sin(x_batch_meta + phase)
return x_batch_update, y_batch_update, x_batch_meta, y_batch_meta
board_writer = tf.summary.FileWriter('tensorboard_log',
tf.get_default_graph())
n_batch = 10
print_every = 10
with tf.Session() as session:
session.run(tf.global_variables_initializer())
for j in range(1000):
phase = np.random.uniform(0, np.pi,
size=FLAGS.meta_batch_size)[:, None]
x_batch_update, y_batch_update, x_batch_meta, y_batch_meta = gen_data(
phase)
feed_dict = {
x_update_: x_batch_update[:, :, None],
x_meta_: x_batch_meta[:, :, None],
y_update_: y_batch_update[:, :, None],
y_meta_: y_batch_meta[:, :, None]
}
session.run(update_step, feed_dict=feed_dict)
if j % print_every == 0:
phase = np.random.uniform(0, np.pi,
size=FLAGS.meta_batch_size)[:, None]
x_batch_update, y_batch_update, x_batch_meta, y_batch_meta = gen_data(
phase)
feed_dict = {
x_update_: x_batch_update[:, :, None],
x_meta_: x_batch_meta[:, :, None],
y_update_: y_batch_update[:, :, None],
y_meta_: y_batch_meta[:, :, None]
}
first_update_losses = list(a[0] for a in update_losses)
u, m = session.run([first_update_losses, meta_loss],
feed_dict=feed_dict)
print('{:.4g} | {:.4g}'.format(np.mean(u), np.mean(m)))
def __call__(self, model, update_loss_function, x_update_=None, y_update_=None, x_meta_=None):
updated_parameters = list()
update_losses = list()
non_model_trainable_variables = tf.trainable_variables()
model_variables = []
if x_meta_ is None:
x_meta_ = tf.placeholder(dtype=tf.float32, shape=(None, x_update_.get_shape[1]))
self.x_update_ = x_update_
self.y_update_ = y_update_
self.x_meta_ = x_meta_
with tf.variable_scope(self.name):
for i in range(self.n_update_steps):
with tf.name_scope('update_{}'.format(i)):
with tf.variable_scope('model', reuse=(i != 0)):
y_update = model(x_update_)
if i == 0:
# __________________________________________________________________________
# determine the models variables
model_variables = list(item for item in tf.trainable_variables()
if item not in non_model_trainable_variables)
updated_parameters.append(model_variables)
update_loss = update_loss_function(y_update, y_update_)
update_losses.append(update_loss)
# ______________________________________________________________________________
# ATTENTION:
# need to differentiate wrt the original parameters, since the updated model
# steps are still connected to the original parameters.
grads = tf.gradients(update_loss, model_variables)
if self.stop_gradients:
grads = list(tf.stop_gradient(grad) for grad in grads)
updated_parameters.append(
list(v - self.update_lr * g for v, g in zip(updated_parameters[-1], grads)))
with tf.name_scope('meta'):
with tf.variable_scope('model', reuse=True):
y_meta = model(x_meta_)
# ______________________________________________________________________________________
# connect updated parameters to each update step
for i in range(1, self.n_update_steps):
pv = ge.sgv(*updated_parameters[i])
s = ge.sgv_scope('{}/update_{}'.format(self.name, i), tf.get_default_graph())
ind = list(s.input_index(tf.convert_to_tensor(a)) for a in model_variables)
s = s.remap_inputs(ind)
ge.connect(pv, s)
# _____________________________________________________________________________________
# connect final updated parameters
pv = ge.sgv(*updated_parameters[-1])
s = ge.sgv_scope('{}/meta'.format(self.name), tf.get_default_graph())
ind = list(s.input_index(tf.convert_to_tensor(a)) for a in model_variables)
s = s.remap_inputs(ind)
ge.connect(pv, s)
# _________________________________________________________________________________________
# for debug and evaluation purposes
self.update_losses = update_losses
self.updated_parameters = updated_parameters
return y_meta
def trainESL(checkpoint_path, checkpoint_file, checkpoint_epoch):
global trX, trY, teX, teY
if not os.path.exists(checkpoint_path):
os.makedirs(checkpoint_path)
##### Modify the graph #####
sess = tf.Session()
all_ops = sess.graph.get_operations()
print("all_ops:", all_ops)
#l1_swap_in = sess.graph.get_operation_by_name("layer1/L1_SwapIn")
#print l1_swap_in.outputs[0]
#matmul_1_grad = sess.graph.get_operation_by_name("optimizer/gradients/layer2/MatMul_grad/MatMul_1")
#print matmul_1_grad.inputs[1]
#ret = ge.connect(ge.sgv(l1_swap_in), ge.sgv(matmul_1_grad).remap_inputs([0]))
ret = ge.connect(
ge.sgv("layer1/L1_SwapIn", graph=tf.get_default_graph()),
ge.sgv("optimizer/gradients/layer2/MatMul_grad/MatMul_1",
graph=tf.get_default_graph()).remap_inputs([0]))
sess.close()
graph = tf.get_default_graph()
writer = tf.summary.FileWriter("./simple_graph_events")
writer.add_graph(graph=graph)
############################
# Launch the graph
with tf.Session() as sess:
merged = tf.summary.merge_all()
# tf.train.SummaryWriter soon be deprecated, use following
writer = tf.summary.FileWriter("logs/", sess.graph)
# tf.initialize_all_variables() no long valid from
# 2017-03-02 if using tensorflow >= 0.12
sess.run(tf.global_variables_initializer())
# restore all variables
"""
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if ckpt and ckpt.model_checkpoint_path:
print(ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
"""
# Training cycle
for epoch in range(checkpoint_epoch + 1):
# Shuffle the data before each training iteration.
p = np.random.permutation(range(len(trX)))
trX, trY = trX[p], trY[p]
# Train in batches of 4 inputs
for start in range(0, n_samples, BATCH_SIZE):
end = start + BATCH_SIZE
sess.run(optimizer,
feed_dict={
X: trX[start:end],
Y: trY[start:end]
})
# Record the summary of weights and biases
if epoch % 50 == 0:
rs = sess.run(merged, feed_dict={X: trX, Y: trY})
writer.add_summary(rs, epoch)
# And print the current accuracy on the training data.
if epoch % 100 == 0:
saver.save(sess,
os.path.join(checkpoint_path, checkpoint_file),
global_step=epoch)
# Loop over all batches
# Display logs per epoch step
print("Epoch:", '%04d' % (epoch + 1), "cost=",
sess.run(cost, feed_dict={
X: trX,
Y: trY
}))
print("Optimization Finished!")
# USe testing samples to predict the result
trY_pred = sess.run(predict_op, feed_dict={X: trX})
teY_pred = sess.run(predict_op, feed_dict={X: teX})
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
print('MSE train: %.3f, test: %.3f' % (mean_squared_error(
trY, trY_pred), mean_squared_error(teY, teY_pred)))
print('R^2 train: %.3f, test: %.3f' %
(r2_score(trY, trY_pred), r2_score(teY, teY_pred)))
# Inverse_transforming for standardization
teY_pred = scalerY.inverse_transform(teY_pred)
teY = scalerY.inverse_transform(teY)
# Plot the chart
plt.figure()
plt.subplot(211)
plt.plot(teY[:, 0], 'bo', teY_pred[:, 0], 'k')
plt.legend(['Original ESL1', 'Predicted ESL1'], loc='upper left')
plt.subplot(212)
plt.plot(teY[:, 1], 'ro', teY_pred[:, 1], 'k')
plt.legend(['Original ESL2', 'Predicted ESL2'], loc='upper left')
plt.show()
def convert(file_path,
inputNodeName,
outputNodeName,
msuffix,
binaryPB,
readMode,
folderPath,
checkpointExt,
checkpointPath,
modelName,
shapeArray,
modifyshapeAttribue,
fixBatchNormal=True):
tf.reset_default_graph()
os.environ['CUDA_VISIBLE_DEVICES'] = ''
config = tf.ConfigProto(allow_soft_placement=True,
device_count={
"GPU": 0,
"CPU": 1
})
runIncommdLine = False
if (os.path.isfile(file_path)):
g_in, graph_def = loadGraph(file_path, binaryPB)
else:
raise ValueError('Can not find : %s ' % folderDir)
fixTrainingBNAndDropout = False
if (fixTrainingBNAndDropout):
fcQuantWeightRemoved = False
import tensorflow.contrib.graph_editor as ge
maxModuleNumber = 6
for i in range(1, maxModuleNumber +
1): #if fire 6 is last fire layer set to 7
blockName = 'fire' + str(i)
if (i == 1):
bnNames = [
'/bn'
] # mobile net First layer Quant model all 7 fires are '/bn'
else:
bnNames = [
'/bn'
] # mobile net ['/dw_bn','/1x1_bn'] Squeeze Net ['/bn']
useConnectMethod = False
for bnName in bnNames:
if (useConnectMethod):
nodeName = blockName + bnName + '/moments/Squeeze_1'
oldInputName = blockName + bnName + '/moments/variance'
newInputName = blockName + bnName + '/moving_variance'
node = g_in.get_operation_by_name(nodeName)
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
expDim = tf.expand_dims(
tf.expand_dims(tf.expand_dims(placeHolderNew, 0), 0),
0)
ge.detach(ge.sgv(oldInputNode))
ge.connect(ge.sgv(expDim), ge.sgv(node))
nodeName = blockName + bnName + '/moments/Squeeze'
oldInputName = blockName + bnName + '/moments/mean'
newInputName = blockName + bnName + '/moving_mean'
node = g_in.get_operation_by_name(nodeName)
print(
'%s before edit new node node.node_def .inputs[0]' %
blockName, node.node_def, node.inputs[0])
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
expDim = tf.expand_dims(
tf.expand_dims(tf.expand_dims(placeHolderNew, 0), 0),
0)
ge.detach(ge.sgv(oldInputNode))
ge.connect(ge.sgv(expDim), ge.sgv(node))
print(
'%s after edit new node node.node_def .inputs[0]' %
blockName, node.node_def, node.inputs[0])
else:
oldInputName = blockName + bnName + '/moments/Squeeze_1'
newInputName = blockName + bnName + '/moving_variance'
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
ge.swap_outputs(ge.sgv(placeHolderNew),
ge.sgv(oldInputNode))
oldInputName = blockName + bnName + '/moments/Squeeze'
newInputName = blockName + bnName + '/moving_mean'
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
ge.swap_outputs(ge.sgv(placeHolderNew),
ge.sgv(oldInputNode))
removeWeightQuantize = False
if (removeWeightQuantize):
oldInputName = blockName + '/conv3x3/add'
newInputName = blockName + '/conv3x3/kernels'
#print('%s before edit new node node.node_def .inputs[0]'%blockName, node.node_def , node.inputs[0])
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
# expDim=tf.expand_dims(tf.expand_dims(tf.expand_dims(placeHolderNew,0),0),0)
# ge.detach (ge.sgv(oldInputNode))
ge.swap_outputs(
ge.sgv(placeHolderNew),
ge.sgv(oldInputNode)) #reroute_outputs get same results
#Remove FC layer
if (fcQuantWeightRemoved == False):
oldInputName = 'logit/add'
newInputName = 'logit/weights'
#print('%s before edit new node node.node_def .inputs[0]'%blockName, node.node_def , node.inputs[0])
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
# expDim=tf.expand_dims(tf.expand_dims(tf.expand_dims(placeHolderNew,0),0),0)
# ge.detach (ge.sgv(oldInputNode))
ge.swap_outputs(ge.sgv(placeHolderNew), ge.sgv(
oldInputNode)) #reroute_outputs get same results
fcQuantWeightRemoved = True
with tf.Session(config=config, graph=g_in) as sess:
graph_def = sess.graph_def
for node in graph_def.node:
if 'dropout/mul' in node.name:
deleteDropOut = True
if (deleteDropOut):
oldInputName = 'dropout/mul'
newInputName = 'fire6/pool/MaxPool'
oldInputNode = g_in.get_operation_by_name(
oldInputName)
newInputNode = g_in.get_operation_by_name(
newInputName)
placeHolderNew = tf.identity(
newInputNode.outputs[0])
ge.swap_outputs(
ge.sgv(placeHolderNew), ge.sgv(oldInputNode)
) #reroute_outputs get same results
else:
for node in graph_def.node:
if node.name == 'dropout/keep_prob':
#node.attr['value'].tensor.float_val=tf.convert_to_tensor (1,dtype=tf.float32)
#node.attr['value'].tensor.float_val=1
node.attr['value'].tensor.CopyFrom(
tensor_util.make_tensor_proto(
1.0, dtype=tf.float32))
#node.attr['value'].value =1.0
# fix batch normal node nodes https://github.com/tensorflow/tensorflow/issues/3628
if (fixBatchNormal):
for node in graph_def.node:
if node.op == 'RefSwitch':
node.op = 'Switch'
for index in range(len(node.input)):
if 'moving_' in node.input[index]:
node.input[index] = node.input[index] + '/read'
elif node.op == 'AssignSub':
node.op = 'Sub'
if 'use_locking' in node.attr: del node.attr['use_locking']
elif node.op == 'AssignAdd':
node.op = 'Add'
if 'use_locking' in node.attr: del node.attr['use_locking']
if ('dilations') in node.attr: del node.attr['dilations']
node.device = ""
#fixVariables not Working
fixVariables = False
if (fixVariables and node.op == 'VariableV2' and
('batchnorm/var' in node.name or 'batchnorm/mean' in node.name)):
outputNodes = find_output_nodes(graph_def, node)
for index in range(len(outputNodes)):
if (outputNodes[index].op == 'Assign'):
#node.output[index] = node.output[index] + '/read'
#outputNodes[index].op ='Identity'
outputNodes[index].name = outputNodes[index].name + '/read'
print('Modified %s ' % outputNodes[index].name)
#################### Step 1 Training to inference simplification , need checkpoint and .pbtxt files from training ######################################################
graphDef = optimize_for_inference_lib.optimize_for_inference(
graph_def,
[inputNodeName], # an array of the input node(s)
[outputNodeName] if type(outputNodeName) is str else
[item for item in outputNodeName], # an array of output nodes
tf.float32.as_datatype_enum)
if (modifyshapeAttribue):
inputOpType = 'Placeholder'
for n in graphDef.node:
#print('node to modify',n.name)
if (n.name == inputNodeName):
print('node to modify', n)
setNodeAttribute(n, 'shape', shapeArray)
if (n.op != inputOpType):
print(
"--Node %s op %s set to op=%s" %
(inputNodeName, n.op, inputOpType), shapeArray)
n.op = inputOpType
print("--Name of the node - %s shape set to " % n.name,
shapeArray)
print('node after modify', n)
modifyClipValue = False
if (modifyClipValue):
for i in range(1, maxModuleNumber + 1):
blockName = 'fire' + str(i)
newClipValue = 127
clipVNodeName = blockName + '/conv3x3/Rint_1/x'
#clipnode= g_in.get_operation_by_name(clipVNodeName)
clipnode = find_node_by_name(graphDef, clipVNodeName)
print('clipnode to modify', clipnode)
setNodeConstValue(graph_def, clipnode, newClipValue)
print("--Name of the node - %s shape set to %f" %
(clipnode.name, newClipValue))
print('clipnode after modify', clipnode)
modifyFCClip = True
if (modifyFCClip and i == maxModuleNumber):
clipVNodeName = 'conv12/Rint_1/x'
clipnodeFC = find_node_by_name(graphDef, clipVNodeName)
#clipnodeFC= g_in.get_operation_by_name(clipVNodeName)
setNodeConstValue(graph_def, clipnodeFC, newClipValue)
print('clipnode after modify', clipnodeFC)
if (runIncommdLine):
copyfile(file_path, file_path + trainModelSuffix)
outputNameSuffix = '%s_frozenforInference.pb' % checkpointExt
inferenceSuffix = '.Inference'
tf.train.write_graph(graphDef,
folderPath,
checkpointPath + modelName + '.pb' + inferenceSuffix,
as_text=False)
tf.train.write_graph(graphDef,
folderPath,
checkpointPath + modelName + '.pbtxt' +
inferenceSuffix,
as_text=True)
pbfileoutput_path = checkpointPath + modelName + outputNameSuffix
checkpointfile_path = checkpointPath + modelName + checkpointExt
pbfile_path = checkpointPath + modelName + msuffix + inferenceSuffix
#################### Step 2 Frozen Inference mode ######################################################
freeze_graph.freeze_graph(
input_graph=pbfile_path,
input_saver='',
input_binary=binaryPB,
input_checkpoint=checkpointfile_path, # an array of the input node(s)
output_node_names=outputNodeName
if type(outputNodeName) is str else ",".join(outputNodeName),
restore_op_name="save/restore_all", #Unused.
filename_tensor_name="save/Const:0", # Unused.
output_graph=pbfileoutput_path, # an array of output nodes
clear_devices=True,
initializer_nodes='')
#################### Step 3 Save in tensor board ######################################################
saveTensorboardForVisualizatoin = False
if (saveTensorboardForVisualizatoin):
modelFullPath = checkpointPath + modelName + outputNameSuffix
tensorboardPath = checkpointPath + '\\tensorboard'
#if not os.path.exists(tensorboardPath):
# os.mkdir(tensorboardPath)
createTensorboard(modelFullPath, tensorboardPath)
biases['biases_h2']))
layer_final = tf.add(tf.matmul(layer_2, weights['weights_output']),
biases['biases_output'])
# Define loss and optimizer, minimize the squared error
cost = tf.reduce_mean(tf.square(layer_final - Y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
train = optimizer.minimize(cost)
# Modify the graph
from tensorflow.contrib import graph_editor as ge
sess = tf.Session()
all_ops = sess.graph.get_operations()
l1_swap_in = sess.graph.get_operation_by_name("L1_SwapIn")
matmul_1_grad = sess.graph.get_operation_by_name("gradients/MatMul_1_grad/MatMul_1")
print(ge.sgv(matmul_1_grad))
print(ge.sgv(l1_swap_in))
ret = ge.connect(ge.sgv(l1_swap_in), ge.sgv(matmul_1_grad).remap_inputs([0]))
sess.close()
# Generate graph visualization
graph = tf.get_default_graph()
writer = tf.summary.FileWriter("./simple_graph_events")
writer.add_graph(graph=graph)
# #remove all conncetions from matmul
ge.detach(ge.sgv(matmul.op))
with tf.Session(graph=graph) as sess:
os.system("mkdir /flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + "/fact_" + str(lvl))
# for op in sess.graph.get_operations():
# print(op.name)
W = W1.eval()
u, s, v, ss = svd_compress_gs(W, lvl)
logEntry("structural_similarity == > " + str(ss))
u1 = tf.matmul(reshape, u, name="prefix/u1")
s1 = tf.matmul(u1, s, name="prefix/s1")
v1 = tf.matmul(s1, v, name="prefix/v1")
ge.connect(ge.sgv(v1.op), ge.sgv(add.op).remap_inputs([0]))
sess.run(tf.variables_initializer([tf.Variable(5, name="dummy" + str(lvl))]))
saver = tf.train.Saver()
# save log for tensorboad
LOGDIR = "/flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + '/fact_' + str(lvl) + '/LOG'
train_writer = tf.summary.FileWriter(LOGDIR)
train_writer.add_graph(sess.graph)
train_writer.close()
# save the freezed model
os.system("mkdir /flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + "/fact_" + str(lvl) + "/pb")
tf.train.write_graph(sess.graph_def, "/flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + '/fact_' +
str(lvl) + '/pb/', "RNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt")
saver.save(sess, save_path="/flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + '/fact_' + str(lvl) + "/model.ckpt")
if next_op is None:
raise ValueError('No suitable next op found to connect to. Try looking at the graph or full list of forward ops')
# Add placeholder and variable
add_op = tf.add(var, delta_placeholder) # TODO - might be neater if these were created in the same scope as the variable; also might solve issue with connecting add ops within while loop
# Connect add_op output to next op input
# Create subgraph 1 (outputs)
sgv0 = ge.sgv(add_op.op)
# Create subgraph 2 (inputs)
sgv1 = ge.sgv(next_op).remap_inputs([1])
# Connect
ge.connect(sgv0, sgv1) # TODO - sort out error with tf.while loops; may not be possible: try the assign_add method first
# Define parameter update ops
update = 0.01*delta_placeholder
new_update_op = tf.assign_sub(var, update)
if gamma_update_op is not None:
gamma_update_op = tf.group(new_update_op, gamma_update_op)
else:
gamma_update_op = new_update_op
merged = tf.summary.merge_all()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
summary_writer = tf.summary.FileWriter('results\\add_noise_experiments\\', sess.graph)
l = tf.zeros_like(b)
e1 = tf.identity(l)
d = tf.placeholder(tf.bool)
e = tf.cond(d, lambda: tf.identity(a), lambda: tf.identity(e1))
e2 = tf.identity(e)
gr = tf.get_default_graph()
pp = gr.get_operation_by_name('add')
ss = ge.sgv(pp.outputs[0])
pp1 = gr.get_operation_by_name('Identity')
ss1 = ge.sgv(pp1.outputs[0])
pp2 = gr.get_operation_by_name('Identity_1')
ss2 = ge.sgv(pp2.inputs[0])
# ge.detach_inputs(c)
# x = ge.graph_replace(x,{b:e})
# ge.detach_outputs(c)
m1 = ge.connect(ss2, ss)
m2 = ge.connect(ss, ss1)
# m = ge.swap_inputs(ss,ss1)
sess = tf.InteractiveSession()
PoseTools.output_graph('ge')
sess.close()
##
with tf.Session() as sess:
q = sess.run(x, {a: np.zeros([3, 3]), d: True})
print q
q = sess.run(x, {a: np.zeros([3, 3]), d: False})
print q
