def convert_consts_to_var(graph, const_names_list):
const_var_names_pairs = []
ops_to_delete = []
with graph.as_default():
preexisting_vars = [
tf.get_variable(i.name, i.outputs[0].shape)
for i in graph.get_operations()
if i.type == "VariableV2" or i.type == "Variable"
]
var_list = []
for name in const_names_list:
tensor = graph.get_operation_by_name(name).outputs[0]
with tf.Session() as sess:
t_value = sess.run(tensor)
t_name = '{}_mpc_const_var'.format(name)
var = tf.Variable(t_value, name=t_name)
const_var_names_pairs.append((name, t_name))
var_list.append(var)
for const_name, var_name in const_var_names_pairs:
const_op = graph.get_operation_by_name(const_name)
var_op = graph.get_operation_by_name('{}/read'.format(var_name))
ge.swap_outputs(ge.sgv(const_op), ge.sgv(var_op))
ops_to_delete.append(const_op)
tf.compat.v1.variables_initializer(var_list + preexisting_vars,
'init_constvars')
return delete_nodes(graph, ops_to_delete)
def convert_consts_to_var(graph_def):
graph = get_graph_from(graph_def)
all_const_ops = set(i.name for i in graph.get_operations() if i.type == "Const")
const_names_list = list(all_const_ops - set(get_white_list(graph)))
const_var_names_pairs = []
ops_to_delete = []
with graph.as_default():
preexisting_vars = [
tf.get_variable(i.name, i.outputs[0].shape)
for i in graph.get_operations()
if i.type == "VariableV2" or i.type == "Variable"
]
var_list = []
for name in const_names_list:
tensor = graph.get_operation_by_name(name).outputs[0]
with tf.compat.v1.Session() as sess:
t_value = sess.run(tensor)
t_name = "{}_mpc_const_var".format(name)
var = tf.compat.v1.Variable(t_value, name=t_name)
var_read_op_name = var.to_proto().snapshot_name[:-2]
const_var_names_pairs.append((name, var_read_op_name))
var_list.append(var)
for const_name, var_read_name in const_var_names_pairs:
const_op = graph.get_operation_by_name(const_name)
var_op = graph.get_operation_by_name(var_read_name)
ge.swap_outputs(ge.sgv(const_op), ge.sgv(var_op))
ops_to_delete.append(const_op)
tf.compat.v1.variables_initializer(
var_list + preexisting_vars, "init_constvars"
)
return delete_nodes(graph.as_graph_def(), ops_to_delete)
def replace_nodes_with_identity(graph, nop_splits):
with graph.as_default():
for split in nop_splits:
inp_var = split.inputs[1]
identity = tf.identity(inp_var).op
ge.swap_outputs(ge.sgv(split), ge.sgv(identity))
return graph
def create_network(self):
unet = self.dep_nets
sel_layers = self.sel_layers_ndx
self.sel_layers = []
for ndx, s in enumerate(sel_layers):
self.sel_layers.append(unet.all_layers[s])
debug_layers = []
with tf.variable_scope('pose_u_att'):
for ndx, ll in enumerate(self.sel_layers):
with tf.variable_scope('att_layer_{}'.format(ndx)):
n_channels = ll.get_shape().as_list()[-1]
# no batch norm for this. 2 fully connected layers to generate the attention weights from scores.
int1 = tf.contrib.layers.fully_connected(self.ph['scores'], n_channels, activation_fn=tf.nn.relu)
wts = tf.sigmoid(tf.contrib.layers.fully_connected(int1, n_channels, activation_fn=None))
# with sigmoid the weights are > 0 and bounded
debug_layers.append(wts)
# multiply by linear to weight closer inputs more as compared to far away
t_wts = tf.range(self.att_hist, 0, -1, dtype=tf.float32) / self.att_hist
debug_layers.append(t_wts)
t_wts = tf.expand_dims(t_wts, 1)
t_wts = tf.expand_dims(t_wts, 0)
wts = tf.multiply(t_wts, wts)
debug_layers.append(wts)
# wts is no Batch x T X C
# ensure weights sum to 1 for each channel
wts_sum = tf.reduce_sum(wts, axis=1, keep_dims=True)
wts = tf.div(wts, wts_sum)
debug_layers.append(wts)
# exapnd dims to match the activations in prev_in_{}
wts = tf.expand_dims(wts, 2)
wts = tf.expand_dims(wts, 2)
# multiply wts with previous activations
cur_activations = self.ph['prev_in_{}'.format(ndx)]
att_prod = tf.multiply(wts, cur_activations)
debug_layers.append(att_prod)
# sum along the dimension 1 to get attention context.
# After att_context should have the same size as layer in.
att_context = tf.reduce_sum(att_prod, axis=1)
debug_layers.append(att_context)
# concat with current examples layer.
ll_concat = tf.concat([ll, att_context], axis=-1)
att_out = PoseCommon.conv_relu3(ll_concat, n_channels, self.ph['phase_train'], self.ph['keep_prob'])
layer_2_remove = unet.all_layers[sel_layers[ndx] + 1]
ge.swap_outputs(att_out, layer_2_remove)
self.debug_layers = debug_layers
return self.dep_nets.pred
def export_pb_tflite_model(net, file_path_meta, file_path_pb, file_path_tflite, edit_graph):
"""Export *.pb & *.tflite models from checkpoint files.
Args:
* net: network configurations
* file_path_meta: file path to the *.meta data
* file_path_pb: file path to the *.pb model
* file_path_tflite: file path to the *.tflite model
* edit_graph: whether the graph should be edited
"""
# convert checkpoint files to a *.pb model
with tf.Graph().as_default() as graph:
sess = tf.Session()
# restore the graph with inputs replaced
net_input = tf.placeholder(tf.float32, shape=net['input_shape'], name=net['input_name'])
saver = tf.train.import_meta_graph(
file_path_meta, input_map={net['input_name_ckpt']: net_input})
saver.restore(sess, file_path_meta.replace('.meta', ''))
# obtain the output node
net_logits = tf.get_collection(FLAGS.output_coll)[0]
net_output = tf.nn.softmax(net_logits, name=net['output_name'])
tf.logging.info('input: {} / output: {}'.format(net_input.name, net_output.name))
tf.logging.info('input\'s shape: {}'.format(net_input.shape))
tf.logging.info('output\'s shape: {}'.format(net_output.shape))
# replace dropout layers with identity mappings (TF-Lite does not support dropout layers)
op_outputs_old, op_outputs_new = replace_dropout_layers()
sess.close()
graph_editor.swap_outputs(op_outputs_old, op_outputs_new)
sess = tf.Session() # open a new session
saver.restore(sess, file_path_meta.replace('.meta', ''))
# edit the graph by inserting alternative routines for each convolutional layer
if edit_graph:
op_outputs_old, op_outputs_new = insert_alt_routines(sess)
sess.close()
graph_editor.swap_outputs(op_outputs_old, op_outputs_new)
sess = tf.Session() # open a new session
saver.restore(sess, file_path_meta.replace('.meta', ''))
# write the original grpah to *.pb file
graph_def = graph.as_graph_def()
graph_def = tf.graph_util.convert_variables_to_constants(sess, graph_def, [net['output_name']])
file_name_pb = os.path.basename(file_path_pb)
tf.train.write_graph(graph_def, FLAGS.model_dir, file_name_pb, as_text=False)
tf.logging.info(file_path_pb + ' generated')
# convert the *.pb model to a *.tflite model
# convert_pb_model_to_tflite(file_path_pb, file_path_tflite, net['input_name'], net['output_name'])
# tf.logging.info(file_path_tflite + ' generated')
# test *.pb & *.tflite models
test_pb_model(file_path_pb, net['input_name'], net['output_name'], net['input_data'])
def replace_node_with_const(node):
print("Trying to execute node {}".format(node.name))
graph = node.graph
with graph.as_default():
const_lists = []
with tf.Session() as sess:
for out_t in node.outputs:
const_val = sess.run(out_t)
const_op = tf.constant(const_val).op
const_lists.append(const_op)
ge.swap_outputs(ge.sgv(node), ge.sgv(const_lists))
def __swap_outputs(self, op_outputs_old, op_outputs_new):
"""Swap two list of output nodes.
Args:
* op_outputs_old: output nodes to be swapped in the old graph
* op_outputs_new: output nodes to be swapped in the new graph
"""
tf.logging.info('# of output pairs to be swapped: %d' %
len(op_outputs_old))
self.sess.close()
graph_editor.swap_outputs(op_outputs_old, op_outputs_new)
self.sess = create_session() # open a new session
def test_reroute_can_modify(self):
graph = ops.Graph()
# create a special graph where "a" is an ambiguous tensor. That is
# it is both an input and an output of the ops in sgv0.
with graph.as_default():
a = constant_op.constant(1.0, shape=[2], name="a")
b = constant_op.constant(2.0, shape=[2], name="b")
c = math_ops.add(a, b, name="c")
d = math_ops.add(a, c, name="d")
e = constant_op.constant(1.0, shape=[2], name="e")
f = constant_op.constant(2.0, shape=[2], name="f")
g = math_ops.add(e, f, name="g")
sgv0 = ge.sgv(a.op, b.op, c.op)
sgv1 = ge.sgv(e.op, f.op)
ge.swap_outputs(sgv0, sgv1)
self.assertTrue(
match.OpMatcher("g").input_ops(
"a", match.OpMatcher("c").input_ops("a", "b"))(g.op))
self.assertTrue(match.OpMatcher("d").input_ops("e", "f")(d.op))
def convert_consts_to_var(graph, const_names_list):
const_var_names_pairs = []
ops_to_delete = []
with graph.as_default():
var_list = []
for name in const_names_list:
#tensor = graph.get_tensor_by_name('{}:0'.format(name))
tensor = graph.get_operation_by_name(name).outputs[0]
with tf.Session() as sess:
t_value = sess.run(tensor)
t_name = '{}_const_var'.format(name)
var = tf.Variable(t_value, name=t_name)
const_var_names_pairs.append((name, t_name))
var_list.append(var)
for const_name, var_name in const_var_names_pairs:
const_op = graph.get_operation_by_name(const_name)
var_op = graph.get_operation_by_name('{}/read'.format(var_name))
ge.swap_outputs(ge.sgv(const_op), ge.sgv(var_op))
ops_to_delete.append(const_op)
tf.compat.v1.variables_initializer(var_list, 'init_constvars')
return delete_nodes(graph, ops_to_delete)
def test_reroute_can_modify(self):
graph = ops.Graph()
# create a special graph where "a" is an ambiguous tensor. That is
# it is both an input and an output of the ops in sgv0.
with graph.as_default():
a = constant_op.constant(1.0, shape=[2], name="a")
b = constant_op.constant(2.0, shape=[2], name="b")
c = math_ops.add(a, b, name="c")
d = math_ops.add(a, c, name="d")
e = constant_op.constant(1.0, shape=[2], name="e")
f = constant_op.constant(2.0, shape=[2], name="f")
g = math_ops.add(e, f, name="g")
sgv0 = ge.sgv(a.op, b.op, c.op)
sgv1 = ge.sgv(e.op, f.op)
ge.swap_outputs(sgv0, sgv1)
self.assertTrue(
match.OpMatcher("g").input_ops(
"a",
match.OpMatcher("c").input_ops("a", "b"))(g.op))
self.assertTrue(match.OpMatcher("d").input_ops("e", "f")(d.op))
def export_pb_tflite_model(net, meta_path, pb_path, tflite_path):
"""Export *.pb & *.tflite models from checkpoint files.
Args:
* net: network configurations
* meta_path: path to the *.meta file
* pb_path: path to the *.pb file
* tflite_path: path to the *.tflite file
"""
# convert checkpoint files to a *.pb model
with tf.Graph().as_default() as graph:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True # pylint: disable=no-member
sess = tf.Session(config=config)
# restore the graph with inputs replaced
net_input = tf.placeholder(tf.float32,
shape=net['input_shape'],
name=net['input_name'])
saver = tf.train.import_meta_graph(
meta_path, input_map={net['input_name_ckpt']: net_input})
saver.restore(sess, meta_path.replace('.meta', ''))
# obtain the data format and determine which graph transformation method to be used
data_format = get_data_format()
graph_trans_mthd = 'gather' if data_format == 'NCHW' else '1x1_conv'
# obtain the output node
net_logits = tf.get_collection(FLAGS.output_coll)[0]
net_output = tf.nn.softmax(net_logits, name=net['output_name'])
tf.logging.info('input: {} / output: {}'.format(
net_input.name, net_output.name))
tf.logging.info('input\'s shape: {}'.format(net_input.shape))
tf.logging.info('output\'s shape: {}'.format(net_output.shape))
# replace dropout layers with identity mappings (TF-Lite does not support dropout layers)
op_outputs_old, op_outputs_new = replace_dropout_layers()
sess.close()
graph_editor.swap_outputs(op_outputs_old, op_outputs_new)
sess = tf.Session(config=config) # open a new session
saver.restore(sess, meta_path.replace('.meta', ''))
# edit the graph by inserting alternative routines for each convolutional layer
if FLAGS.enbl_chn_prune:
op_outputs_old, op_outputs_new = insert_alt_routines(
sess, graph_trans_mthd)
sess.close()
graph_editor.swap_outputs(op_outputs_old, op_outputs_new)
sess = tf.Session(config=config) # open a new session
saver.restore(sess, meta_path.replace('.meta', ''))
# write the original grpah to *.pb file
graph_def = graph.as_graph_def()
graph_def = tf.graph_util.convert_variables_to_constants(
sess, graph_def, [net['output_name']])
file_name_pb = os.path.basename(pb_path)
tf.train.write_graph(graph_def,
FLAGS.model_dir,
file_name_pb,
as_text=False)
tf.logging.info(pb_path + ' generated')
# convert the *.pb model to a *.tflite model
convert_pb_model_to_tflite(net, pb_path, tflite_path)
# test *.pb & *.tflite models
test_pb_model(pb_path, net['input_name'], net['output_name'],
net['input_data'])
test_tflite_model(tflite_path, net['input_data'])
def export_pb_tflite_model(net, file_path_meta, file_path_pb,
file_paths_tflite):
"""Export *.pb & *.tflite models from checkpoint files.
Args:
* net: network configurations
* file_path_meta: file path to the *.meta data
* file_path_pb: file path to the *.pb model
* file_paths_tflite: dictionary of file paths to *.tflite models
"""
# convert checkpoint files to a *.pb model
with tf.Graph().as_default() as graph:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True # pylint: disable=no-member
sess = tf.Session(config=config)
# restore the graph with inputs replaced
net_input = tf.placeholder(tf.float32,
shape=net['input_shape'],
name=net['input_name'])
saver = tf.train.import_meta_graph(
file_path_meta, input_map={net['input_name_ckpt']: net_input})
saver.restore(sess, file_path_meta.replace('.meta', ''))
# obtain the output node
net_logits = tf.get_collection(FLAGS.output_coll)[0]
net_output = tf.nn.softmax(net_logits, name=net['output_name'])
tf.logging.info('input: {} / output: {}'.format(
net_input.name, net_output.name))
tf.logging.info('input\'s shape: {}'.format(net_input.shape))
tf.logging.info('output\'s shape: {}'.format(net_output.shape))
# replace dropout layers with identity mappings (TF-Lite does not support dropout layers)
op_outputs_old, op_outputs_new = replace_dropout_layers()
sess.close()
graph_editor.swap_outputs(op_outputs_old, op_outputs_new)
sess = tf.Session(config=config) # open a new session
saver.restore(sess, file_path_meta.replace('.meta', ''))
# write the original grpah to *.pb file
graph_def = graph.as_graph_def()
graph_def = tf.graph_util.convert_variables_to_constants(
sess, graph_def, [net['output_name']])
file_name_pb = os.path.basename(file_path_pb)
tf.train.write_graph(graph_def,
FLAGS.model_dir,
file_name_pb,
as_text=False)
tf.logging.info(file_path_pb + ' generated')
# convert the *.pb model to a *.tflite model
# convert_pb_model_to_tflite(net, file_path_pb, file_paths_tflite['float'], enbl_quant=False)
# convert_pb_model_to_tflite(net, file_path_pb, file_paths_tflite['quant'], enbl_quant=True)
convert_pb_model_to_tflite(net, file_path_pb, file_paths_tflite)
# test *.pb & *.tflite models
tf.logging.info('test pb file')
test_pb_model(file_path_pb, net['input_name'], net['output_name'],
net['input_data'])
tf.logging.info('test original tflite file')
test_tflite_model(file_paths_tflite['original'], net['input_data'])
tf.logging.info('test post_quant tflite file')
test_tflite_model(file_paths_tflite['post_quant'], net['input_data'])
tf.logging.info('test infer_quant tflite file')
test_tflite_model(file_paths_tflite['infer_quant'],
net['input_data'].astype(np.uint8))
def get_model(model_path, batch_size, z_sdev, fixed_init=False):
assert os.path.exists(model_path), f'model_path does not exist: {model_path}'
with tf.gfile.GFile(model_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def)
inputs = {
'dec_eps_0': 'Placeholder',
'dec_eps_1': 'Placeholder_1',
'dec_eps_2': 'Placeholder_2',
'dec_eps_3': 'Placeholder_3',
'dec_eps_4': 'Placeholder_4',
'dec_eps_5': 'Placeholder_5',
'enc_x': 'input/image',
'enc_x_d': 'input/downsampled_image',
'enc_y': 'input/label'
}
outputs = {
'dec_x': 'model_1/Reshape_4',
'enc_eps_0': 'model/pool0/truediv_1',
'enc_eps_1': 'model/pool1/truediv_1',
'enc_eps_2': 'model/pool2/truediv_1',
'enc_eps_3': 'model/pool3/truediv_1',
'enc_eps_4': 'model/pool4/truediv_1',
'enc_eps_5': 'model/truediv_4'
}
eps_shapes = [(128, 128, 6), (64, 64, 12), (32, 32, 24),
(16, 16, 48), (8, 8, 96), (4, 4, 384)]
eps_sizes = [np.prod(e) for e in eps_shapes]
eps_size = 256 * 256 * 3
dec_eps = []
dec_eps_shapes = [(batch_size,128, 128, 6), (batch_size,64, 64, 12), (batch_size,32, 32, 24),
(batch_size,16, 16, 48), (batch_size,8, 8, 96), (batch_size,4, 4, 384)]
# replace the decoder placeholders with differentiable variables
target_var_name_pairs = []
for i in range(6):
# name of i-th decoder placeholder
name = 'import/' + inputs[f'dec_eps_{i}']
var_shape = dec_eps_shapes[i]
# Give each variable a name that doesn't already exist in the graph
var_name = f'dec_eps_{i}_turned_var'
# Create TensorFlow variable initialized by values of original const.
# var = tf.get_variable(name=var_name, dtype='float32', shape=var_shape,initializer=tf.constant_initializer(tensor_as_numpy_array))
if not fixed_init:
var = tf.get_variable(name=var_name, dtype='float32', shape=var_shape,initializer=tf.random_normal_initializer(stddev=z_sdev))
else:
init_value = np.load(f'./initializations/dec_eps_{i}_sdev_1.npy')
init_value = init_value.repeat(batch_size, 0)
var = tf.get_variable(name=var_name, dtype='float32', shape=var_shape,initializer=tf.constant_initializer(z_sdev * init_value))
# We want to keep track of our variables names for later.
target_var_name_pairs.append((name, var_name))
# add new variable to list
dec_eps.append(var)
# At this point, we added a bunch of tf.Variables to the graph, but they're
# not connected to anything.
# The magic: we use TF Graph Editor to swap the Constant nodes' outputs with
# the outputs of our newly created Variables.
for const_name, var_name in target_var_name_pairs:
const_op = tf.get_default_graph().get_operation_by_name(const_name)
var_reader_op = tf.get_default_graph().get_operation_by_name(var_name + '/read')
ge.swap_outputs(ge.sgv(const_op), ge.sgv(var_reader_op))
# remove floor operations from the graph
floor_op = tf.get_default_graph().get_operation_by_name('import/model/Floor_1')
div_op = tf.get_default_graph().get_operation_by_name('import/model/truediv_2')
ge.swap_outputs(ge.sgv(floor_op), ge.sgv(div_op))
# remove random noise from encoder
c_op = tf.get_default_graph().get_operation_by_name('import/model/random_uniform/sub/_4108__cf__4108')
c_ = tf.constant(0, dtype=tf.float32)
c_zero_op = tf.get_default_graph().get_operation_by_name('Const')
ge.swap_outputs(ge.sgv(c_op), ge.sgv(c_zero_op))
n_eps = 6
def get(name):
return tf.get_default_graph().get_tensor_by_name('import/' + name + ':0')
# Encoder
enc_x = get(inputs['enc_x'])
enc_eps = [get(outputs['enc_eps_' + str(i)]) for i in range(n_eps)]
enc_x_d = get(inputs['enc_x_d'])
enc_y = get(inputs['enc_y'])
# Decoder
dec_x = get(outputs['dec_x'])
def encode(img):
if len(img.shape) == 3:
img = np.expand_dims(img, 0)
bs = img.shape[0]
assert img.shape[1:] == (256, 256, 3)
feed_dict = {enc_x: img}
update_feed(feed_dict, bs) # For unoptimized model
return flatten_eps(run(sess, enc_eps, feed_dict))
def decode(feps):
if len(feps.shape) == 1:
feps = np.expand_dims(feps, 0)
bs = feps.shape[0]
# assert len(eps) == n_eps
# for i in range(n_eps):
# shape = (BATCH_SIZE, 128 // (2 ** i), 128 // (2 ** i), 6 * (2 ** i) * (2 ** (i == (n_eps - 1))))
# assert eps[i].shape == shape
eps = unflatten_eps(feps)
feed_dict = {}
for i in range(n_eps):
feed_dict[dec_eps[i]] = eps[i]
update_feed(feed_dict, bs) # For unoptimized model
return run(sess, dec_x, feed_dict)
def random(bs=1, eps_std=0.7):
feps = np.random.normal(scale=eps_std, size=[bs, eps_size])
return decode(feps), feps
# function that updates the feed_dict to include a downsampled image
# and a conditional label set to all zeros.
def update_feed(feed_dict, bs):
x_d = 128 * np.ones([bs, 128, 128, 3], dtype=np.uint8)
y = np.zeros([bs], dtype=np.int32)
feed_dict[enc_x_d] = x_d
feed_dict[enc_y] = y
return feed_dict
feed_dict = {}
update_feed(feed_dict, batch_size)
return dec_x, dec_eps, feed_dict, run
def create_network(self):
unet = self.dep_nets
sel_layers = self.sel_layers_ndx
self.sel_layers = []
for ndx, s in enumerate(sel_layers):
self.sel_layers.append(unet.all_layers[s])
debug_layers = []
with tf.variable_scope('pose_u_att'):
for ndx, ll in enumerate(self.sel_layers):
with tf.variable_scope('att_layer_{}'.format(ndx)):
n_channels = ll.get_shape().as_list()[-1]
# no batch norm for this. 2 fully connected layers to generate the attention weights from scores.
int1 = tf.contrib.layers.fully_connected(
self.ph['scores'],
n_channels,
activation_fn=tf.nn.relu)
wts = tf.sigmoid(
tf.contrib.layers.fully_connected(int1,
n_channels,
activation_fn=None))
# with sigmoid the weights are > 0 and bounded
debug_layers.append(wts)
# multiply by linear to weight closer inputs more as compared to far away
t_wts = tf.range(self.att_hist, 0, -1,
dtype=tf.float32) / self.att_hist
debug_layers.append(t_wts)
t_wts = tf.expand_dims(t_wts, 1)
t_wts = tf.expand_dims(t_wts, 0)
wts = tf.multiply(t_wts, wts)
debug_layers.append(wts)
# wts is no Batch x T X C
# ensure weights sum to 1 for each channel
wts_sum = tf.reduce_sum(wts, axis=1, keep_dims=True)
wts = tf.div(wts, wts_sum)
debug_layers.append(wts)
# exapnd dims to match the activations in prev_in_{}
wts = tf.expand_dims(wts, 2)
wts = tf.expand_dims(wts, 2)
# multiply wts with previous activations
cur_activations = self.ph['prev_in_{}'.format(ndx)]
att_prod = tf.multiply(wts, cur_activations)
debug_layers.append(att_prod)
# sum along the dimension 1 to get attention context.
# After att_context should have the same size as layer in.
att_context = tf.reduce_sum(att_prod, axis=1)
debug_layers.append(att_context)
# concat with current examples layer.
ll_concat = tf.concat([ll, att_context], axis=-1)
att_out = PoseCommon.conv_relu3(ll_concat, n_channels,
self.ph['phase_train'],
self.ph['keep_prob'])
layer_2_remove = unet.all_layers[sel_layers[ndx] + 1]
ge.swap_outputs(att_out, layer_2_remove)
self.debug_layers = debug_layers
return self.dep_nets.pred
def convert(file_path,
inputNodeName,
outputNodeName,
msuffix,
binaryPB,
readMode,
folderPath,
checkpointExt,
checkpointPath,
modelName,
shapeArray,
modifyshapeAttribue,
fixBatchNormal=True):
tf.reset_default_graph()
os.environ['CUDA_VISIBLE_DEVICES'] = ''
config = tf.ConfigProto(allow_soft_placement=True,
device_count={
"GPU": 0,
"CPU": 1
})
runIncommdLine = False
if (os.path.isfile(file_path)):
g_in, graph_def = loadGraph(file_path, binaryPB)
else:
raise ValueError('Can not find : %s ' % folderDir)
fixTrainingBNAndDropout = False
if (fixTrainingBNAndDropout):
fcQuantWeightRemoved = False
import tensorflow.contrib.graph_editor as ge
maxModuleNumber = 6
for i in range(1, maxModuleNumber +
1): #if fire 6 is last fire layer set to 7
blockName = 'fire' + str(i)
if (i == 1):
bnNames = [
'/bn'
] # mobile net First layer Quant model all 7 fires are '/bn'
else:
bnNames = [
'/bn'
] # mobile net ['/dw_bn','/1x1_bn'] Squeeze Net ['/bn']
useConnectMethod = False
for bnName in bnNames:
if (useConnectMethod):
nodeName = blockName + bnName + '/moments/Squeeze_1'
oldInputName = blockName + bnName + '/moments/variance'
newInputName = blockName + bnName + '/moving_variance'
node = g_in.get_operation_by_name(nodeName)
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
expDim = tf.expand_dims(
tf.expand_dims(tf.expand_dims(placeHolderNew, 0), 0),
0)
ge.detach(ge.sgv(oldInputNode))
ge.connect(ge.sgv(expDim), ge.sgv(node))
nodeName = blockName + bnName + '/moments/Squeeze'
oldInputName = blockName + bnName + '/moments/mean'
newInputName = blockName + bnName + '/moving_mean'
node = g_in.get_operation_by_name(nodeName)
print(
'%s before edit new node node.node_def .inputs[0]' %
blockName, node.node_def, node.inputs[0])
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
expDim = tf.expand_dims(
tf.expand_dims(tf.expand_dims(placeHolderNew, 0), 0),
0)
ge.detach(ge.sgv(oldInputNode))
ge.connect(ge.sgv(expDim), ge.sgv(node))
print(
'%s after edit new node node.node_def .inputs[0]' %
blockName, node.node_def, node.inputs[0])
else:
oldInputName = blockName + bnName + '/moments/Squeeze_1'
newInputName = blockName + bnName + '/moving_variance'
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
ge.swap_outputs(ge.sgv(placeHolderNew),
ge.sgv(oldInputNode))
oldInputName = blockName + bnName + '/moments/Squeeze'
newInputName = blockName + bnName + '/moving_mean'
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
ge.swap_outputs(ge.sgv(placeHolderNew),
ge.sgv(oldInputNode))
removeWeightQuantize = False
if (removeWeightQuantize):
oldInputName = blockName + '/conv3x3/add'
newInputName = blockName + '/conv3x3/kernels'
#print('%s before edit new node node.node_def .inputs[0]'%blockName, node.node_def , node.inputs[0])
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
# expDim=tf.expand_dims(tf.expand_dims(tf.expand_dims(placeHolderNew,0),0),0)
# ge.detach (ge.sgv(oldInputNode))
ge.swap_outputs(
ge.sgv(placeHolderNew),
ge.sgv(oldInputNode)) #reroute_outputs get same results
#Remove FC layer
if (fcQuantWeightRemoved == False):
oldInputName = 'logit/add'
newInputName = 'logit/weights'
#print('%s before edit new node node.node_def .inputs[0]'%blockName, node.node_def , node.inputs[0])
oldInputNode = g_in.get_operation_by_name(oldInputName)
newInputNode = g_in.get_operation_by_name(newInputName)
placeHolderNew = tf.identity(newInputNode.outputs[0])
# expDim=tf.expand_dims(tf.expand_dims(tf.expand_dims(placeHolderNew,0),0),0)
# ge.detach (ge.sgv(oldInputNode))
ge.swap_outputs(ge.sgv(placeHolderNew), ge.sgv(
oldInputNode)) #reroute_outputs get same results
fcQuantWeightRemoved = True
with tf.Session(config=config, graph=g_in) as sess:
graph_def = sess.graph_def
for node in graph_def.node:
if 'dropout/mul' in node.name:
deleteDropOut = True
if (deleteDropOut):
oldInputName = 'dropout/mul'
newInputName = 'fire6/pool/MaxPool'
oldInputNode = g_in.get_operation_by_name(
oldInputName)
newInputNode = g_in.get_operation_by_name(
newInputName)
placeHolderNew = tf.identity(
newInputNode.outputs[0])
ge.swap_outputs(
ge.sgv(placeHolderNew), ge.sgv(oldInputNode)
) #reroute_outputs get same results
else:
for node in graph_def.node:
if node.name == 'dropout/keep_prob':
#node.attr['value'].tensor.float_val=tf.convert_to_tensor (1,dtype=tf.float32)
#node.attr['value'].tensor.float_val=1
node.attr['value'].tensor.CopyFrom(
tensor_util.make_tensor_proto(
1.0, dtype=tf.float32))
#node.attr['value'].value =1.0
# fix batch normal node nodes https://github.com/tensorflow/tensorflow/issues/3628
if (fixBatchNormal):
for node in graph_def.node:
if node.op == 'RefSwitch':
node.op = 'Switch'
for index in range(len(node.input)):
if 'moving_' in node.input[index]:
node.input[index] = node.input[index] + '/read'
elif node.op == 'AssignSub':
node.op = 'Sub'
if 'use_locking' in node.attr: del node.attr['use_locking']
elif node.op == 'AssignAdd':
node.op = 'Add'
if 'use_locking' in node.attr: del node.attr['use_locking']
if ('dilations') in node.attr: del node.attr['dilations']
node.device = ""
#fixVariables not Working
fixVariables = False
if (fixVariables and node.op == 'VariableV2' and
('batchnorm/var' in node.name or 'batchnorm/mean' in node.name)):
outputNodes = find_output_nodes(graph_def, node)
for index in range(len(outputNodes)):
if (outputNodes[index].op == 'Assign'):
#node.output[index] = node.output[index] + '/read'
#outputNodes[index].op ='Identity'
outputNodes[index].name = outputNodes[index].name + '/read'
print('Modified %s ' % outputNodes[index].name)
#################### Step 1 Training to inference simplification , need checkpoint and .pbtxt files from training ######################################################
graphDef = optimize_for_inference_lib.optimize_for_inference(
graph_def,
[inputNodeName], # an array of the input node(s)
[outputNodeName] if type(outputNodeName) is str else
[item for item in outputNodeName], # an array of output nodes
tf.float32.as_datatype_enum)
if (modifyshapeAttribue):
inputOpType = 'Placeholder'
for n in graphDef.node:
#print('node to modify',n.name)
if (n.name == inputNodeName):
print('node to modify', n)
setNodeAttribute(n, 'shape', shapeArray)
if (n.op != inputOpType):
print(
"--Node %s op %s set to op=%s" %
(inputNodeName, n.op, inputOpType), shapeArray)
n.op = inputOpType
print("--Name of the node - %s shape set to " % n.name,
shapeArray)
print('node after modify', n)
modifyClipValue = False
if (modifyClipValue):
for i in range(1, maxModuleNumber + 1):
blockName = 'fire' + str(i)
newClipValue = 127
clipVNodeName = blockName + '/conv3x3/Rint_1/x'
#clipnode= g_in.get_operation_by_name(clipVNodeName)
clipnode = find_node_by_name(graphDef, clipVNodeName)
print('clipnode to modify', clipnode)
setNodeConstValue(graph_def, clipnode, newClipValue)
print("--Name of the node - %s shape set to %f" %
(clipnode.name, newClipValue))
print('clipnode after modify', clipnode)
modifyFCClip = True
if (modifyFCClip and i == maxModuleNumber):
clipVNodeName = 'conv12/Rint_1/x'
clipnodeFC = find_node_by_name(graphDef, clipVNodeName)
#clipnodeFC= g_in.get_operation_by_name(clipVNodeName)
setNodeConstValue(graph_def, clipnodeFC, newClipValue)
print('clipnode after modify', clipnodeFC)
if (runIncommdLine):
copyfile(file_path, file_path + trainModelSuffix)
outputNameSuffix = '%s_frozenforInference.pb' % checkpointExt
inferenceSuffix = '.Inference'
tf.train.write_graph(graphDef,
folderPath,
checkpointPath + modelName + '.pb' + inferenceSuffix,
as_text=False)
tf.train.write_graph(graphDef,
folderPath,
checkpointPath + modelName + '.pbtxt' +
inferenceSuffix,
as_text=True)
pbfileoutput_path = checkpointPath + modelName + outputNameSuffix
checkpointfile_path = checkpointPath + modelName + checkpointExt
pbfile_path = checkpointPath + modelName + msuffix + inferenceSuffix
#################### Step 2 Frozen Inference mode ######################################################
freeze_graph.freeze_graph(
input_graph=pbfile_path,
input_saver='',
input_binary=binaryPB,
input_checkpoint=checkpointfile_path, # an array of the input node(s)
output_node_names=outputNodeName
if type(outputNodeName) is str else ",".join(outputNodeName),
restore_op_name="save/restore_all", #Unused.
filename_tensor_name="save/Const:0", # Unused.
output_graph=pbfileoutput_path, # an array of output nodes
clear_devices=True,
initializer_nodes='')
#################### Step 3 Save in tensor board ######################################################
saveTensorboardForVisualizatoin = False
if (saveTensorboardForVisualizatoin):
modelFullPath = checkpointPath + modelName + outputNameSuffix
tensorboardPath = checkpointPath + '\\tensorboard'
#if not os.path.exists(tensorboardPath):
# os.mkdir(tensorboardPath)
createTensorboard(modelFullPath, tensorboardPath)