def test_model_with_simple_rnn_multiple_layers(self):
""" Test connected graph construction on a model with multiple simple RNN layers """
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session()
with sess.graph.as_default():
inputs = tf.keras.Input(shape=(3, 100))
# Add an RNN layer with 12 internal units.
x = tf.keras.layers.SimpleRNN(12, name='rnn0', activation='tanh', return_sequences=True)(inputs)
x = tf.keras.layers.SimpleRNN(12, name='rnn1', activation='relu', return_sequences=True)(x)
x = tf.keras.layers.SimpleRNN(12, name='rnn2', activation='tanh')(x)
_ = tf.keras.layers.Dense(12, activation=tf.nn.softmax, name="matmul0")(x)
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
# _ = tf.compat.v1.summary.FileWriter('./simple_rnn', sess.graph)
# construct a connected graph
conn_graph = ConnectedGraph(sess.graph, ['input_1'], ['matmul0/Softmax'])
# there should be only 4 connected graph ops, input, simpleRNN , Dense and Softmax
self.assertEqual(6, len(conn_graph.get_all_ops()))
num_detected_rnns = 0
for op in conn_graph.get_all_ops().values():
if op.type == 'SimpleRNN':
num_detected_rnns += 1
inner_list = op.internal_ops
self.assertEqual(49, len(inner_list))
self.assertEqual(3, num_detected_rnns)
def test_prune_model_tf_slim(self):
""" Punning a model with tf slim api """
# create tf.compat.v1.Session and initialize the weights and biases with zeros
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
# create session with graph
sess = tf.compat.v1.Session(graph=tf.Graph(), config=config)
with sess.graph.as_default():
# by default, model will be constructed in default graph
x = tf.compat.v1.placeholder(tf.float32, [1, 32, 32, 3])
_ = tf_slim_basic_model(x)
sess.run(tf.compat.v1.global_variables_initializer())
conn_graph_orig = ConnectedGraph(sess.graph, ['Placeholder'],
['tf_slim_model/Softmax'])
num_ops_orig = len(conn_graph_orig.get_all_ops())
# Create a layer database
orig_layer_db = LayerDatabase(model=sess,
input_shape=(1, 32, 32, 3),
working_dir=None)
conv1 = orig_layer_db.find_layer_by_name('Conv_1/Conv2D')
conv1_bias = BiasUtils.get_bias_as_numpy_data(orig_layer_db.model,
conv1.module)
layer_comp_ratio_list = [LayerCompRatioPair(conv1, Decimal(0.5))]
spatial_svd_pruner = SpatialSvdPruner()
comp_layer_db = spatial_svd_pruner.prune_model(orig_layer_db,
layer_comp_ratio_list,
CostMetric.mac,
trainer=None)
# Check that svd added these ops
_ = comp_layer_db.model.graph.get_operation_by_name('Conv_1_a/Conv2D')
_ = comp_layer_db.model.graph.get_operation_by_name('Conv_1_b/Conv2D')
conn_graph_new = ConnectedGraph(comp_layer_db.model.graph,
['Placeholder'],
['tf_slim_model/Softmax'])
num_ops_new = len(conn_graph_new.get_all_ops())
self.assertEqual(num_ops_orig + 1, num_ops_new)
bias_add_op = comp_layer_db.model.graph.get_operation_by_name(
'Conv_1_b/BiasAdd')
conv_1_b_op = comp_layer_db.model.graph.get_operation_by_name(
'Conv_1_b/Conv2D')
self.assertEqual(
conn_graph_new._module_identifier.get_op_info(bias_add_op),
conn_graph_new._module_identifier.get_op_info(conv_1_b_op))
self.assertTrue(
np.array_equal(
conv1_bias,
BiasUtils.get_bias_as_numpy_data(comp_layer_db.model,
conv_1_b_op)))
def test_dropout_slim_get_op_product_graph(self):
""" Test connected graph construction on a slim graph with dropout op """
tf.compat.v1.reset_default_graph()
_ = dropout_slim_model()
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['input_1'], ['dropout_slim_model/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(10, len(conn_graph.get_all_ops()))
self.assertEqual(9 + len(tf.compat.v1.get_default_graph().get_collection('variables')),
len(conn_graph.get_all_products()))
self.assertTrue(conn_graph.get_all_ops()['Dropout'], 'Dropout_training_True')
def test_multiple_input_model_get_op_product_graph(self):
""" Test connected graph construction on a multiple input graph """
tf.compat.v1.reset_default_graph()
_ = multiple_input_model()
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['input1', 'input2'], ['multiple_input_model/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(9, len(conn_graph.get_all_ops()))
self.assertEqual(8 + len(tf.compat.v1.get_default_graph().get_collection('variables')),
len(conn_graph.get_all_products()))
self.assertTrue(conn_graph.get_all_ops()['input1'].output.is_model_input)
self.assertTrue(conn_graph.get_all_ops()['input2'].output.is_model_input)
def test_model_with_lstm_layer_sigmoid(self):
""" Test connected graph construction on a model with LSTM op with sigmoid activation """
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session()
with sess.graph.as_default():
inputs = tf.keras.Input(shape=(3, 100))
# Add an RNN layer with 12 internal units.
x = tf.keras.layers.LSTM(12, recurrent_activation='sigmoid', name='lstm0')(inputs)
_ = tf.keras.layers.Dense(12, activation=tf.nn.softmax,
name="matmul0")(x)
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
# _ = tf.compat.v1.summary.FileWriter('./lstm_sigmoid', sess.graph)
# construct a connected graph
conn_graph = ConnectedGraph(sess.graph, ['input_1'], ['matmul0/Softmax'])
# there should be only 4 connected graph ops, input, LSTM , Dense and Softmax
self.assertEqual(4, len(conn_graph.get_all_ops()))
lstm_detected = False
for op in conn_graph.get_all_ops().values():
if op.type == 'LSTM':
lstm_detected = True
inner_list = op.internal_ops
self.assertEqual(77, len(inner_list))
self.assertEqual(op.get_module(), sess.graph.get_operation_by_name('lstm0/while/MatMul'))
self.assertEqual('lstm0', op.name)
self.assertTrue(lstm_detected)
valid_matmuls = []
valid_muls = []
valid_bias_add = []
valid_activation = []
for op in inner_list:
if op.type == 'MatMul' and op not in valid_matmuls:
valid_matmuls.append(op)
if op.type == 'Mul' and op not in valid_matmuls:
valid_muls.append(op)
if op.type == 'BiasAdd' and op not in valid_bias_add:
valid_bias_add.append(op)
if op.type == 'Sigmoid' and op not in valid_activation:
valid_activation.append(op)
self.assertEqual(8, len(valid_matmuls))
self.assertEqual(4, len(valid_muls))
self.assertEqual(4, len(valid_bias_add))
self.assertEqual(3, len(valid_activation))
def test_upsample_get_op_product_graph(self):
""" Test connected graph construction on a graph with upsample op
Need to perform one round of winnowing first to insert the upsample op """
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session()
module_zero_channels_list = []
_ = upsample_model()
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
input_op_names = ['input_1']
output_op_names = ['upsample_model/Softmax']
tf_op = tf.compat.v1.get_default_graph().get_operation_by_name("conv2d_3/Conv2D")
input_channels_to_winnow = [3, 5, 7]
module_mask_pair = (tf_op, input_channels_to_winnow)
module_zero_channels_list.append(module_mask_pair)
new_sess, _ = winnow.winnow_tf_model(sess, input_op_names, output_op_names,
list_of_modules_to_winnow=module_zero_channels_list,
reshape=True, in_place=True, verbose=True)
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), input_op_names, output_op_names)
self.assertEqual(18, len(conn_graph.get_all_ops()))
reduced_bn_1_op = conn_graph.get_op_from_module_name('reduced_batch_normalization_1/cond/FusedBatchNormV3_1')
self.assertTrue(reduced_bn_1_op.output.consumers[0].type == 'Upsample')
new_sess.close()
def test_simple_rnn_keras_single_timestep_with_placeholder_input(self):
""" simple RNN layer with placeholder input type """
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session()
with sess.graph.as_default():
X_batch = np.random.random([32, 1, 8]).astype(np.float32)
X = tf.compat.v1.placeholder(tf.float32, [32, 1, 8], name='input')
simple_rnn = tf.keras.layers.SimpleRNN(1)
output = simple_rnn(X)
_ = tf.keras.layers.Dense(12, activation=tf.nn.softmax,
name="matmul0")(output)
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
# _ = tf.compat.v1.summary.FileWriter('./test_simple_rnn_keras_single_timestep_with_placeholder_input', sess.graph)
sess.run(output, feed_dict={X: X_batch})
# construct a connected graph
conn_graph = ConnectedGraph(sess.graph, ['input'], ['matmul0/Softmax'])
# there should be only 4 connected graph ops, input, simpleRNN , Dense and Softmax
self.assertEqual(4, len(conn_graph.get_all_ops()))
simple_rnn_detected = False
for op in conn_graph.get_all_ops().values():
if op.type == 'SimpleRNN':
simple_rnn_detected = True
inner_list = op.internal_ops
self.assertEqual(25, len(inner_list))
self.assertEqual(op.get_module(), sess.graph.get_operation_by_name('simple_rnn/while/MatMul'))
self.assertEqual('simple_rnn', op.name)
self.assertTrue(simple_rnn_detected)
valid_matmuls = []
valid_bias_add = []
valid_activation = []
for op in inner_list:
if op.type == 'MatMul' and op not in valid_matmuls:
valid_matmuls.append(op)
if op.type == 'BiasAdd' and op not in valid_bias_add:
valid_bias_add.append(op)
if op.type == 'Tanh' and op not in valid_activation:
valid_activation.append(op)
self.assertEqual(2, len(valid_matmuls))
self.assertEqual(1, len(valid_bias_add))
self.assertEqual(1, len(valid_activation))
def test_model_with_simple_rnn_layer(self):
""" Test connected graph construction on a model with simple RNN op """
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session()
with sess.graph.as_default():
inputs = tf.keras.Input(shape=(3, 100))
# Add an RNN layer with 12 internal units.
x = tf.keras.layers.SimpleRNN(12, name='rnn0')(inputs)
_ = tf.keras.layers.Dense(12, activation=tf.nn.softmax,
name="matmul0")(x)
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
# _ = tf.compat.v1.summary.FileWriter('./simple_rnn', sess.graph)
# construct a connected graph
conn_graph = ConnectedGraph(sess.graph, ['input_1'], ['matmul0/Softmax'])
# there should be only 4 connected graph ops, input, simpleRNN , Dense and Softmax
self.assertEqual(4, len(conn_graph.get_all_ops()))
simple_rnn_detected = False
for op in conn_graph.get_all_ops().values():
if op.type == 'SimpleRNN':
simple_rnn_detected = True
inner_list = op.internal_ops
self.assertEqual(49, len(inner_list))
self.assertEqual(op.get_module(), sess.graph.get_operation_by_name('rnn0/while/MatMul'))
self.assertEqual('rnn0', op.name)
self.assertTrue(simple_rnn_detected)
# check for 2 MatMuls, 1 BiasAdd and an activation function in the inner op list
valid_matmuls = []
valid_bias_add = []
valid_activation = []
for op in inner_list:
if op.type == 'MatMul' and op not in valid_matmuls:
valid_matmuls.append(op)
if op.type == 'BiasAdd' and op not in valid_bias_add:
valid_bias_add.append(op)
if op.type == 'Tanh' and op not in valid_activation:
valid_activation.append(op)
self.assertEqual(2, len(valid_matmuls))
self.assertEqual(1, len(valid_bias_add))
self.assertEqual(1, len(valid_activation))
def test_bn_fold_with_no_bias(self):
tf.compat.v1.reset_default_graph()
inputs = tf.keras.Input(shape=(
32,
32,
3,
))
conv_op = tf.keras.layers.Conv2D(32, (3, 3), use_bias=False)(inputs)
bn_op = tf.keras.layers.BatchNormalization(fused=True)(conv_op,
training=False)
_ = tf.nn.relu(bn_op)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session()
sess.run(init)
conv_op = sess.graph.get_operation_by_name('conv2d/Conv2D')
np.random.seed(0)
w_shape = conv_op.inputs[0].shape
numpy_data = np.random.rand(1, w_shape[1], w_shape[2], w_shape[3])
relu_op = sess.graph.get_operation_by_name('Relu')
baseline_output = sess.run(relu_op.outputs[0],
feed_dict={conv_op.inputs[0]: numpy_data})
old_conn_graph = ConnectedGraph(sess.graph,
starting_op_names=['input_1'],
output_op_names=['Relu'])
new_sess, pairs = fold_all_batch_norms(sess, "input_1", 'Relu')
new_conv_op = new_sess.graph.get_operation_by_name('conv2d/Conv2D')
w2 = new_conv_op.inputs[0]
feed_dict = {w2: numpy_data}
new_relu_op = new_sess.graph.get_operation_by_name('Relu')
output_after_fold = new_sess.run(new_relu_op.outputs[0],
feed_dict=feed_dict)
new_conn_graph = ConnectedGraph(new_sess.graph,
starting_op_names=['input_1'],
output_op_names=['Relu'])
self.assertTrue(
np.allclose(baseline_output, output_after_fold, atol=1.e-4))
# New connected graph should have one less op since bn was removed
self.assertTrue(len(old_conn_graph.get_all_ops()),
len(new_conn_graph.get_all_ops()) - 1)
def test_vgg16_slim_get_op_product_graph(self):
"""
Test connected graph construction on vgg16 from tf slim
This model includes dropout pattern 3 which does not appear in other models.
"""
tf.compat.v1.reset_default_graph()
inp = tf.compat.v1.placeholder(tf.float32, [1, 224, 224, 3])
_ = vgg.vgg_16(inp)
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['Placeholder'], ['vgg_16/fc8/squeezed'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(40, len(conn_graph.get_all_ops()))
self.assertEqual(39 + len(tf.compat.v1.get_default_graph().get_collection('variables')),
len(conn_graph.get_all_products()))
self.assertTrue(conn_graph.get_all_ops()['vgg_16/dropout6'], 'Dropout_training_True_unknown_shape')
self.assertTrue(conn_graph.get_all_ops()['vgg_16/dropout7'], 'Dropout_training_True_unknown_shape')
def test_keras_model_functional_with_non_fused_batchnorms_get_op_product_graph(self):
""" Test connected graph construction on keras model functional with non fused batchnorms """
tf.compat.v1.reset_default_graph()
_ = keras_model_functional_with_non_fused_batchnorms()
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['input_1'],
['keras_model_functional_with_non_fused_batchnorms/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
_ = conn_graph.get_all_ops()['batch_normalization']
_ = conn_graph.get_all_ops()['scope_1/batch_normalization_1']
_ = conn_graph.get_all_ops()['scope_1/batch_normalization_2']
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(14, len(conn_graph.get_all_ops()))
# 13 products from inter module connections
# 22 products from parameters
self.assertEqual(35, len(conn_graph.get_all_products()))
def test_model_with_PReLU(self):
"""
PreLU
"""
tf.compat.v1.reset_default_graph()
inputs = tf.keras.Input(shape=(1, 10, 10), name="inputs")
x = tf.keras.layers.PReLU()(inputs)
x = tf.keras.layers.ReLU()(x)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session()
sess.run(init)
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), starting_op_names=['inputs'],
output_op_names=['re_lu/Relu'])
self.assertEqual(3, len(conn_graph.get_all_ops()))
self.assertEqual(5, len(conn_graph.get_all_ops()['p_re_lu/Relu'].internal_ops))
def test_concat_get_op_product_graph(self):
""" Test connected graph construction on a graph with concat op """
tf.compat.v1.reset_default_graph()
_ = concat_model()
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['input_1'], ['concat_model/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(2, conn_graph.branch_count)
self.assertEqual(13, len(conn_graph.get_all_ops()))
self.assertEqual(12 + len(tf.compat.v1.get_default_graph().get_collection('variables')),
len(conn_graph.get_all_products()))
# Check that the order of input products to the concat op matches the order of input tensors in the tf graph
concat_tf_op = tf.compat.v1.get_default_graph().get_operation_by_name("concatenate/concat")
concat_op = conn_graph.get_all_ops()['concatenate/concat']
for index, product in enumerate(concat_op.get_input_products()):
self.assertTrue(len(product.consumers) == 1)
self.assertEqual(product.tensor_dict[product.consumers[0]], concat_tf_op.inputs[index])
def test_model_with_global_max_pool2d(self):
""" Test connected graph construction on model with leaky relu op """
tf.compat.v1.reset_default_graph()
_ = model_with_global_max_pool2d()
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), starting_op_names=['input_1'],
output_op_names=['model_with_global_max_pool2d/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(6, len(conn_graph.get_all_ops()))
# 5 products from inter module connections
# 4 products from parameters
self.assertEqual(9, len(conn_graph.get_all_products()))
found_global_max_pool2d = False
for op in conn_graph.get_all_ops().values():
if op.type == 'GlobalMaxPool2D':
found_global_max_pool2d = True
self.assertTrue(found_global_max_pool2d)
tf.compat.v1.reset_default_graph()
def test_single_residual_get_op_product_graph(self):
""" Test connected graph construction on single residual model """
tf.compat.v1.reset_default_graph()
_ = single_residual()
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['input_1'], ['single_residual/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(1, conn_graph.branch_count)
self.assertEqual(18, len(conn_graph.get_all_ops()))
# 17 products from interop connections, 20 from parameters
self.assertEqual(37, len(conn_graph.get_all_products()))
def test_model_with_leaky_relu(self):
""" Test connected graph construction on model with leaky relu op """
tf.compat.v1.reset_default_graph()
_ = model_with_leaky_relu()
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), starting_op_names=['input_1'],
output_op_names=['model_with_leaky_relu/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(7, len(conn_graph.get_all_ops()))
# 6 products from inter module connections
# 6 products from parameters
self.assertEqual(12, len(conn_graph.get_all_products()))
found_leaky_relu = False
for op in conn_graph.get_all_ops().values():
if op.type == 'LeakyRelu':
found_leaky_relu = True
self.assertTrue(found_leaky_relu)
tf.compat.v1.reset_default_graph()
def test_model_with_lstm_layer_deepspeech_time_major_true_sigmoid(self):
""" Test connected graph construction on a model with stacked LSTM op in DeepSpeech model"""
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session()
with sess.graph.as_default():
inputs = tf.keras.Input(shape=(3, 100))
# Defaults
# return_state=False, unit_forget_bias=True
# return_sequences=False, time_major=False
x = tf.keras.layers.LSTM(12,
recurrent_activation='sigmoid',
unit_forget_bias=False,
time_major=True,
return_sequences=True,
name='lstm_stacked')(inputs)
x2 = tf.keras.layers.LSTM(12, name='last_lstm')(x)
_ = tf.keras.layers.Dense(12, activation=tf.nn.softmax,
name="matmul0")(x2)
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
# _ = tf.compat.v1.summary.FileWriter('./lstm_deepspeech', sess.graph)
# construct a connected graph
conn_graph = ConnectedGraph(sess.graph, ['input_1'], ['matmul0/Softmax'])
self.assertEqual(5, len(conn_graph.get_all_ops()))
lstm_detected = False
for op in conn_graph.get_all_ops().values():
if op.type == 'LSTM' and op.name == 'lstm_stacked':
self.assertEqual(op.pattern_type, 'LSTM_Stacked_TimeMajor_True_Sigmoid')
lstm_detected = True
inner_list = op.internal_ops
self.assertEqual(75, len(inner_list))
self.assertEqual(op.get_module(), sess.graph.get_operation_by_name('lstm_stacked/while/MatMul'))
self.assertTrue(lstm_detected)
def test_split_get_op_product_graph(self):
""" Test connected graph construction on a graph with split op """
tf.compat.v1.reset_default_graph()
_ = split_and_concat_model()
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['input_1'], ['split_and_concat_model/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(1, conn_graph.branch_count)
self.assertEqual(9, len(conn_graph.get_all_ops()))
self.assertEqual(8 + len(tf.compat.v1.get_default_graph().get_collection('variables')),
len(conn_graph.get_all_products()))
def fold_given_batch_norms(
sess: tf.compat.v1.Session, input_op_names: Union[str, List[str]],
output_op_names: Union[str, List[str]],
layer_pairs: List[Tuple[tf.Operation, tf.Operation, bool]]
) -> tf.compat.v1.Session:
"""
Api to fold custom set of bn layers in a model
:param sess: active tensorflow session
:param input_op_names: starting op in model or a list of starting ops in the model
:param layer_pairs: List of tuple with conv and bn op layers as tf.Operation and
a flag to indicate fold upstream or downstream
:param output_op_names: List of output op names of the model, used to help ConnectedGraph determine valid ops
(to ignore training ops for example).
:return: updated_session after fold
"""
# check for valid types
if not isinstance(input_op_names, (str, List)):
logger.error(
'start op names must be passed as a string or a List of strings')
# if passed start op name is a single string, create a list
if isinstance(input_op_names, str):
input_op_names = [input_op_names]
connected_graph = ConnectedGraph(sess.graph, input_op_names,
output_op_names)
conn_tf_n_op_map = {}
for op in connected_graph.get_all_ops().values():
if op.type in ['FusedBatchNormV3', 'FusedBatchNorm']:
conn_tf_n_op_map[op.get_module()] = op
layer_pairs_internal_format = []
for layer_pair in layer_pairs:
conv_op, bn_op, is_bn_op_second = layer_pair
layer_pairs_internal_format.append(
(conv_op, conn_tf_n_op_map[bn_op].get_tf_op_with_io_tensor(),
is_bn_op_second))
# invoke internal api
new_sess = _fold_given_auto_selected_batch_norms(
sess, layer_pairs_internal_format)
# save and load graph
after_fold_sess = save_and_load_graph('./temp_graph', new_sess)
return after_fold_sess
def test_keras_model_functional_get_op_product_graph(self):
""" Test connected graph construction on keras model functional """
tf.compat.v1.reset_default_graph()
_ = keras_model_functional()
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['input_1'], ['keras_model_functional/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(14, len(conn_graph.get_all_ops()))
# 13 products from inter module connections
# 22 products from parameters
self.assertEqual(35, len(conn_graph.get_all_products()))
def test_connected_graph_with_detached_ops(self):
""" Test connected graph construction on a graph with detached ops """
tf.compat.v1.reset_default_graph()
_ = single_residual()
# Detach everything starting from conv2d_4/Conv2D and below
detach_inputs(tf.compat.v1.get_default_graph().get_operation_by_name('conv2d_4/Conv2D'))
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['input_1'], ['Relu_2'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(1, conn_graph.branch_count)
self.assertEqual(13, len(conn_graph.get_all_ops()))
# 12 products from interop connections, 16 from parameters
self.assertEqual(28, len(conn_graph.get_all_products()))
def test_tf_slim_with_softmax_model_get_op_product_graph(self):
""" Test connected graph construction on tf_slim with softmax model """
tf.compat.v1.reset_default_graph()
x = tf.compat.v1.placeholder(tf.float32, [1, 32, 32, 3])
_ = tf_slim_with_softmax(x)
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['Placeholder'], ['softmax/Reshape_1'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(8, len(conn_graph.get_all_ops()))
# 7 products from interop connections
# need to add 2 since gamma is treated as a parameter, even though it is a constant in this graph
self.assertEqual(7 + len(tf.compat.v1.get_default_graph().get_collection('variables')) + 2,
len(conn_graph.get_all_products()))
def test_tf_slim_model_get_op_product_graph(self):
""" Test connected graph construction on tf_slim model """
tf.compat.v1.reset_default_graph()
x = tf.compat.v1.placeholder(tf.float32, [1, 32, 32, 3])
_ = tf_slim_basic_model(x)
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ['Placeholder'], ['tf_slim_model/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(15, len(conn_graph.get_all_ops()))
# 14 products from interop connections
# need to add 1 since gamma is treated as a parameter for the training = True bn, even though it is a constant
# in the graph
self.assertEqual(14 + len(tf.compat.v1.get_default_graph().get_collection('variables')) + 1,
len(conn_graph.get_all_products()))
def test_get_ordered_operations(self):
""" Test the creation of the ordered operations list """
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session()
with sess.graph.as_default():
_ = single_residual()
conn_graph = ConnectedGraph(sess.graph, ["input_1"], ['Relu_2'])
ordered_ops = get_ordered_ops(conn_graph.starting_ops)
# check that there are the same number of modules in the ordered ops list as there are in the main ops dict
self.assertEqual(len(ordered_ops), len(conn_graph.get_all_ops()))
# check that for any module in the ordered ops list, all of its parent modules are earlier in the list
seen_ops = set()
for op in ordered_ops:
input_products = op.get_input_products()
for product in input_products:
self.assertTrue(product.producer in seen_ops)
seen_ops.add(op)
def test_keras_model_functional_with_training_ops_get_op_product_graph(self):
""" Test connected graph construction on keras model functional with training ops attached """
tf.compat.v1.reset_default_graph()
_ = keras_model_functional()
# add training ops
optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=1e-3, name='Adam_new')
_ = optimizer.minimize(loss=tf.compat.v1.get_default_graph().get_tensor_by_name('keras_model_functional/Softmax:0'),
name='train_step_new')
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), ["input_1"],
output_op_names=['keras_model_functional/Softmax'])
self.assertTrue(validate_branch_ops(conn_graph))
self.assertTrue(validate_product_tensor_lists(conn_graph))
self.assertEqual(0, conn_graph.branch_count)
self.assertEqual(14, len(conn_graph.get_all_ops()))
# 13 products from inter module connections
# 22 products from parameters
self.assertEqual(35, len(conn_graph.get_all_products()))
def test_model_zoo_videnn_pose_estimation_model_with_input_split(self):
"""
create a smaller network with connections as in pose estimation model and ViDeNN model
Testwhen input is split and fed into two different ops
:return:
"""
tf.compat.v1.reset_default_graph()
inputs = tf.keras.Input(shape=(None, None, 2), name="inputs")
x = tf.keras.layers.Conv2D(2, kernel_size=3, padding='same')(inputs)
x = tf.keras.layers.BatchNormalization()(x)
x = tf.nn.relu(x)
x = tf.keras.layers.Conv2D(2, kernel_size=3, padding='same')(x)
x = tf.keras.layers.BatchNormalization()(x)
z = tf.keras.layers.Add()([inputs, x])
x = tf.nn.relu(z)
init = tf.compat.v1.global_variables_initializer()
sess = tf.compat.v1.Session()
sess.run(init)
conn_graph = ConnectedGraph(tf.compat.v1.get_default_graph(), starting_op_names=['inputs'],
output_op_names=['Relu_1'])
# get input ops
# Find the input node(s) in the graph
input_nodes = []
for op in conn_graph.get_all_ops().values():
if op.inputs and op.inputs[0].is_model_input:
input_nodes.append(op)
# there should be two ops marked as inputs in this model
self.assertEqual(len(input_nodes), 2)
self.assertEqual(input_nodes[0].name, 'add/add')
self.assertEqual(input_nodes[1].name, "conv2d/Conv2D")
class GraphSearchUtils:
""" Implements graph search utils required by CLE feature"""
def __init__(self, model: tf.Graph, start_op_names: Union[str, List[str]], output_op_names: Union[str, List[str]]):
if isinstance(start_op_names, str):
start_op_names = [start_op_names]
if isinstance(output_op_names, str):
output_op_names = [output_op_names]
self._connected_graph = ConnectedGraph(model, start_op_names, output_op_names)
def find_and_replace_relu6_with_relu(self, sess: tf.compat.v1.Session) -> tf.compat.v1.Session:
"""
finds and replaces Relu6 ops with Relu
:return: updated session
"""
for op in self._connected_graph.get_all_ops().values():
if op.type in ['Relu6']:
# send the session here, so we make the update on sess.graph (active graph)
ReluUtils.replace_relu6_with_relu(sess, op.get_module())
# in the end update the session
after_relu_replace_sess = save_and_load_graph('./replace_relu6_with_relu', sess)
return after_relu_replace_sess
@staticmethod
def find_downstream_layer_groups_to_scale(op, layer_groups, visited_nodes, current_group=None):
"""
Populates all the layer groups eligible for cross layer scaling
:param op: starting op
:param layer_groups: layer_groups as empty list
:param visited_nodes: all the ops that have been visited
:param current_group: op groups
:return: None. Updates layer_groups[] if groups are found.
"""
if not current_group:
current_group = []
if op in visited_nodes:
return
visited_nodes.append(op)
logger.debug("Visiting node: {%s}", op.dotted_name)
# If current node is Conv2D, add to the current group
if op.type in ['Conv2D', 'DepthwiseConv2dNative']:
current_group.append(op)
# Terminating condition for current group
if not (op.type in ['Conv2D', 'DepthwiseConv2dNative', 'Relu', 'PReLU', 'Pad', 'Identity']):
if (len(current_group) > 1) and (current_group not in layer_groups):
layer_groups.append(current_group)
node_set = [op.dotted_name for op in current_group]
logger.debug("Added new set of nodes: {%s}", node_set)
current_group = []
if op.output:
for consumer in op.output.consumers:
GraphSearchUtils.find_downstream_layer_groups_to_scale(consumer, layer_groups, visited_nodes,
current_group)
# Reached a leaf.. See if the current group has something to grab
if (len(current_group) > 1) and (current_group not in layer_groups):
layer_groups.append(current_group)
node_set = [op.dotted_name for op in current_group]
logger.debug("Added new set of nodes: {%s}", node_set)
def find_layer_groups_to_scale_as_conn_ops(self) -> List[List[Op]]:
"""
:return: List of groups of layers. Each group can be independently equalized
"""
# Find the input node(s) in the graph
input_nodes = []
for op in self._connected_graph.get_all_ops().values():
if op.inputs and op.inputs[0].is_model_input:
input_nodes.append(op)
layer_groups = []
visited_nodes = []
for op in input_nodes:
self.find_downstream_layer_groups_to_scale(op=op, layer_groups=layer_groups,
visited_nodes=visited_nodes)
return layer_groups
def find_layer_groups_to_scale(self):
"""
Find layer groups for scaling as tf ops
:return: groups for scaling as tf ops
"""
layer_groups_as_conn_graph_ops = self.find_layer_groups_to_scale_as_conn_ops()
layer_groups_as_tf_ops, tf_op_to_conn_graph_op_map = self.convert_conn_graph_ops_to_tf_op(layer_groups_as_conn_graph_ops)
return tf_op_to_conn_graph_op_map, layer_groups_as_tf_ops
@staticmethod
def convert_conn_graph_ops_to_tf_op(op_groups: List[List[Op]]) -> \
List[List[tf.Operation]]:
"""
Helper function to get op list as tf.Operation type to be usable for updating/scaling weights and biases
using generic apis for tensor updates.
:param op_groups: list of op groups as TfOperation type of used by Connected Graph
:return: lis of op groups as tf.Operation (standard TF op type)
"""
tf_op_to_conn_graph_op_map = {}
layer_groups_as_tf_ops = []
for ops in op_groups:
curr_group = []
for op in ops:
tf_op_to_conn_graph_op_map[op.get_module()] = op
curr_group.append(op.get_module())
layer_groups_as_tf_ops.append(curr_group)
return layer_groups_as_tf_ops, tf_op_to_conn_graph_op_map
@staticmethod
def convert_layer_group_to_cls_sets(layer_group):
"""
Helper function to convert a layer group to a list of cls sets
:param layer_group: Given layer group to convert
:return: List of cls sets
"""
cls_sets = []
prev_layer_to_scale = layer_group.pop(0)
while layer_group:
next_layer_to_scale = layer_group.pop(0)
if next_layer_to_scale.type in ['DepthwiseConv2dNative']:
next_non_depthwise_conv_layer = layer_group.pop(0)
cls_sets.append((prev_layer_to_scale, next_layer_to_scale, next_non_depthwise_conv_layer))
prev_layer_to_scale = next_non_depthwise_conv_layer
else:
cls_sets.append((prev_layer_to_scale, next_layer_to_scale))
prev_layer_to_scale = next_layer_to_scale
return cls_sets
@staticmethod
def is_relu_activation_present_in_cls_sets(cls_sets: List[ClsSet],
tf_op_to_conn_graph_op_map: Dict) -> List[bool]:
"""
check if there is Relu activations between cls sets
:param cls_sets: cls conv op pairs
:param tf_op_to_conn_graph_op_map: Map of tf-op => connected graph op
:return: list of relu activation preset flags(True or False)
corresponding to input cls_sets list
"""
is_relu_activation_in_cls_sets = []
for cls_set in cls_sets:
# We need to check activation functions for all layers but the last one in the set
# Because we are only interested in checking activation functions between the layers we will scale
cls_set = cls_set[:-1]
is_relu_activation_in_cls_set = ()
for conv_op in cls_set:
conn_graph_conv_op = tf_op_to_conn_graph_op_map[conv_op]
is_relu_activation_in_cls_set += (ReluUtils.does_conv_have_relu_activation(conn_graph_conv_op), )
if len(is_relu_activation_in_cls_set) == 1:
is_relu_activation_in_cls_set = is_relu_activation_in_cls_set[0]
is_relu_activation_in_cls_sets.append(is_relu_activation_in_cls_set)
return is_relu_activation_in_cls_sets
@staticmethod
def map_op_names_to_ops(sess: tf.compat.v1.Session) -> Dict[str, tf.Operation]:
"""
After the fold and cls , the graph is updated, so are the ops
So, we need a way to map ops we stored on graph we began with, to perform
high bias fold operation on latest ops in the updated graph.
:param sess: active tf.compat.v1.Session (tf.compat.v1.Session type)
:return: a dictionary of op names mapped to ops in the given new session.
Note : only stores infor pertaining to bn and conv ops required by high bias fold.
"""
tf_names_op_dict = {}
with sess.graph.as_default():
op_list = sess.graph.get_operations()
for op in op_list:
if op.type in ['Conv2D', 'DepthwiseConv2dNative', 'FusedBatchNormV3']:
tf_names_op_dict[op.name] = op
return tf_names_op_dict