def backprop_pool(self, activation, relevance, ksize, strides, pooling_type, padding='SAME'):
if pooling_type.lower() in 'avg':
z = nn_ops.avg_pool(activation, ksize, strides, padding) + 1e-10
s = relevance / z
c = gen_nn_ops._avg_pool_grad(tf.shape(activation), s, ksize, strides, padding)
return activation * c
else:
z = nn_ops.max_pool(activation, ksize, strides, padding) + 1e-10
s = relevance / z
c = gen_nn_ops._max_pool_grad(activation, z, s, ksize, strides, padding)
return activation * c
def testDirectUseOverlapping(self):
for num_batches in [1, 3]:
for row_window_size in [2, 5]:
for col_window_size in [2, 4]:
num_rows = (row_window_size - 1) * 5 + 1
num_cols = (col_window_size - 1) * 7 + 1
for num_channels in [1, 2]:
input_shape = (num_batches, num_rows, num_cols,
num_channels)
with self.cached_session() as _:
input_tensor = constant_op.constant(
self._GenerateRandomInputTensor(
input_shape).astype(np.float32))
window_size = [
1, row_window_size, col_window_size, 1
]
stride_size = [
1, row_window_size - 1, col_window_size - 1, 1
]
padding = "VALID"
output_tensor = nn_ops.avg_pool(
input_tensor, window_size, stride_size,
padding)
output_data = self.evaluate(output_tensor)
num_elements = 1
for dim_size in output_data.shape:
num_elements *= dim_size
output_backprop = (self._PRNG.rand(num_elements) *
1000).reshape(output_data.shape)
input_backprop_tensor = gen_nn_ops.avg_pool_grad(
input_tensor.get_shape(), output_backprop,
window_size, stride_size, padding)
input_backprop = self.evaluate(
input_backprop_tensor)
row_seq = list(
range(0, num_rows, row_window_size - 1))
col_seq = list(
range(0, num_cols, col_window_size - 1))
row_seq[-1] += 1
col_seq[-1] += 1
fap_input_backprop_tensor = gen_nn_ops.fractional_avg_pool_grad(
input_tensor.get_shape(),
output_backprop,
row_seq,
col_seq,
overlapping=True)
fap_input_backprop = self.evaluate(
fap_input_backprop_tensor)
self.assertShapeEqual(input_backprop,
fap_input_backprop_tensor)
self.assertAllClose(input_backprop,
fap_input_backprop)
def _RedoRestAvgPool(graph):
"""Finds fused batch norm layers and folds them into preceding layers.
Folding only affects the following layers: Conv2D, fully connected, depthwise
convolution.
Args:
graph: Graph to walk and modify.
is_training: Bool, true if training.
Raises:
ValueError: When batch norm folding fails.
"""
matches = _FindRestAvgPool(graph)
print("Replacing", len(matches), "AvgPool")
for match in matches:
scope, sep, _ = match['layer_op'].name.rpartition('/')
# Make sure new ops are added to `graph` and put on the same device as
# `bn_op`. The '/' (i.e. `sep`) ensures that we reuse the existing scope
# named `scope`. Otherwise, TF creates a unique scope whose name starts with
# `scope`.
with graph.as_default(), graph.name_scope(scope + sep):
# with graph.name_scope(scope + sep + '_psb' + sep):
input_tensor = match['input_tensor']
layer_op = match['layer_op']
# output_tensor = match['output_tensor']
# >>>>> CUSTOM >>>>>>>>>>>>>>
avg_size = np.prod(layer_op.get_attr("ksize")).astype(np.float32)
if avg_size == 2**np.log2(avg_size):
continue
output_tensor = nn_ops.avg_pool(
input_tensor,
ksize=layer_op.get_attr('ksize'),
strides=layer_op.get_attr('strides'),
padding=layer_op.get_attr('padding'),
data_format=layer_op.get_attr('data_format'),
name=layer_op.name.split('/')[-1] + '_psb')
avg_size_new = variableFromSettings(
[], hiddenVar=(1.0 / avg_size).astype(np.float32))[0]
new_layer_tensor = output_tensor * avg_size * avg_size_new
# <<<<<<<<<<<<<<<<<<<<<<<<<<<
nodes_modified_count = common.RerouteTensor(
new_layer_tensor, match['output_tensor'])
if nodes_modified_count == 0:
raise ValueError(
'Folding batch norms failed, %s had no outputs.' %
match['output_tensor'].name)
def backprop_pool(activation,
relevance,
ksize,
strides,
pooling_type,
padding='VALID'):
if pooling_type.lower() is 'avg': # avg pooling
z = nn_ops.avg_pool(activation, ksize, strides, padding) + 1e-10
s = relevance / z
c = gen_nn_ops._avg_pool_grad(tf.shape(activation), s, ksize, strides,
padding)
return activation * c
else: # max pooling
z = nn_ops.max_pool(activation, ksize, strides, padding) + 1e-10
s = relevance / z
c = gen_nn_ops.max_pool_grad(activation, z, s, ksize, strides, padding)
return activation * c
def testDirectUseOverlapping(self):
for num_batches in [1, 3]:
for row_window_size in [2, 5]:
for col_window_size in [2, 4]:
num_rows = (row_window_size - 1) * 5 + 1
num_cols = (col_window_size - 1) * 7 + 1
for num_channels in [1, 2]:
input_shape = (num_batches, num_rows, num_cols, num_channels)
with self.cached_session() as _:
input_tensor = constant_op.constant(
self._GenerateRandomInputTensor(input_shape).astype(
np.float32))
window_size = [1, row_window_size, col_window_size, 1]
stride_size = [1, row_window_size - 1, col_window_size - 1, 1]
padding = "VALID"
output_tensor = nn_ops.avg_pool(input_tensor, window_size,
stride_size, padding)
output_data = self.evaluate(output_tensor)
num_elements = 1
for dim_size in output_data.shape:
num_elements *= dim_size
output_backprop = (self._PRNG.rand(num_elements) *
1000).reshape(output_data.shape)
input_backprop_tensor = gen_nn_ops.avg_pool_grad(
input_tensor.get_shape(), output_backprop, window_size,
stride_size, padding)
input_backprop = self.evaluate(input_backprop_tensor)
row_seq = list(range(0, num_rows, row_window_size - 1))
col_seq = list(range(0, num_cols, col_window_size - 1))
row_seq[-1] += 1
col_seq[-1] += 1
fap_input_backprop_tensor = gen_nn_ops.fractional_avg_pool_grad(
input_tensor.get_shape(),
output_backprop,
row_seq,
col_seq,
overlapping=True)
fap_input_backprop = self.evaluate(fap_input_backprop_tensor)
self.assertShapeEqual(input_backprop, fap_input_backprop_tensor)
self.assertAllClose(input_backprop, fap_input_backprop)
def atrous_avg_pool(value, size, rate, padding, name=None, info=DummyDict()):
with tfops.op_scope([value], name, "atrous_avg_pool") as name:
value = tfops.convert_to_tensor(value, name="value")
if rate < 1:
raise ValueError("rate {} cannot be less than one".format(rate))
if rate == 1:
value = nn_ops.avg_pool(value=value,
strides=[1, 1, 1, 1],
ksize=[1, size, size, 1],
padding=padding)
return value
# We have two padding contributions. The first is used for converting "SAME"
# to "VALID". The second is required so that the height and width of the
# zero-padded value tensor are multiples of rate.
# Padding required to reduce to "VALID" convolution
if padding == "SAME":
filter_height, filter_width = size, size
# Spatial dimensions of the filters and the upsampled filters in which we
# introduce (rate - 1) zeros between consecutive filter values.
filter_height_up = filter_height + (filter_height - 1) * (rate - 1)
filter_width_up = filter_width + (filter_width - 1) * (rate - 1)
pad_height = filter_height_up - 1
pad_width = filter_width_up - 1
# When pad_height (pad_width) is odd, we pad more to bottom (right),
# following the same convention as avg_pool().
pad_top = pad_height // 2
pad_bottom = pad_height - pad_top
pad_left = pad_width // 2
pad_right = pad_width - pad_left
elif padding == "VALID":
pad_top = 0
pad_bottom = 0
pad_left = 0
pad_right = 0
else:
raise ValueError("Invalid padding")
# Handle input whose shape is unknown during graph creation.
if value.get_shape().is_fully_defined():
value_shape = value.get_shape().as_list()
else:
value_shape = array_ops.shape(value)
in_height = value_shape[1] + pad_top + pad_bottom
in_width = value_shape[2] + pad_left + pad_right
# More padding so that rate divides the height and width of the input.
pad_bottom_extra = (rate - in_height % rate) % rate
pad_right_extra = (rate - in_width % rate) % rate
# The paddings argument to space_to_batch includes both padding components.
space_to_batch_pad = [[pad_top, pad_bottom + pad_bottom_extra],
[pad_left, pad_right + pad_right_extra]]
value = array_ops.space_to_batch(input=value,
paddings=space_to_batch_pad,
block_size=rate)
value = nn_ops.avg_pool(value=value,
ksize=[1, size, size, 1],
strides=[1, 1, 1, 1],
padding="VALID",
name=name)
# The crops argument to batch_to_space is just the extra padding component.
batch_to_space_crop = [[0, pad_bottom_extra], [0, pad_right_extra]]
value = array_ops.batch_to_space(input=value,
crops=batch_to_space_crop,
block_size=rate)
info['activations'][name] = value
return value
def test_avgpool2d():
''' Run tests on the Wave custom avgpool2d operator.
'''
tf.reset_default_graph()
# Turn off graph-rewriting optimizations
config = tf.ConfigProto(graph_options=tf.GraphOptions(optimizer_options=tf.OptimizerOptions(opt_level=tf.OptimizerOptions.L0)))
iterations = 100
for i in range(iterations):
tf.reset_default_graph()
# NCHW
t_n = 1
t_h = 64
t_w = 64
t_c = 2
# window
w_n = 1
w_h = 2
w_w = 2
w_c = 1
#strides
s_n = 1
s_h = 2
s_w = 2
s_c = 1
# N H W C
max_in = tf.get_variable("a", [t_n, t_h, t_w, t_c], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.1))
t_init = tf.global_variables_initializer()
# SAME variant
with tf.Session('', config=config) as sess:
t_init.run()
# print("Wave Kernel:\n-------------------------------------------------")
z_op = waveflow.wavecomp_ops_module.wave_avg_pool_dfx(
max_in,
ksize=[w_n, w_h, w_w, w_c],
strides=[s_n, s_h, s_w, s_c],
padding='SAME',
data_format='NHWC')
# Base tensorflow. Only supports NHWC.
z2_op = nn_ops.avg_pool(
max_in,
ksize=[w_n, w_h, w_w, w_c],
strides=[s_n, s_h, s_w, s_c],
padding='SAME',
data_format='NHWC')
# z = z_op.eval()
# z2 = z2_op.eval()
z, z2 = sess.run([z_op, z2_op])
# print("\nTF:\n-------------------------------------------------")
assert_str = "Failure on i: %d, mode: SAME" % (i)
if not compare_tensor(z, z2, assert_str):
print("z: shape: %s, %s" % (z.shape, z))
print("z (np): shape: %s, %s" % (z2.shape, z2))
print("\n\n")
assert False
# Valid variant
with tf.Session('', config=config) as sess:
t_init.run()
# print("Wave Kernel:\n-------------------------------------------------")
z_op = waveflow.wavecomp_ops_module.wave_avg_pool_dfx(
max_in,
ksize=[w_n, w_h, w_w, w_c],
strides=[s_n, s_h, s_w, s_c],
padding='VALID',
data_format='NHWC')
# Base tensorflow. Only supports NHWC.
z2_op = nn_ops.avg_pool(
max_in,
ksize= [w_n, w_h, w_w, w_c],
strides=[s_n, s_h, s_w, s_c],
padding='VALID',
data_format='NHWC')
z, z2 = sess.run([z_op, z2_op])
# print("\nTF:\n-------------------------------------------------")
assert_str = "Failure on i: %d, mode: VALID" % (i)
if not compare_tensor(z, z2, assert_str):
print("z: shape: %s, %s" % (z.shape, z))
print("z (np): shape: %s, %s" % (z2.shape, z2))
print("\n\n")
assert False
return True
def atrous_pool2d(value,ksize, rate, padding, name=None, pooling_type="MAX"):
with ops.name_scope(name, "atrous_pool2d", [value]) as name:
value = ops.convert_to_tensor(value, name="value")
if rate < 1:
raise ValueError("rate {} cannot be less than one".format(rate))
if rate == 1:
if pooling_type == "MAX":
value = nn_ops.max_pool(value=value,
ksize=ksize,
strides=[1, 1, 1, 1],
padding=padding)
return value
elif pooling_type == "AVG":
value = nn_ops.avg_pool(value=value,
ksize=ksize,
strides=[1, 1, 1, 1],
padding=padding)
return value
else:
raise ValueError("Invalid pooling type")
# We have two padding contributions. The first is used for converting "SAME"
# to "VALID". The second is required so that the height and width of the
# zero-padded value tensor are multiples of rate.
# Padding required to reduce to "VALID" convolution
if padding == "SAME":
# Handle filters whose shape is unknown during graph creation.
# if filters.get_shape().is_fully_defined():
# filter_shape = filters.get_shape().as_list()
# else:
# filter_shape = array_ops.shape(filters)
# filter_height, filter_width = filter_shape[0], filter_shape[1]
kernel_height, kernel_width = ksize[1], ksize[2]
# Spatial dimensions of the filters and the upsampled filters in which we
# introduce (rate - 1) zeros between consecutive filter values.
kernel_height_up = kernel_height + (kernel_height - 1) * (rate - 1)
kernel_width_up = kernel_width + (kernel_width - 1) * (rate - 1)
pad_height = kernel_height_up - 1
pad_width = kernel_width_up - 1
# When pad_height (pad_width) is odd, we pad more to bottom (right),
# following the same convention as conv2d().
pad_top = pad_height // 2
pad_bottom = pad_height - pad_top
pad_left = pad_width // 2
pad_right = pad_width - pad_left
elif padding == "VALID":
pad_top = 0
pad_bottom = 0
pad_left = 0
pad_right = 0
else:
raise ValueError("Invalid padding")
# Handle input whose shape is unknown during graph creation.
if value.get_shape().is_fully_defined():
value_shape = value.get_shape().as_list()
else:
value_shape = array_ops.shape(value)
in_height = value_shape[1] + pad_top + pad_bottom
in_width = value_shape[2] + pad_left + pad_right
# More padding so that rate divides the height and width of the input.
pad_bottom_extra = (rate - in_height % rate) % rate
pad_right_extra = (rate - in_width % rate) % rate
# The paddings argument to space_to_batch includes both padding components.
space_to_batch_pad = [[pad_top, pad_bottom + pad_bottom_extra],
[pad_left, pad_right + pad_right_extra]]
value = array_ops.space_to_batch(input=value,
paddings=space_to_batch_pad,
block_size=rate)
if pooling_type == "MAX":
value = nn_ops.max_pool(value=value,
ksize=ksize,
strides=[1, 1, 1, 1],
padding="VALID",
name=name)
elif pooling_type == "AVG":
value = nn_ops.avg_pool(value=value,
ksize=ksize,
strides=[1, 1, 1, 1],
padding="VALID",
name=name)
else:
raise ValueError("Invalid pooling type")
# The crops argument to batch_to_space is just the extra padding component.
batch_to_space_crop = [[0, pad_bottom_extra], [0, pad_right_extra]]
value = array_ops.batch_to_space(input=value,
crops=batch_to_space_crop,
block_size=rate)
return value
