def backprop_pool(self, activation, relevance, ksize, strides, pooling_type, padding='SAME'): if pooling_type.lower() in 'avg': z = nn_ops.avg_pool(activation, ksize, strides, padding) + 1e-10 s = relevance / z c = gen_nn_ops._avg_pool_grad(tf.shape(activation), s, ksize, strides, padding) return activation * c else: z = nn_ops.max_pool(activation, ksize, strides, padding) + 1e-10 s = relevance / z c = gen_nn_ops._max_pool_grad(activation, z, s, ksize, strides, padding) return activation * c
def testDirectUseOverlapping(self): for num_batches in [1, 3]: for row_window_size in [2, 5]: for col_window_size in [2, 4]: num_rows = (row_window_size - 1) * 5 + 1 num_cols = (col_window_size - 1) * 7 + 1 for num_channels in [1, 2]: input_shape = (num_batches, num_rows, num_cols, num_channels) with self.cached_session() as _: input_tensor = constant_op.constant( self._GenerateRandomInputTensor( input_shape).astype(np.float32)) window_size = [ 1, row_window_size, col_window_size, 1 ] stride_size = [ 1, row_window_size - 1, col_window_size - 1, 1 ] padding = "VALID" output_tensor = nn_ops.avg_pool( input_tensor, window_size, stride_size, padding) output_data = self.evaluate(output_tensor) num_elements = 1 for dim_size in output_data.shape: num_elements *= dim_size output_backprop = (self._PRNG.rand(num_elements) * 1000).reshape(output_data.shape) input_backprop_tensor = gen_nn_ops.avg_pool_grad( input_tensor.get_shape(), output_backprop, window_size, stride_size, padding) input_backprop = self.evaluate( input_backprop_tensor) row_seq = list( range(0, num_rows, row_window_size - 1)) col_seq = list( range(0, num_cols, col_window_size - 1)) row_seq[-1] += 1 col_seq[-1] += 1 fap_input_backprop_tensor = gen_nn_ops.fractional_avg_pool_grad( input_tensor.get_shape(), output_backprop, row_seq, col_seq, overlapping=True) fap_input_backprop = self.evaluate( fap_input_backprop_tensor) self.assertShapeEqual(input_backprop, fap_input_backprop_tensor) self.assertAllClose(input_backprop, fap_input_backprop)
def _RedoRestAvgPool(graph): """Finds fused batch norm layers and folds them into preceding layers. Folding only affects the following layers: Conv2D, fully connected, depthwise convolution. Args: graph: Graph to walk and modify. is_training: Bool, true if training. Raises: ValueError: When batch norm folding fails. """ matches = _FindRestAvgPool(graph) print("Replacing", len(matches), "AvgPool") for match in matches: scope, sep, _ = match['layer_op'].name.rpartition('/') # Make sure new ops are added to `graph` and put on the same device as # `bn_op`. The '/' (i.e. `sep`) ensures that we reuse the existing scope # named `scope`. Otherwise, TF creates a unique scope whose name starts with # `scope`. with graph.as_default(), graph.name_scope(scope + sep): # with graph.name_scope(scope + sep + '_psb' + sep): input_tensor = match['input_tensor'] layer_op = match['layer_op'] # output_tensor = match['output_tensor'] # >>>>> CUSTOM >>>>>>>>>>>>>> avg_size = np.prod(layer_op.get_attr("ksize")).astype(np.float32) if avg_size == 2**np.log2(avg_size): continue output_tensor = nn_ops.avg_pool( input_tensor, ksize=layer_op.get_attr('ksize'), strides=layer_op.get_attr('strides'), padding=layer_op.get_attr('padding'), data_format=layer_op.get_attr('data_format'), name=layer_op.name.split('/')[-1] + '_psb') avg_size_new = variableFromSettings( [], hiddenVar=(1.0 / avg_size).astype(np.float32))[0] new_layer_tensor = output_tensor * avg_size * avg_size_new # <<<<<<<<<<<<<<<<<<<<<<<<<<< nodes_modified_count = common.RerouteTensor( new_layer_tensor, match['output_tensor']) if nodes_modified_count == 0: raise ValueError( 'Folding batch norms failed, %s had no outputs.' % match['output_tensor'].name)
def backprop_pool(activation, relevance, ksize, strides, pooling_type, padding='VALID'): if pooling_type.lower() is 'avg': # avg pooling z = nn_ops.avg_pool(activation, ksize, strides, padding) + 1e-10 s = relevance / z c = gen_nn_ops._avg_pool_grad(tf.shape(activation), s, ksize, strides, padding) return activation * c else: # max pooling z = nn_ops.max_pool(activation, ksize, strides, padding) + 1e-10 s = relevance / z c = gen_nn_ops.max_pool_grad(activation, z, s, ksize, strides, padding) return activation * c
def testDirectUseOverlapping(self): for num_batches in [1, 3]: for row_window_size in [2, 5]: for col_window_size in [2, 4]: num_rows = (row_window_size - 1) * 5 + 1 num_cols = (col_window_size - 1) * 7 + 1 for num_channels in [1, 2]: input_shape = (num_batches, num_rows, num_cols, num_channels) with self.cached_session() as _: input_tensor = constant_op.constant( self._GenerateRandomInputTensor(input_shape).astype( np.float32)) window_size = [1, row_window_size, col_window_size, 1] stride_size = [1, row_window_size - 1, col_window_size - 1, 1] padding = "VALID" output_tensor = nn_ops.avg_pool(input_tensor, window_size, stride_size, padding) output_data = self.evaluate(output_tensor) num_elements = 1 for dim_size in output_data.shape: num_elements *= dim_size output_backprop = (self._PRNG.rand(num_elements) * 1000).reshape(output_data.shape) input_backprop_tensor = gen_nn_ops.avg_pool_grad( input_tensor.get_shape(), output_backprop, window_size, stride_size, padding) input_backprop = self.evaluate(input_backprop_tensor) row_seq = list(range(0, num_rows, row_window_size - 1)) col_seq = list(range(0, num_cols, col_window_size - 1)) row_seq[-1] += 1 col_seq[-1] += 1 fap_input_backprop_tensor = gen_nn_ops.fractional_avg_pool_grad( input_tensor.get_shape(), output_backprop, row_seq, col_seq, overlapping=True) fap_input_backprop = self.evaluate(fap_input_backprop_tensor) self.assertShapeEqual(input_backprop, fap_input_backprop_tensor) self.assertAllClose(input_backprop, fap_input_backprop)
def atrous_avg_pool(value, size, rate, padding, name=None, info=DummyDict()): with tfops.op_scope([value], name, "atrous_avg_pool") as name: value = tfops.convert_to_tensor(value, name="value") if rate < 1: raise ValueError("rate {} cannot be less than one".format(rate)) if rate == 1: value = nn_ops.avg_pool(value=value, strides=[1, 1, 1, 1], ksize=[1, size, size, 1], padding=padding) return value # We have two padding contributions. The first is used for converting "SAME" # to "VALID". The second is required so that the height and width of the # zero-padded value tensor are multiples of rate. # Padding required to reduce to "VALID" convolution if padding == "SAME": filter_height, filter_width = size, size # Spatial dimensions of the filters and the upsampled filters in which we # introduce (rate - 1) zeros between consecutive filter values. filter_height_up = filter_height + (filter_height - 1) * (rate - 1) filter_width_up = filter_width + (filter_width - 1) * (rate - 1) pad_height = filter_height_up - 1 pad_width = filter_width_up - 1 # When pad_height (pad_width) is odd, we pad more to bottom (right), # following the same convention as avg_pool(). pad_top = pad_height // 2 pad_bottom = pad_height - pad_top pad_left = pad_width // 2 pad_right = pad_width - pad_left elif padding == "VALID": pad_top = 0 pad_bottom = 0 pad_left = 0 pad_right = 0 else: raise ValueError("Invalid padding") # Handle input whose shape is unknown during graph creation. if value.get_shape().is_fully_defined(): value_shape = value.get_shape().as_list() else: value_shape = array_ops.shape(value) in_height = value_shape[1] + pad_top + pad_bottom in_width = value_shape[2] + pad_left + pad_right # More padding so that rate divides the height and width of the input. pad_bottom_extra = (rate - in_height % rate) % rate pad_right_extra = (rate - in_width % rate) % rate # The paddings argument to space_to_batch includes both padding components. space_to_batch_pad = [[pad_top, pad_bottom + pad_bottom_extra], [pad_left, pad_right + pad_right_extra]] value = array_ops.space_to_batch(input=value, paddings=space_to_batch_pad, block_size=rate) value = nn_ops.avg_pool(value=value, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding="VALID", name=name) # The crops argument to batch_to_space is just the extra padding component. batch_to_space_crop = [[0, pad_bottom_extra], [0, pad_right_extra]] value = array_ops.batch_to_space(input=value, crops=batch_to_space_crop, block_size=rate) info['activations'][name] = value return value
def test_avgpool2d(): ''' Run tests on the Wave custom avgpool2d operator. ''' tf.reset_default_graph() # Turn off graph-rewriting optimizations config = tf.ConfigProto(graph_options=tf.GraphOptions(optimizer_options=tf.OptimizerOptions(opt_level=tf.OptimizerOptions.L0))) iterations = 100 for i in range(iterations): tf.reset_default_graph() # NCHW t_n = 1 t_h = 64 t_w = 64 t_c = 2 # window w_n = 1 w_h = 2 w_w = 2 w_c = 1 #strides s_n = 1 s_h = 2 s_w = 2 s_c = 1 # N H W C max_in = tf.get_variable("a", [t_n, t_h, t_w, t_c], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.1)) t_init = tf.global_variables_initializer() # SAME variant with tf.Session('', config=config) as sess: t_init.run() # print("Wave Kernel:\n-------------------------------------------------") z_op = waveflow.wavecomp_ops_module.wave_avg_pool_dfx( max_in, ksize=[w_n, w_h, w_w, w_c], strides=[s_n, s_h, s_w, s_c], padding='SAME', data_format='NHWC') # Base tensorflow. Only supports NHWC. z2_op = nn_ops.avg_pool( max_in, ksize=[w_n, w_h, w_w, w_c], strides=[s_n, s_h, s_w, s_c], padding='SAME', data_format='NHWC') # z = z_op.eval() # z2 = z2_op.eval() z, z2 = sess.run([z_op, z2_op]) # print("\nTF:\n-------------------------------------------------") assert_str = "Failure on i: %d, mode: SAME" % (i) if not compare_tensor(z, z2, assert_str): print("z: shape: %s, %s" % (z.shape, z)) print("z (np): shape: %s, %s" % (z2.shape, z2)) print("\n\n") assert False # Valid variant with tf.Session('', config=config) as sess: t_init.run() # print("Wave Kernel:\n-------------------------------------------------") z_op = waveflow.wavecomp_ops_module.wave_avg_pool_dfx( max_in, ksize=[w_n, w_h, w_w, w_c], strides=[s_n, s_h, s_w, s_c], padding='VALID', data_format='NHWC') # Base tensorflow. Only supports NHWC. z2_op = nn_ops.avg_pool( max_in, ksize= [w_n, w_h, w_w, w_c], strides=[s_n, s_h, s_w, s_c], padding='VALID', data_format='NHWC') z, z2 = sess.run([z_op, z2_op]) # print("\nTF:\n-------------------------------------------------") assert_str = "Failure on i: %d, mode: VALID" % (i) if not compare_tensor(z, z2, assert_str): print("z: shape: %s, %s" % (z.shape, z)) print("z (np): shape: %s, %s" % (z2.shape, z2)) print("\n\n") assert False return True
def atrous_pool2d(value,ksize, rate, padding, name=None, pooling_type="MAX"): with ops.name_scope(name, "atrous_pool2d", [value]) as name: value = ops.convert_to_tensor(value, name="value") if rate < 1: raise ValueError("rate {} cannot be less than one".format(rate)) if rate == 1: if pooling_type == "MAX": value = nn_ops.max_pool(value=value, ksize=ksize, strides=[1, 1, 1, 1], padding=padding) return value elif pooling_type == "AVG": value = nn_ops.avg_pool(value=value, ksize=ksize, strides=[1, 1, 1, 1], padding=padding) return value else: raise ValueError("Invalid pooling type") # We have two padding contributions. The first is used for converting "SAME" # to "VALID". The second is required so that the height and width of the # zero-padded value tensor are multiples of rate. # Padding required to reduce to "VALID" convolution if padding == "SAME": # Handle filters whose shape is unknown during graph creation. # if filters.get_shape().is_fully_defined(): # filter_shape = filters.get_shape().as_list() # else: # filter_shape = array_ops.shape(filters) # filter_height, filter_width = filter_shape[0], filter_shape[1] kernel_height, kernel_width = ksize[1], ksize[2] # Spatial dimensions of the filters and the upsampled filters in which we # introduce (rate - 1) zeros between consecutive filter values. kernel_height_up = kernel_height + (kernel_height - 1) * (rate - 1) kernel_width_up = kernel_width + (kernel_width - 1) * (rate - 1) pad_height = kernel_height_up - 1 pad_width = kernel_width_up - 1 # When pad_height (pad_width) is odd, we pad more to bottom (right), # following the same convention as conv2d(). pad_top = pad_height // 2 pad_bottom = pad_height - pad_top pad_left = pad_width // 2 pad_right = pad_width - pad_left elif padding == "VALID": pad_top = 0 pad_bottom = 0 pad_left = 0 pad_right = 0 else: raise ValueError("Invalid padding") # Handle input whose shape is unknown during graph creation. if value.get_shape().is_fully_defined(): value_shape = value.get_shape().as_list() else: value_shape = array_ops.shape(value) in_height = value_shape[1] + pad_top + pad_bottom in_width = value_shape[2] + pad_left + pad_right # More padding so that rate divides the height and width of the input. pad_bottom_extra = (rate - in_height % rate) % rate pad_right_extra = (rate - in_width % rate) % rate # The paddings argument to space_to_batch includes both padding components. space_to_batch_pad = [[pad_top, pad_bottom + pad_bottom_extra], [pad_left, pad_right + pad_right_extra]] value = array_ops.space_to_batch(input=value, paddings=space_to_batch_pad, block_size=rate) if pooling_type == "MAX": value = nn_ops.max_pool(value=value, ksize=ksize, strides=[1, 1, 1, 1], padding="VALID", name=name) elif pooling_type == "AVG": value = nn_ops.avg_pool(value=value, ksize=ksize, strides=[1, 1, 1, 1], padding="VALID", name=name) else: raise ValueError("Invalid pooling type") # The crops argument to batch_to_space is just the extra padding component. batch_to_space_crop = [[0, pad_bottom_extra], [0, pad_right_extra]] value = array_ops.batch_to_space(input=value, crops=batch_to_space_crop, block_size=rate) return value