def forward(self, inputs): inputs = quantize_active_overflow(inputs, self.bitA) W_ = quantize_weight_overflow(self.W, self.bitW) outputs = tf.matmul(inputs, self.W) if self.b_init is not None: outputs = tf.nn.bias_add(outputs, self.b, name='bias_add') if self.act: outputs = self.act(outputs) return outputs
def forward(self, inputs): inputs = quantize_active_overflow(inputs, self.bitA) W_ = quantize_weight_overflow(self.W, self.bitW) # outputs = tf.matmul(inputs, self.W) outputs = tf.matmul(inputs, W_) # hao dong change to this if self.b_init is not None: outputs = tf.nn.bias_add(outputs, self.b, name='bias_add') if self.act: outputs = self.act(outputs) return outputs
def forward(self, inputs): x = inputs inputs = quantize_active_overflow(inputs, self.bitA) # Do not remove outputs = tf.nn.conv2d(input=x, filters=self.W, strides=self._strides, padding=self.padding, data_format=self.data_format, dilations=self._dilation_rate, name=self.name) mean, variance = tf.nn.moments( outputs, axes=list(range(len(outputs.get_shape()) - 1))) update_moving_mean = moving_averages.assign_moving_average( self.moving_mean, mean, self.decay, zero_debias=False) # if zero_debias=True, has bias update_moving_variance = moving_averages.assign_moving_average( self.moving_variance, mean, self.decay, zero_debias=False) # if zero_debias=True, has bias if self.is_train: mean, var = self.mean_var_with_update(update_moving_mean, update_moving_variance, mean, variance) else: mean, var = self.moving_mean, self.moving_variance w_fold = self._w_fold(self.W, self.scale_para, var, self.epsilon) W_ = quantize_weight_overflow(w_fold, self.bitW) conv_fold = tf.nn.conv2d(inputs, W_, strides=self.strides, padding=self.padding, data_format=self.data_format) if self.beta_init: bias_fold = self._bias_fold(self.offset_para, self.scale_para, mean, var, self.epsilon) conv_fold = tf.nn.bias_add(conv_fold, bias_fold, name='bn_bias_add') if self.act: conv_fold = self.act(conv_fold) return conv_fold
def forward(self, inputs): inputs = quantize_active_overflow(inputs, self.bitA) # Do not remove W_ = quantize_weight_overflow(self.W, self.bitW) outputs = tf.nn.conv2d( inputs, W_, strides=self.strides, padding=self.padding, use_cudnn_on_gpu=self.use_cudnn_on_gpu, data_format=self.data_format ) if self.b_init: outputs = tf.nn.bias_add(outputs, self.b, name='bias_add') if self.act: outputs = self.act(outputs) return outputs
def forward(self, inputs): x = inputs inputs = quantize_active_overflow(inputs, self.bitA) mid_out = tf.matmul(x, self.W) mean, variance = tf.nn.moments( x=mid_out, axes=list(range(len(mid_out.get_shape()) - 1))) update_moving_mean = moving_averages.assign_moving_average( self.moving_mean, mean, self.decay, zero_debias=False) # if zero_debias=True, has bias update_moving_variance = moving_averages.assign_moving_average( self.moving_variance, variance, self.decay, zero_debias=False) # if zero_debias=True, has bias if self.is_train: mean, var = self.mean_var_with_update(update_moving_mean, update_moving_variance, mean, variance) else: mean, var = self.moving_mean, self.moving_variance w_fold = self._w_fold(self.W, self.scale_para, var, self.epsilon) W = quantize_weight_overflow(w_fold, self.bitW) outputs = tf.matmul(inputs, W) if self.beta_init: bias_fold = self._bias_fold(self.offset_para, self.scale_para, mean, var, self.epsilon) outputs = tf.nn.bias_add(outputs, bias_fold, name='bias_add') else: outputs = outputs if self.act: outputs = self.act(outputs) else: outputs = outputs return outputs
def forward(self, inputs): inputs = quantize_active_overflow(inputs, self.bitA) # Do not remove W_ = quantize_weight_overflow(self.W, self.bitW) outputs = tf.nn.conv2d(input=inputs, filters=W_, strides=self.strides, padding=self.padding, data_format=self.data_format, dilations=self._dilation_rate, name=self.name) if self.b_init: outputs = tf.nn.bias_add(outputs, self.b, data_format=self.data_format, name='bias_add') if self.act: outputs = self.act(outputs) return outputs
def __init__( self, prev_layer, n_units=100, act=None, bitW=8, bitA=8, use_gemm=False, W_init=tf.truncated_normal_initializer(stddev=0.1), b_init=tf.constant_initializer(value=0.0), W_init_args=None, b_init_args=None, name='quan_dense', ): super(QuanDenseLayer, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, b_init_args=b_init_args, name=name) logging.info( "QuanDenseLayer %s: %d %s" % (self.name, n_units, self.act.__name__ if self.act is not None else 'No Activation')) if self.inputs.get_shape().ndims != 2: raise Exception( "The input dimension must be rank 2, please reshape or flatten it" ) if use_gemm: raise Exception( "TODO. The current version use tf.matmul for inferencing.") n_in = int(self.inputs.get_shape()[-1]) self.inputs = quantize_active_overflow(self.inputs, bitA) self.n_units = n_units with tf.variable_scope(name): W = tf.get_variable(name='W', shape=(n_in, n_units), initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args) W = quantize_weight_overflow(W, bitW) self.outputs = tf.matmul(self.inputs, W) if b_init is not None: try: b = tf.get_variable(name='b', shape=(n_units), initializer=b_init, dtype=LayersConfig.tf_dtype, **self.b_init_args) except Exception: # If initializer is a constant, do not specify shape. b = tf.get_variable(name='b', initializer=b_init, dtype=LayersConfig.tf_dtype, **self.b_init_args) self.outputs = tf.nn.bias_add(self.outputs, b, name='bias_add') self.outputs = self._apply_activation(self.outputs) self._add_layers(self.outputs) if b_init is not None: self._add_params([W, b]) else: self._add_params(W)
def __init__( self, prev_layer, n_units=100, act=None, decay=0.9, epsilon=1e-5, is_train=False, bitW=8, bitA=8, gamma_init=tf.compat.v1.initializers.ones, beta_init=tf.compat.v1.initializers.zeros, use_gemm=False, W_init=tf.compat.v1.initializers.truncated_normal(stddev=0.1), W_init_args=None, name=None, #'quan_dense_with_bn', ): super(QuanDenseLayerWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name) logging.info( "QuanDenseLayerWithBN %s: %d %s" % (self.name, n_units, self.act.__name__ if self.act is not None else 'No Activation')) if self.inputs.get_shape().ndims != 2: raise Exception( "The input dimension must be rank 2, please reshape or flatten it" ) if use_gemm: raise Exception( "TODO. The current version use tf.matmul for inferencing.") n_in = int(self.inputs.get_shape()[-1]) x = self.inputs self.inputs = quantize_active_overflow(self.inputs, bitA) self.n_units = n_units with tf.compat.v1.variable_scope(name): W = tf.compat.v1.get_variable(name='W', shape=(n_in, n_units), initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args) mid_out = tf.matmul(x, W) para_bn_shape = mid_out.get_shape()[-1:] if gamma_init: scale_para = tf.compat.v1.get_variable( name='scale_para', shape=para_bn_shape, initializer=gamma_init, dtype=LayersConfig.tf_dtype, trainable=is_train) else: scale_para = None if beta_init: offset_para = tf.compat.v1.get_variable( name='offset_para', shape=para_bn_shape, initializer=beta_init, dtype=LayersConfig.tf_dtype, trainable=is_train) else: offset_para = None moving_mean = tf.compat.v1.get_variable( 'moving_mean', para_bn_shape, initializer=tf.compat.v1.initializers.constant(1.), dtype=LayersConfig.tf_dtype, trainable=False) moving_variance = tf.compat.v1.get_variable( 'moving_variance', para_bn_shape, initializer=tf.compat.v1.initializers.constant(1.), dtype=LayersConfig.tf_dtype, trainable=False, ) mean, variance = tf.nn.moments( x=mid_out, axes=list(range(len(mid_out.get_shape()) - 1))) update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean, decay, zero_debias=False) # if zero_debias=True, has bias update_moving_variance = moving_averages.assign_moving_average( moving_variance, variance, decay, zero_debias=False) # if zero_debias=True, has bias def mean_var_with_update(): with tf.control_dependencies( [update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) if is_train: mean, var = mean_var_with_update() else: mean, var = moving_mean, moving_variance w_fold = _w_fold(W, scale_para, var, epsilon) bias_fold = _bias_fold(offset_para, scale_para, mean, var, epsilon) W = quantize_weight_overflow(w_fold, bitW) # W = tl.act.sign(W) # dont update ... # W = tf.Variable(W) self.outputs = tf.matmul(self.inputs, W) # self.outputs = xnor_gemm(self.inputs, W) # TODO self.outputs = tf.nn.bias_add(self.outputs, bias_fold, name='bias_add') self.outputs = self._apply_activation(self.outputs) self._add_layers(self.outputs) self._add_params( [W, scale_para, offset_para, moving_mean, moving_variance])
def __init__( self, prev_layer, n_filter=32, filter_size=(3, 3), strides=(1, 1), padding='SAME', act=None, bitW=8, bitA=8, use_gemm=False, W_init=tf.truncated_normal_initializer(stddev=0.02), b_init=tf.constant_initializer(value=0.0), W_init_args=None, b_init_args=None, use_cudnn_on_gpu=None, data_format=None, name='quan_cnn2d', ): super(QuanConv2d, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, b_init_args=b_init_args, name=name) logging.info( "QuanConv2d %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s" % (self.name, n_filter, str(filter_size), str(strides), padding, self.act.__name__ if self.act is not None else 'No Activation')) self.inputs = quantize_active_overflow(self.inputs, bitA) # Do not remove if use_gemm: raise Exception( "TODO. The current version use tf.matmul for inferencing.") if len(strides) != 2: raise ValueError("len(strides) should be 2.") try: pre_channel = int(prev_layer.outputs.get_shape()[-1]) except Exception: # if pre_channel is ?, it happens when using Spatial Transformer Net pre_channel = 1 logging.warning("[warnings] unknow input channels, set to 1") shape = (filter_size[0], filter_size[1], pre_channel, n_filter) strides = (1, strides[0], strides[1], 1) with tf.variable_scope(name): W = tf.get_variable(name='W_conv2d', shape=shape, initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args) W = quantize_weight_overflow(W, bitW) self.outputs = tf.nn.conv2d(self.inputs, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) if b_init: b = tf.get_variable(name='b_conv2d', shape=(shape[-1]), initializer=b_init, dtype=LayersConfig.tf_dtype, **self.b_init_args) self.outputs = tf.nn.bias_add(self.outputs, b, name='bias_add') self.outputs = self._apply_activation(self.outputs) self._add_layers(self.outputs) if b_init: self._add_params([W, b]) else: self._add_params(W)
def __init__( self, prev_layer, n_filter=32, filter_size=(3, 3), strides=(1, 1), padding='SAME', act=None, decay=0.9, epsilon=1e-5, is_train=False, gamma_init=tf.compat.v1.initializers.ones, beta_init=tf.compat.v1.initializers.zeros, bitW=8, bitA=8, use_gemm=False, W_init=tf.compat.v1.initializers.truncated_normal(stddev=0.02), W_init_args=None, use_cudnn_on_gpu=None, data_format=None, name='quan_cnn2d_bn', ): super(QuanConv2dWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name) logging.info( "QuanConv2dWithBN %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s " % ( self.name, n_filter, filter_size, str(strides), padding, self.act.__name__ if self.act is not None else 'No Activation' ) ) x = self.inputs self.inputs = quantize_active_overflow(self.inputs, bitA) # Do not remove if use_gemm: raise Exception("TODO. The current version use tf.matmul for inferencing.") if len(strides) != 2: raise ValueError("len(strides) should be 2.") try: pre_channel = int(prev_layer.outputs.get_shape()[-1]) except Exception: # if pre_channel is ?, it happens when using Spatial Transformer Net pre_channel = 1 logging.warning("[warnings] unknow input channels, set to 1") shape = (filter_size[0], filter_size[1], pre_channel, n_filter) strides = (1, strides[0], strides[1], 1) with tf.compat.v1.variable_scope(name): W = tf.compat.v1.get_variable( name='W_conv2d', shape=shape, initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args ) conv = tf.nn.conv2d( x, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format ) para_bn_shape = conv.get_shape()[-1:] if gamma_init: scale_para = tf.compat.v1.get_variable( name='scale_para', shape=para_bn_shape, initializer=gamma_init, dtype=LayersConfig.tf_dtype, trainable=is_train ) else: scale_para = None if beta_init: offset_para = tf.compat.v1.get_variable( name='offset_para', shape=para_bn_shape, initializer=beta_init, dtype=LayersConfig.tf_dtype, trainable=is_train ) else: offset_para = None moving_mean = tf.compat.v1.get_variable( 'moving_mean', para_bn_shape, initializer=tf.compat.v1.initializers.constant(1.), dtype=LayersConfig.tf_dtype, trainable=False ) moving_variance = tf.compat.v1.get_variable( 'moving_variance', para_bn_shape, initializer=tf.compat.v1.initializers.constant(1.), dtype=LayersConfig.tf_dtype, trainable=False, ) mean, variance = tf.nn.moments(x=conv, axes=list(range(len(conv.get_shape()) - 1))) update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean, decay, zero_debias=False ) # if zero_debias=True, has bias update_moving_variance = moving_averages.assign_moving_average( moving_variance, variance, decay, zero_debias=False ) # if zero_debias=True, has bias def mean_var_with_update(): with tf.control_dependencies([update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) if is_train: mean, var = mean_var_with_update() else: mean, var = moving_mean, moving_variance w_fold = _w_fold(W, scale_para, var, epsilon) bias_fold = _bias_fold(offset_para, scale_para, mean, var, epsilon) W = quantize_weight_overflow(w_fold, bitW) conv_fold = tf.nn.conv2d( self.inputs, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format ) self.outputs = tf.nn.bias_add(conv_fold, bias_fold, name='bn_bias_add') self.outputs = self._apply_activation(self.outputs) self._add_layers(self.outputs) self._add_params([W, scale_para, offset_para, moving_mean, moving_variance])
def __init__( self, prev_layer, n_units=100, act=None, decay=0.9, epsilon=1e-5, is_train=False, bitW=8, bitA=8, gamma_init=tf.compat.v1.initializers.ones, beta_init=tf.compat.v1.initializers.zeros, use_gemm=False, W_init=tf.compat.v1.initializers.truncated_normal(stddev=0.1), W_init_args=None, name=None, #'quan_dense_with_bn', ): super(QuanDenseLayerWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name) logging.info( "QuanDenseLayerWithBN %s: %d %s" % (self.name, n_units, self.act.__name__ if self.act is not None else 'No Activation') ) if self.inputs.get_shape().ndims != 2: raise Exception("The input dimension must be rank 2, please reshape or flatten it") if use_gemm: raise Exception("TODO. The current version use tf.matmul for inferencing.") n_in = int(self.inputs.get_shape()[-1]) x = self.inputs self.inputs = quantize_active_overflow(self.inputs, bitA) self.n_units = n_units with tf.compat.v1.variable_scope(name): W = tf.compat.v1.get_variable( name='W', shape=(n_in, n_units), initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args ) mid_out = tf.matmul(x, W) para_bn_shape = mid_out.get_shape()[-1:] if gamma_init: scale_para = tf.compat.v1.get_variable( name='scale_para', shape=para_bn_shape, initializer=gamma_init, dtype=LayersConfig.tf_dtype, trainable=is_train ) else: scale_para = None if beta_init: offset_para = tf.compat.v1.get_variable( name='offset_para', shape=para_bn_shape, initializer=beta_init, dtype=LayersConfig.tf_dtype, trainable=is_train ) else: offset_para = None moving_mean = tf.compat.v1.get_variable( 'moving_mean', para_bn_shape, initializer=tf.compat.v1.initializers.constant(1.), dtype=LayersConfig.tf_dtype, trainable=False ) moving_variance = tf.compat.v1.get_variable( 'moving_variance', para_bn_shape, initializer=tf.compat.v1.initializers.constant(1.), dtype=LayersConfig.tf_dtype, trainable=False, ) mean, variance = tf.nn.moments(x=mid_out, axes=list(range(len(mid_out.get_shape()) - 1))) update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean, decay, zero_debias=False ) # if zero_debias=True, has bias update_moving_variance = moving_averages.assign_moving_average( moving_variance, variance, decay, zero_debias=False ) # if zero_debias=True, has bias def mean_var_with_update(): with tf.control_dependencies([update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) if is_train: mean, var = mean_var_with_update() else: mean, var = moving_mean, moving_variance w_fold = _w_fold(W, scale_para, var, epsilon) bias_fold = _bias_fold(offset_para, scale_para, mean, var, epsilon) W = quantize_weight_overflow(w_fold, bitW) # W = tl.act.sign(W) # dont update ... # W = tf.Variable(W) self.outputs = tf.matmul(self.inputs, W) # self.outputs = xnor_gemm(self.inputs, W) # TODO self.outputs = tf.nn.bias_add(self.outputs, bias_fold, name='bias_add') self.outputs = self._apply_activation(self.outputs) self._add_layers(self.outputs) self._add_params([W, scale_para, offset_para, moving_mean, moving_variance])