def forward(self, inputs):
inputs = quantize_active_overflow(inputs, self.bitA)
W_ = quantize_weight_overflow(self.W, self.bitW)
outputs = tf.matmul(inputs, self.W)
if self.b_init is not None:
outputs = tf.nn.bias_add(outputs, self.b, name='bias_add')
if self.act:
outputs = self.act(outputs)
return outputs
def forward(self, inputs):
inputs = quantize_active_overflow(inputs, self.bitA)
W_ = quantize_weight_overflow(self.W, self.bitW)
# outputs = tf.matmul(inputs, self.W)
outputs = tf.matmul(inputs, W_) # hao dong change to this
if self.b_init is not None:
outputs = tf.nn.bias_add(outputs, self.b, name='bias_add')
if self.act:
outputs = self.act(outputs)
return outputs
def forward(self, inputs):
x = inputs
inputs = quantize_active_overflow(inputs, self.bitA) # Do not remove
outputs = tf.nn.conv2d(input=x,
filters=self.W,
strides=self._strides,
padding=self.padding,
data_format=self.data_format,
dilations=self._dilation_rate,
name=self.name)
mean, variance = tf.nn.moments(
outputs, axes=list(range(len(outputs.get_shape()) - 1)))
update_moving_mean = moving_averages.assign_moving_average(
self.moving_mean, mean, self.decay,
zero_debias=False) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
self.moving_variance, mean, self.decay,
zero_debias=False) # if zero_debias=True, has bias
if self.is_train:
mean, var = self.mean_var_with_update(update_moving_mean,
update_moving_variance, mean,
variance)
else:
mean, var = self.moving_mean, self.moving_variance
w_fold = self._w_fold(self.W, self.scale_para, var, self.epsilon)
W_ = quantize_weight_overflow(w_fold, self.bitW)
conv_fold = tf.nn.conv2d(inputs,
W_,
strides=self.strides,
padding=self.padding,
data_format=self.data_format)
if self.beta_init:
bias_fold = self._bias_fold(self.offset_para, self.scale_para,
mean, var, self.epsilon)
conv_fold = tf.nn.bias_add(conv_fold,
bias_fold,
name='bn_bias_add')
if self.act:
conv_fold = self.act(conv_fold)
return conv_fold
def forward(self, inputs):
inputs = quantize_active_overflow(inputs, self.bitA) # Do not remove
W_ = quantize_weight_overflow(self.W, self.bitW)
outputs = tf.nn.conv2d(
inputs, W_, strides=self.strides, padding=self.padding, use_cudnn_on_gpu=self.use_cudnn_on_gpu,
data_format=self.data_format
)
if self.b_init:
outputs = tf.nn.bias_add(outputs, self.b, name='bias_add')
if self.act:
outputs = self.act(outputs)
return outputs
def forward(self, inputs):
x = inputs
inputs = quantize_active_overflow(inputs, self.bitA)
mid_out = tf.matmul(x, self.W)
mean, variance = tf.nn.moments(
x=mid_out, axes=list(range(len(mid_out.get_shape()) - 1)))
update_moving_mean = moving_averages.assign_moving_average(
self.moving_mean, mean, self.decay,
zero_debias=False) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
self.moving_variance, variance, self.decay,
zero_debias=False) # if zero_debias=True, has bias
if self.is_train:
mean, var = self.mean_var_with_update(update_moving_mean,
update_moving_variance, mean,
variance)
else:
mean, var = self.moving_mean, self.moving_variance
w_fold = self._w_fold(self.W, self.scale_para, var, self.epsilon)
W = quantize_weight_overflow(w_fold, self.bitW)
outputs = tf.matmul(inputs, W)
if self.beta_init:
bias_fold = self._bias_fold(self.offset_para, self.scale_para,
mean, var, self.epsilon)
outputs = tf.nn.bias_add(outputs, bias_fold, name='bias_add')
else:
outputs = outputs
if self.act:
outputs = self.act(outputs)
else:
outputs = outputs
return outputs
def __init__(
self,
prev_layer,
n_units=100,
act=None,
decay=0.9,
epsilon=1e-5,
is_train=False,
bitW=8,
bitA=8,
gamma_init=tf.compat.v1.initializers.ones,
beta_init=tf.compat.v1.initializers.zeros,
use_gemm=False,
W_init=tf.compat.v1.initializers.truncated_normal(stddev=0.1),
W_init_args=None,
name=None, #'quan_dense_with_bn',
):
super(QuanDenseLayerWithBN, self).__init__(prev_layer=prev_layer,
act=act,
W_init_args=W_init_args,
name=name)
logging.info(
"QuanDenseLayerWithBN %s: %d %s" %
(self.name, n_units,
self.act.__name__ if self.act is not None else 'No Activation'))
if self.inputs.get_shape().ndims != 2:
raise Exception(
"The input dimension must be rank 2, please reshape or flatten it"
)
if use_gemm:
raise Exception(
"TODO. The current version use tf.matmul for inferencing.")
n_in = int(self.inputs.get_shape()[-1])
x = self.inputs
self.inputs = quantize_active_overflow(self.inputs, bitA)
self.n_units = n_units
with tf.compat.v1.variable_scope(name):
W = tf.compat.v1.get_variable(name='W',
shape=(n_in, n_units),
initializer=W_init,
dtype=LayersConfig.tf_dtype,
**self.W_init_args)
mid_out = tf.matmul(x, W)
para_bn_shape = mid_out.get_shape()[-1:]
if gamma_init:
scale_para = tf.compat.v1.get_variable(
name='scale_para',
shape=para_bn_shape,
initializer=gamma_init,
dtype=LayersConfig.tf_dtype,
trainable=is_train)
else:
scale_para = None
if beta_init:
offset_para = tf.compat.v1.get_variable(
name='offset_para',
shape=para_bn_shape,
initializer=beta_init,
dtype=LayersConfig.tf_dtype,
trainable=is_train)
else:
offset_para = None
moving_mean = tf.compat.v1.get_variable(
'moving_mean',
para_bn_shape,
initializer=tf.compat.v1.initializers.constant(1.),
dtype=LayersConfig.tf_dtype,
trainable=False)
moving_variance = tf.compat.v1.get_variable(
'moving_variance',
para_bn_shape,
initializer=tf.compat.v1.initializers.constant(1.),
dtype=LayersConfig.tf_dtype,
trainable=False,
)
mean, variance = tf.nn.moments(
x=mid_out, axes=list(range(len(mid_out.get_shape()) - 1)))
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay,
zero_debias=False) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay,
zero_debias=False) # if zero_debias=True, has bias
def mean_var_with_update():
with tf.control_dependencies(
[update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
if is_train:
mean, var = mean_var_with_update()
else:
mean, var = moving_mean, moving_variance
w_fold = _w_fold(W, scale_para, var, epsilon)
bias_fold = _bias_fold(offset_para, scale_para, mean, var, epsilon)
W = quantize_weight_overflow(w_fold, bitW)
# W = tl.act.sign(W) # dont update ...
# W = tf.Variable(W)
self.outputs = tf.matmul(self.inputs, W)
# self.outputs = xnor_gemm(self.inputs, W) # TODO
self.outputs = tf.nn.bias_add(self.outputs,
bias_fold,
name='bias_add')
self.outputs = self._apply_activation(self.outputs)
self._add_layers(self.outputs)
self._add_params(
[W, scale_para, offset_para, moving_mean, moving_variance])
def __init__(
self,
prev_layer,
n_filter=32,
filter_size=(3, 3),
strides=(1, 1),
padding='SAME',
act=None,
bitW=8,
bitA=8,
use_gemm=False,
W_init=tf.truncated_normal_initializer(stddev=0.02),
b_init=tf.constant_initializer(value=0.0),
W_init_args=None,
b_init_args=None,
use_cudnn_on_gpu=None,
data_format=None,
name='quan_cnn2d',
):
super(QuanConv2d, self).__init__(prev_layer=prev_layer,
act=act,
W_init_args=W_init_args,
b_init_args=b_init_args,
name=name)
logging.info(
"QuanConv2d %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s"
% (self.name, n_filter, str(filter_size), str(strides), padding,
self.act.__name__ if self.act is not None else 'No Activation'))
self.inputs = quantize_active_overflow(self.inputs,
bitA) # Do not remove
if use_gemm:
raise Exception(
"TODO. The current version use tf.matmul for inferencing.")
if len(strides) != 2:
raise ValueError("len(strides) should be 2.")
try:
pre_channel = int(prev_layer.outputs.get_shape()[-1])
except Exception: # if pre_channel is ?, it happens when using Spatial Transformer Net
pre_channel = 1
logging.warning("[warnings] unknow input channels, set to 1")
shape = (filter_size[0], filter_size[1], pre_channel, n_filter)
strides = (1, strides[0], strides[1], 1)
with tf.variable_scope(name):
W = tf.get_variable(name='W_conv2d',
shape=shape,
initializer=W_init,
dtype=LayersConfig.tf_dtype,
**self.W_init_args)
W = quantize_weight_overflow(W, bitW)
self.outputs = tf.nn.conv2d(self.inputs,
W,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
if b_init:
b = tf.get_variable(name='b_conv2d',
shape=(shape[-1]),
initializer=b_init,
dtype=LayersConfig.tf_dtype,
**self.b_init_args)
self.outputs = tf.nn.bias_add(self.outputs, b, name='bias_add')
self.outputs = self._apply_activation(self.outputs)
self._add_layers(self.outputs)
if b_init:
self._add_params([W, b])
else:
self._add_params(W)
def __init__(
self,
prev_layer,
n_filter=32,
filter_size=(3, 3),
strides=(1, 1),
padding='SAME',
act=None,
decay=0.9,
epsilon=1e-5,
is_train=False,
gamma_init=tf.compat.v1.initializers.ones,
beta_init=tf.compat.v1.initializers.zeros,
bitW=8,
bitA=8,
use_gemm=False,
W_init=tf.compat.v1.initializers.truncated_normal(stddev=0.02),
W_init_args=None,
use_cudnn_on_gpu=None,
data_format=None,
name='quan_cnn2d_bn',
):
super(QuanConv2dWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name)
logging.info(
"QuanConv2dWithBN %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s " % (
self.name, n_filter, filter_size, str(strides), padding,
self.act.__name__ if self.act is not None else 'No Activation'
)
)
x = self.inputs
self.inputs = quantize_active_overflow(self.inputs, bitA) # Do not remove
if use_gemm:
raise Exception("TODO. The current version use tf.matmul for inferencing.")
if len(strides) != 2:
raise ValueError("len(strides) should be 2.")
try:
pre_channel = int(prev_layer.outputs.get_shape()[-1])
except Exception: # if pre_channel is ?, it happens when using Spatial Transformer Net
pre_channel = 1
logging.warning("[warnings] unknow input channels, set to 1")
shape = (filter_size[0], filter_size[1], pre_channel, n_filter)
strides = (1, strides[0], strides[1], 1)
with tf.compat.v1.variable_scope(name):
W = tf.compat.v1.get_variable(
name='W_conv2d', shape=shape, initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args
)
conv = tf.nn.conv2d(
x, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format
)
para_bn_shape = conv.get_shape()[-1:]
if gamma_init:
scale_para = tf.compat.v1.get_variable(
name='scale_para', shape=para_bn_shape, initializer=gamma_init, dtype=LayersConfig.tf_dtype,
trainable=is_train
)
else:
scale_para = None
if beta_init:
offset_para = tf.compat.v1.get_variable(
name='offset_para', shape=para_bn_shape, initializer=beta_init, dtype=LayersConfig.tf_dtype,
trainable=is_train
)
else:
offset_para = None
moving_mean = tf.compat.v1.get_variable(
'moving_mean', para_bn_shape, initializer=tf.compat.v1.initializers.constant(1.),
dtype=LayersConfig.tf_dtype, trainable=False
)
moving_variance = tf.compat.v1.get_variable(
'moving_variance',
para_bn_shape,
initializer=tf.compat.v1.initializers.constant(1.),
dtype=LayersConfig.tf_dtype,
trainable=False,
)
mean, variance = tf.nn.moments(x=conv, axes=list(range(len(conv.get_shape()) - 1)))
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay, zero_debias=False
) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay, zero_debias=False
) # if zero_debias=True, has bias
def mean_var_with_update():
with tf.control_dependencies([update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
if is_train:
mean, var = mean_var_with_update()
else:
mean, var = moving_mean, moving_variance
w_fold = _w_fold(W, scale_para, var, epsilon)
bias_fold = _bias_fold(offset_para, scale_para, mean, var, epsilon)
W = quantize_weight_overflow(w_fold, bitW)
conv_fold = tf.nn.conv2d(
self.inputs, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format
)
self.outputs = tf.nn.bias_add(conv_fold, bias_fold, name='bn_bias_add')
self.outputs = self._apply_activation(self.outputs)
self._add_layers(self.outputs)
self._add_params([W, scale_para, offset_para, moving_mean, moving_variance])
def __init__(
self,
prev_layer,
n_units=100,
act=None,
decay=0.9,
epsilon=1e-5,
is_train=False,
bitW=8,
bitA=8,
gamma_init=tf.compat.v1.initializers.ones,
beta_init=tf.compat.v1.initializers.zeros,
use_gemm=False,
W_init=tf.compat.v1.initializers.truncated_normal(stddev=0.1),
W_init_args=None,
name=None, #'quan_dense_with_bn',
):
super(QuanDenseLayerWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name)
logging.info(
"QuanDenseLayerWithBN %s: %d %s" %
(self.name, n_units, self.act.__name__ if self.act is not None else 'No Activation')
)
if self.inputs.get_shape().ndims != 2:
raise Exception("The input dimension must be rank 2, please reshape or flatten it")
if use_gemm:
raise Exception("TODO. The current version use tf.matmul for inferencing.")
n_in = int(self.inputs.get_shape()[-1])
x = self.inputs
self.inputs = quantize_active_overflow(self.inputs, bitA)
self.n_units = n_units
with tf.compat.v1.variable_scope(name):
W = tf.compat.v1.get_variable(
name='W', shape=(n_in, n_units), initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args
)
mid_out = tf.matmul(x, W)
para_bn_shape = mid_out.get_shape()[-1:]
if gamma_init:
scale_para = tf.compat.v1.get_variable(
name='scale_para', shape=para_bn_shape, initializer=gamma_init, dtype=LayersConfig.tf_dtype,
trainable=is_train
)
else:
scale_para = None
if beta_init:
offset_para = tf.compat.v1.get_variable(
name='offset_para', shape=para_bn_shape, initializer=beta_init, dtype=LayersConfig.tf_dtype,
trainable=is_train
)
else:
offset_para = None
moving_mean = tf.compat.v1.get_variable(
'moving_mean', para_bn_shape, initializer=tf.compat.v1.initializers.constant(1.),
dtype=LayersConfig.tf_dtype, trainable=False
)
moving_variance = tf.compat.v1.get_variable(
'moving_variance',
para_bn_shape,
initializer=tf.compat.v1.initializers.constant(1.),
dtype=LayersConfig.tf_dtype,
trainable=False,
)
mean, variance = tf.nn.moments(x=mid_out, axes=list(range(len(mid_out.get_shape()) - 1)))
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay, zero_debias=False
) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay, zero_debias=False
) # if zero_debias=True, has bias
def mean_var_with_update():
with tf.control_dependencies([update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
if is_train:
mean, var = mean_var_with_update()
else:
mean, var = moving_mean, moving_variance
w_fold = _w_fold(W, scale_para, var, epsilon)
bias_fold = _bias_fold(offset_para, scale_para, mean, var, epsilon)
W = quantize_weight_overflow(w_fold, bitW)
# W = tl.act.sign(W) # dont update ...
# W = tf.Variable(W)
self.outputs = tf.matmul(self.inputs, W)
# self.outputs = xnor_gemm(self.inputs, W) # TODO
self.outputs = tf.nn.bias_add(self.outputs, bias_fold, name='bias_add')
self.outputs = self._apply_activation(self.outputs)
self._add_layers(self.outputs)
self._add_params([W, scale_para, offset_para, moving_mean, moving_variance])
