def _policy(self, value, quantized, previous_mask, interval):
previous_pruned = util.sum(previous_mask)
if self.count_zero:
th_arg = util.cast(util.count(value) * interval, int)
else:
tmp = util.count(value[value != 0])
flat_value_arg = util.where(value.flatten() != 0)
th_arg = util.cast(tmp * interval, int)
if th_arg < 0:
raise ValueError('mask has {} elements, interval is {}'.format(
previous_pruned, interval))
off_mask = util.cast(util.logical_not(util.cast(previous_mask, bool)),
float)
metric = value - quantized
flat_value = (metric * off_mask).flatten()
if interval >= 1.0:
th = flat_value.max() + 1.0
else:
if self.count_zero:
th = util.top_k(util.abs(flat_value), th_arg)
else:
th = util.top_k(util.abs(flat_value[flat_value_arg]), th_arg)
th = util.cast(th, float)
new_mask = util.logical_not(util.greater_equal(util.abs(metric), th))
return util.logical_or(new_mask, previous_mask)
def _updated_mask(self, var, mask):
var, mask, alpha = self.session.run([var, mask, self.alpha])
threshold = self._threshold(var, alpha)
on_mask = util.abs(var) > self.on_factor * threshold
mask = util.logical_or(mask, on_mask)
off_mask = util.abs(var) > self.off_factor * threshold
return util.logical_and(mask, off_mask)
def _threshold(self, tensor, alpha=None):
# axes = list(range(len(tensor.get_shape()) - 1))
tensor_shape = util.get_shape(tensor)
axes = list(range(len(tensor_shape)))
mean, var = util.moments(util.abs(tensor), axes)
if alpha is None:
return mean + self.alpha * util.sqrt(var)
return mean + alpha * util.sqrt(var)
def _l1_norm(self, value):
# compute l1 norm for each filter
axes = len(value.shape)
assert axes == 4
# mean, var = tf.nn.moments(util.abs(tensor), axes=[0, 1])
# mean = np.mean(value, axis=(0, 1))
# var = np.var(value, axis=(0, 1))
# return mean + util.sqrt(var)
return util.sum(util.abs(value), axis=(0, 1))
def _quantize(self, value, point, width, compute_overflow_rate=False):
# decompose
sign = util.cast(value > 0, float) - util.cast(value < 0, float)
value = util.log(util.abs(value), 2.0)
# quantize
value = self.quantizer.apply(
value, compute_overflow_rate=compute_overflow_rate)
if compute_overflow_rate:
return value
# represent
return util.where(util.nonzero(sign), sign * (2**value), 0)
def _new_mask(self, mask, value, quantized_value, interval):
loss = util.abs(value - quantized_value)
# check the ones that are not quantized
mask = mask.reshape((1, 1, 1, mask.shape[0]))
unquantized_mask = util.logical_not(mask)
# TODO: mask shape is incorrect
loss_vec = util.mean(loss * unquantized_mask, (0, 1, 2))
# sort
num_active = util.ceil(len(loss_vec) * interval)
threshold = sorted(loss_vec)[num_active]
if interval >= 1.0:
return util.cast(unquantized_mask, float)
new_mask = (unquantized_mask * loss) > threshold
return util.cast(util.logical_or(new_mask, mask), float)
def _decompose(self, value, exponent_bias=None):
"""
Decompose a single-precision floating-point into
sign, exponent and mantissa components.
"""
if exponent_bias is None:
exponent_bias = self.exponent_bias
# smallest non-zero floating point
descriminator = (2**(-exponent_bias)) / 2
sign = util.cast(value > descriminator, int)
sign -= util.cast(value < -descriminator, int)
value = util.abs(value)
exponent = util.floor(util.log(value, 2))
mantissa = value / (2**exponent)
return sign, exponent, mantissa
def _updated_mask(self, var, mask):
return util.abs(var) > self._threshold(var)
