def _transform(self,
sign,
exponent,
mantissa,
exponent_width=None,
mantissa_width=None,
exponent_bias=None):
if exponent_bias is None:
exponent_bias = self.exponent_bias
if exponent_width is None:
exponent_width = self.width - self.mantissa_width
if mantissa_width is None:
mantissa_width = self.mantissa_width
# clip exponent and quantize mantissa
exponent_min = -exponent_bias
exponent_max = 2**exponent_width - 1 - exponent_bias
exponent = util.clip_by_value(exponent, exponent_min, exponent_max)
shift = util.cast(2**mantissa_width, float)
# quantize
if self.stochastic:
mantissa = util.stochastic_round(mantissa * shift, self.stochastic)
mantissa /= shift
else:
mantissa = util.round(mantissa * shift) / shift
# if the mantissa value gets rounded to >= 2 then we need to divide it
# by 2 and increment exponent by 1
is_out_of_range = util.greater_equal(mantissa, 2)
mantissa = util.where(is_out_of_range, mantissa / 2, mantissa)
exponent = util.where(is_out_of_range, exponent + 1, exponent)
return sign, exponent, mantissa
def _policy(self, value, quantized, previous_mask, interval):
previous_pruned = util.sum(previous_mask)
if self.count_zero:
th_arg = util.cast(util.count(value) * interval, int)
else:
tmp = util.count(value[value != 0])
flat_value_arg = util.where(value.flatten() != 0)
th_arg = util.cast(tmp * interval, int)
if th_arg < 0:
raise ValueError('mask has {} elements, interval is {}'.format(
previous_pruned, interval))
off_mask = util.cast(util.logical_not(util.cast(previous_mask, bool)),
float)
metric = value - quantized
flat_value = (metric * off_mask).flatten()
if interval >= 1.0:
th = flat_value.max() + 1.0
else:
if self.count_zero:
th = util.top_k(util.abs(flat_value), th_arg)
else:
th = util.top_k(util.abs(flat_value[flat_value_arg]), th_arg)
th = util.cast(th, float)
new_mask = util.logical_not(util.greater_equal(util.abs(metric), th))
return util.logical_or(new_mask, previous_mask)
def _update(self):
# update positives mask and mean values
value = self.session.run(self.before)
# divide them into two groups
# mean = util.mean(value)
mean = 0.0
# find two central points
positives = value > mean
self.positives = positives
self.positives_mean = util.mean(value[util.where(positives)])
negatives = util.logical_and(util.logical_not(positives), value != 0)
self.negatives_mean = util.mean(value[util.where(negatives)])
if self.positives_mean.eval() == 0 or self.negatives_mean.eval() == 0:
log.warn(
'means are skewed, pos mean is {} and neg mean is {}'.format(
self.positives_mean.eval(), self.negatives_mean.eval()))
# update internal quantizer
self.quantizer.update()
for quantizer in self.parameter_quantizers.values():
quantizer.update()
def _quantize(self, value, point, width, compute_overflow_rate=False):
# decompose
sign = util.cast(value > 0, float) - util.cast(value < 0, float)
value = util.log(util.abs(value), 2.0)
# quantize
value = self.quantizer.apply(
value, compute_overflow_rate=compute_overflow_rate)
if compute_overflow_rate:
return value
# represent
return util.where(util.nonzero(sign), sign * (2**value), 0)
def _represent(self, sign, exponent, mantissa):
"""
Represent the value in floating-point using
sign, exponent and mantissa.
"""
value = util.cast(sign, float) * (2.0**exponent) * mantissa
if util.is_constant(sign, exponent, mantissa):
return value
if util.is_numpy(sign, exponent, mantissa):
zeros = np.zeros(sign.shape, dtype=np.int32)
else:
zeros = tf.zeros(sign.shape, dtype=tf.int32)
is_zero = util.equal(sign, zeros)
return util.where(is_zero, util.cast(zeros, float), value)
