def _policy(self, value, quantized, previous_mask, interval): previous_pruned = util.sum(previous_mask) if self.count_zero: th_arg = util.cast(util.count(value) * interval, int) else: tmp = util.count(value[value != 0]) flat_value_arg = util.where(value.flatten() != 0) th_arg = util.cast(tmp * interval, int) if th_arg < 0: raise ValueError('mask has {} elements, interval is {}'.format( previous_pruned, interval)) off_mask = util.cast(util.logical_not(util.cast(previous_mask, bool)), float) metric = value - quantized flat_value = (metric * off_mask).flatten() if interval >= 1.0: th = flat_value.max() + 1.0 else: if self.count_zero: th = util.top_k(util.abs(flat_value), th_arg) else: th = util.top_k(util.abs(flat_value[flat_value_arg]), th_arg) th = util.cast(th, float) new_mask = util.logical_not(util.greater_equal(util.abs(metric), th)) return util.logical_or(new_mask, previous_mask)
def _updated_mask(self, var, mask): var, mask, alpha = self.session.run([var, mask, self.alpha]) threshold = self._threshold(var, alpha) on_mask = util.abs(var) > self.on_factor * threshold mask = util.logical_or(mask, on_mask) off_mask = util.abs(var) > self.off_factor * threshold return util.logical_and(mask, off_mask)
def _threshold(self, tensor, alpha=None): # axes = list(range(len(tensor.get_shape()) - 1)) tensor_shape = util.get_shape(tensor) axes = list(range(len(tensor_shape))) mean, var = util.moments(util.abs(tensor), axes) if alpha is None: return mean + self.alpha * util.sqrt(var) return mean + alpha * util.sqrt(var)
def _l1_norm(self, value): # compute l1 norm for each filter axes = len(value.shape) assert axes == 4 # mean, var = tf.nn.moments(util.abs(tensor), axes=[0, 1]) # mean = np.mean(value, axis=(0, 1)) # var = np.var(value, axis=(0, 1)) # return mean + util.sqrt(var) return util.sum(util.abs(value), axis=(0, 1))
def _quantize(self, value, point, width, compute_overflow_rate=False): # decompose sign = util.cast(value > 0, float) - util.cast(value < 0, float) value = util.log(util.abs(value), 2.0) # quantize value = self.quantizer.apply( value, compute_overflow_rate=compute_overflow_rate) if compute_overflow_rate: return value # represent return util.where(util.nonzero(sign), sign * (2**value), 0)
def _new_mask(self, mask, value, quantized_value, interval): loss = util.abs(value - quantized_value) # check the ones that are not quantized mask = mask.reshape((1, 1, 1, mask.shape[0])) unquantized_mask = util.logical_not(mask) # TODO: mask shape is incorrect loss_vec = util.mean(loss * unquantized_mask, (0, 1, 2)) # sort num_active = util.ceil(len(loss_vec) * interval) threshold = sorted(loss_vec)[num_active] if interval >= 1.0: return util.cast(unquantized_mask, float) new_mask = (unquantized_mask * loss) > threshold return util.cast(util.logical_or(new_mask, mask), float)
def _decompose(self, value, exponent_bias=None): """ Decompose a single-precision floating-point into sign, exponent and mantissa components. """ if exponent_bias is None: exponent_bias = self.exponent_bias # smallest non-zero floating point descriminator = (2**(-exponent_bias)) / 2 sign = util.cast(value > descriminator, int) sign -= util.cast(value < -descriminator, int) value = util.abs(value) exponent = util.floor(util.log(value, 2)) mantissa = value / (2**exponent) return sign, exponent, mantissa
def _updated_mask(self, var, mask): return util.abs(var) > self._threshold(var)