def summarize_activations(): with tf.variable_scope('activations'): for l in tf.get_collection('conv_layers'): tf.summary.histogram(tensor_name(l), l) tf.summary.scalar(tensor_name(l) + '/sparsity', tf.nn.zero_fraction(l)) montage_summary(tf.transpose(l[0], [2, 0, 1]), name=tensor_name(l) + '/montage') for l in tf.get_collection('dense_layers'): tf.summary.histogram(tensor_name(l), l) tf.summary.scalar(tensor_name(l) + '/sparsity', tf.nn.zero_fraction(l))
def summarize_weights_biases(): with tf.variable_scope('weights'): for l in tf.get_collection('weights'): tf.summary.histogram(tensor_name(l), l) tf.summary.scalar(tensor_name(l) + '/sparsity', tf.nn.zero_fraction(l)) # montage_summary(l, name=tensor_name(l) + '/montage') with tf.variable_scope('biases'): for l in tf.get_collection('biases'): tf.summary.histogram(tensor_name(l), l) tf.summary.scalar(tensor_name(l) + '/sparsity', tf.nn.zero_fraction(l))
def __init__(self, session, config, model, variational, data): cfg = config self.config = config self.session = session self.data = data self.variational = variational self.model = model self.build_elbo(n_samples=cfg['q/n_samples_stats']) self.q_params = fw.get_variables('variational') self.global_step = fw.get_or_create_global_step() self.build_elbo(n_samples=cfg['q/n_samples'], training=True) self.build_elbo_loss() with tf.name_scope('q_neg_elbo_grad'): self.q_neg_elbo_grad = tf.gradients(self.differentiable_elbo_loss, self.q_params) self.q_neg_elbo_grad_norm = [ util.norm(g) for g in self.q_neg_elbo_grad ] for param, norm in zip(self.q_params, self.q_neg_elbo_grad_norm): tf.summary.scalar( 'o/neg_elbo_grad_norm_' + util.tensor_name(param), norm) self.p_params = fw.get_variables('model') self.build_optimizer() self.t0 = time.time() self.t = np.inf self.build_proximity_statistic_summaries() for param in self.q_params + self.p_params: self.build_summaries(param)
def build_q_gradients(self): cfg = self.config q_gradients = self.q_neg_elbo_grad constraint_grad = [0.] * len(self.q_params) magnitude = self.magnitude if cfg['c/decay'] == 'linear': magnitude = tf.maximum(magnitude, 0.) for name, distance in self.distance.items(): distance_grad = tf.gradients(distance, self.q_params) for i in range(len(self.q_params)): if distance_grad[i] is not None: param_name = util.tensor_name(self.q_params[i]) update = magnitude * distance_grad[i] constraint_grad[i] += update q_gradients[i] += update update_norm = util.norm(update) fraction = tf.reduce_mean( update_norm / util.norm(self.q_neg_elbo_grad[i] + update)) fraction = tf.Print(fraction, [fraction], 'fraction: ') tf.summary.scalar( '_'.join(['c/fraction_grad_d', name, param_name]), fraction) tf.summary.scalar( '_'.join(['c/norm_grad_constraint', name, param_name]), update_norm) tf.summary.scalar( '_'.join([ 'c/ratio_grad_constraint_grad_neg_elbo', name, param_name ]), update_norm / self.q_neg_elbo_grad_norm[i]) self.q_gradients = q_gradients self.q_constraint_grad = constraint_grad
def build_statistic(self): statistic = {} q_variables = fw.get_variables('variational') q_weights = [var for var in q_variables if 'weights' in var.name] for weight_matrix in q_weights: identity = tf.matmul(weight_matrix, weight_matrix, transpose_b=True) statistic[util.tensor_name(identity) + '_wwT'] = identity self._statistic = statistic
def summarize_gradients(grads_and_vars, name=None): """Adds histogram summaries for input list. Args: grads_and_vars: List, a list of tuples in form (grad, var).e name: String, name to use for var scope. """ with tf.name_scope(name, 'gradients', grads_and_vars): for g, v in grads_and_vars: tf.summary.histogram(tensor_name(v) + '/gradient', g)
def summarize_losses(): with tf.variable_scope('loss'): for l in tf.get_collection('losses'): tf.summary.scalar(tensor_name(l), l) tf.summary.histogram(tensor_name(l), l)
def build_summaries(self, param): base_name = util.tensor_name(param) tf.summary.scalar(base_name + '/mean', tf.reduce_mean(param)) tf.summary.scalar(base_name + '/max', tf.reduce_max(param)) tf.summary.scalar(base_name + '/min', tf.reduce_min(param)) tf.summary.histogram(base_name, param)