def _compute_fraction_of_zero(variables):
"""Given a linear variables list, compute the fraction of zero weights.
Args:
variables: A list or list of list of variables
Returns:
The fraction of zeros (sparsity) in the linear model.
"""
with ops.name_scope('zero_fraction'):
variables = nest.flatten(variables)
with ops.name_scope('total_size'):
sizes = [array_ops.size(x, out_type=dtypes.int64) for x in variables]
total_size_int64 = math_ops.add_n(sizes)
with ops.name_scope('total_zero'):
total_zero_float32 = math_ops.add_n([
control_flow_ops.cond(
math_ops.equal(size, constant_op.constant(0, dtype=dtypes.int64)),
true_fn=lambda: constant_op.constant(0, dtype=dtypes.float32),
false_fn=lambda: nn.zero_fraction(x) * math_ops.to_float(size),
name='zero_count')
for x, size in zip(variables, sizes)
])
with ops.name_scope('compute'):
total_size_float32 = math_ops.cast(
total_size_int64, dtype=dtypes.float32, name='float32_size')
zero_fraction_or_nan = total_zero_float32 / total_size_float32
zero_fraction_or_nan = array_ops.identity(
zero_fraction_or_nan, name='zero_fraction_or_nan')
return zero_fraction_or_nan
def _compute_fraction_of_zero(cols_to_vars):
all_weight_vars = []
for var_or_var_list in cols_to_vars.values():
# Skip empty-lists associated with columns that created no Variables.
if var_or_var_list:
all_weight_vars += [
array_ops.reshape(var, [-1]) for var in var_or_var_list
]
return nn.zero_fraction(array_ops.concat(all_weight_vars, axis=0))
def testZeroFraction(self):
x_shape = [5, 17]
x_np = np.random.randint(0, 2, size=x_shape).astype(np.float32)
y_np = self._ZeroFraction(x_np)
with self.test_session():
x_tf = constant_op.constant(x_np)
x_tf.set_shape(x_shape)
y_tf = nn.zero_fraction(x_tf)
y_tf_np = y_tf.eval()
eps = 1e-8
self.assertAllClose(y_tf_np, y_np, eps)
def testZeroFraction(self):
x_shape = [5, 17]
x_np = np.random.randint(0, 2, size=x_shape).astype(np.float32)
y_np = self._ZeroFraction(x_np)
with self.test_session():
x_tf = constant_op.constant(x_np)
x_tf.set_shape(x_shape)
y_tf = nn.zero_fraction(x_tf)
y_tf_np = y_tf.eval()
eps = 1e-8
self.assertAllClose(y_tf_np, y_np, eps)
def wide_net_old(wide_input, mode):
with tf.variable_scope('wide_model',
values=(wide_input, )) as dnn_hidden_scope:
wide_logits = tf.layers.dense(wide_input, 1, activation=None)
if mode == "train":
tf.summary.scalar(
"%s/fraction_of_zero_values" % dnn_hidden_scope.name,
nn.zero_fraction(wide_logits))
tf.summary.histogram("%s/activation" % dnn_hidden_scope.name,
wide_logits)
return wide_logits
def add_hidden_layer_summary(activation, name, weight=None):
'''
create the summary of the inout value
usefull for debugging
:param activation: tensor of the activation
:param weight: tensor of the weight
:param name: name of the tensor
:return: zero_fraction, histogram, norm
'''
tf.summary.scalar("%s_fraction_of_zero_values" % name, nn.zero_fraction(activation))
tf.summary.histogram("%s_activation" % name, activation)
if weight is not None:
_norm_summary(weight=weight, name=name)
def _compute_fraction_of_zero(variables):
"""Given a linear variables list, compute the fraction of zero weights.
Args:
variables: A list or list of list of variables
Returns:
The fraction of zeros (sparsity) in the linear model.
"""
all_weight_vars = []
for var_or_var_list in variables:
var_list = nest.flatten(var_or_var_list)
# Skip empty-lists associated with columns that created no Variables.
if var_list:
all_weight_vars += [array_ops.reshape(var, [-1]) for var in var_list]
return nn.zero_fraction(array_ops.concat(all_weight_vars, axis=0))
def _compute_fraction_of_zero(cols_to_vars):
"""Given a linear cols_to_vars dict, compute the fraction of zero weights.
Args:
cols_to_vars: A dictionary mapping FeatureColumns to lists of tf.Variables
like one returned from feature_column_lib.linear_model.
Returns:
The fraction of zeros (sparsity) in the linear model.
"""
all_weight_vars = []
for var_or_var_list in cols_to_vars.values():
# Skip empty-lists associated with columns that created no Variables.
if var_or_var_list:
all_weight_vars += [
array_ops.reshape(var, [-1]) for var in var_or_var_list
]
return nn.zero_fraction(array_ops.concat(all_weight_vars, axis=0))
def _compute_fraction_of_zero(variables):
"""Given a linear variables list, compute the fraction of zero weights.
Args:
variables: A list or list of list of variables
Returns:
The fraction of zeros (sparsity) in the linear model.
"""
all_weight_vars = []
for var_or_var_list in variables:
var_list = nest.flatten(var_or_var_list)
# Skip empty-lists associated with columns that created no Variables.
if var_list:
all_weight_vars += [
array_ops.reshape(var, [-1]) for var in var_list
]
return nn.zero_fraction(array_ops.concat(all_weight_vars, axis=0))
def wide_net(wide_input, w_number, mode):
with tf.variable_scope('wide_model',
values=(wide_input, )) as dnn_hidden_scope:
embeddings = tf.Variable(
tf.truncated_normal((w_number, ),
mean=0.0,
stddev=1.0,
dtype=tf.float32,
seed=None,
name="wide_init_w"))
bias = tf.Variable(tf.random_normal((1, )))
wide_logits = tf.nn.embedding_lookup_sparse(
embeddings, wide_input, None, combiner="sum") + bias
if mode == "train":
tf.summary.scalar(
"%s/fraction_of_zero_values" % dnn_hidden_scope.name,
nn.zero_fraction(wide_logits))
tf.summary.histogram("%s/activation" % dnn_hidden_scope.name,
wide_logits)
return tf.reshape(wide_logits, [-1, 1])
def _add_layer_summary(value, tag):
summary.scalar('%s/fraction_of_zero_values' % tag, nn.zero_fraction(value))
summary.histogram('%s/activation' % tag, value)
def _add_hidden_layer_summary(self, value, tag):
# TODO(zakaria): Move this code to tf.learn and add test.
logging_ops.scalar_summary("%s:fraction_of_zero_values" % tag, nn.zero_fraction(value))
logging_ops.histogram_summary("%s:activation" % tag, value)
def _add_hidden_layer_summary(value, tag):
summary.scalar("%s_fraction_of_zero_values" % tag, nn.zero_fraction(value))
summary.histogram("%s_activation" % tag, value)
def _add_hidden_layer_summary(value, tag):
logging_ops.scalar_summary("%s/fraction_of_zero_values" % tag,
nn.zero_fraction(value))
logging_ops.histogram_summary("%s/activation" % tag, value)
def _add_hidden_layer_summary(value, tag):
summary.scalar('%s/fraction_of_zero_values' % tag, nn.zero_fraction(value))
summary.histogram('%s/activation' % tag, value)
def _add_hidden_layer_summary(value, tag):
logging_ops.scalar_summary("%s/fraction_of_zero_values" % tag,
nn.zero_fraction(value))
logging_ops.histogram_summary("%s/activation" % tag, value)
def deep_net(deep_input, mode):
'''
deep nets
'''
with tf.variable_scope('deep_model',
values=(deep_input, )) as dnn_hidden_scope:
with tf.variable_scope('dnn_hidden_1',
values=(deep_input, )) as dnn_hidden_scope:
deep_hidden_1 = tf.layers.dense(deep_input,
1024,
activation=nn.relu)
deep_hidden_1 = tf.layers.dropout(deep_hidden_1,
rate=0.9,
training=mode == "train")
if mode == "train":
tf.summary.scalar(
"%s/fraction_of_zero_values" % dnn_hidden_scope.name,
nn.zero_fraction(deep_hidden_1))
tf.summary.histogram("%s/activation" % dnn_hidden_scope.name,
deep_hidden_1)
with tf.variable_scope('dnn_hidden_2',
values=(deep_hidden_1, )) as dnn_hidden_scope:
#deep_hidden_2 = tf.contrib.layers.fully_connected(
# deep_hidden_1, 512, activation_fn = nn.relu, variables_collections = ['deep'])
deep_hidden_2 = tf.layers.dense(deep_hidden_1,
512,
activation=nn.relu)
deep_hidden_2 = tf.layers.dropout(deep_hidden_2,
rate=0.9,
training=mode == "train")
if mode == "train":
tf.summary.scalar(
"%s/fraction_of_zero_values" % dnn_hidden_scope.name,
nn.zero_fraction(deep_hidden_2))
tf.summary.histogram("%s/activation" % dnn_hidden_scope.name,
deep_hidden_2)
# deep_hidden_2_merge = tf.concat([deep_hidden_2, deep_input_psid], 1)
with tf.variable_scope('dnn_hidden_3',
values=(deep_hidden_2, )) as dnn_hidden_scope:
deep_hidden_3 = tf.layers.dense(deep_hidden_2,
256,
activation=nn.relu)
deep_hidden_3 = tf.layers.dropout(deep_hidden_3,
rate=0.9,
training=mode == "train")
if mode == "train":
tf.summary.scalar(
"%s/fraction_of_zero_values" % dnn_hidden_scope.name,
nn.zero_fraction(deep_hidden_3))
tf.summary.histogram("%s/activation" % dnn_hidden_scope.name,
deep_hidden_3)
#deep_hidden_3_con = tf.concat([deep_hidden_3,deep_input],1) # high way
#hash_value = tf.string_to_hash_bucket_fast(psid_value[:,0],42)
#psid_value = tf.one_hot(hash_value, 42)
#deep_hidden_3_con = tf.concat([deep_hidden_3,psid_value],1)
with tf.variable_scope('dnn_logits',
values=(deep_hidden_3, )) as dnn_logits_scope:
deep_logits = tf.layers.dense(deep_hidden_3,
1,
activation=None,
bias_initializer=None)
if mode == "train":
tf.summary.scalar(
"%s/fraction_of_zero_values" % dnn_logits_scope.name,
nn.zero_fraction(deep_logits))
tf.summary.histogram("%s/activation" % dnn_logits_scope.name,
deep_logits)
return deep_logits
def _add_hidden_layer_summary(self, value, tag):
# TODO(zakaria): Move this code to tf.learn and add test.
logging_ops.scalar_summary("%s:fraction_of_zero_values" % tag,
nn.zero_fraction(value))
logging_ops.histogram_summary("%s:activation" % tag, value)
def testZeroFractionEmpty(self):
with self.test_session():
x = np.zeros(0)
y = nn.zero_fraction(x).eval()
self.assertTrue(np.isnan(y))
def _add_hidden_layer_summary(self, value, tag):
# TODO(zakaria): Move this code to tf.learn and add test.
summary.scalar("%s/fraction_of_zero_values" % tag, nn.zero_fraction(value))
summary.histogram("%s/activation" % tag, value)
def add_hidden_layer_summary(value, tag):
tf.summary.scalar("%s/fraction_of_zero_values" % tag,
nn.zero_fraction(value))
tf.summary.histogram("%s/activation" % tag, value)
def testZeroFractionEmpty(self):
with self.test_session():
x = np.zeros(0)
y = nn.zero_fraction(x).eval()
self.assertTrue(np.isnan(y))
def _add_layer_summary(value, tag):
summary.scalar("%s/fraction_of_zero_values" % tag, nn.zero_fraction(value))
summary.histogram("%s/activation" % tag, value)
def _add_hidden_layer_summary(self, value, tag):
# TODO (zakaria): Move this code to tf.learn and add test. id:721 gh:722
summary.scalar("%s/fraction_of_zero_values" % tag,
nn.zero_fraction(value))
summary.histogram("%s/activation" % tag, value)
