def __init__(self, params):
super(IdentityRegressionTask, self).__init__(params)
with tf.variable_scope('IdentityRegressionTask'):
self.CreateVariable(
'm',
py_utils.WeightParams(shape=[],
init=py_utils.WeightInit.Uniform()))
self.CreateVariable(
'b',
py_utils.WeightParams(shape=[],
init=py_utils.WeightInit.Uniform()))
self.global_steps = []
self.metrics = []
self.result_per_example_tensors = []
def CreateTensor(self, t_name):
p = self.params
assert t_name not in self._t_names, ('QTensor already registered: %s' %
t_name)
self._t_names.add(t_name)
# Create accumulator
accumulator_name = self._GetAccumulatorNameForTensor(t_name)
self.RegisterAccumulator(accumulator_name,
_CountedMinMaxAccumulator(p.dtype))
# Register vars.
min_pc = py_utils.WeightParams(
(), py_utils.WeightInit.Constant(p.default_min), p.dtype)
max_pc = py_utils.WeightParams(
(), py_utils.WeightInit.Constant(p.default_max), p.dtype)
self._CreateQStateVar(t_name, 'min', min_pc)
self._CreateQStateVar(t_name, 'max', max_pc)
def __init__(self, name):
self._name = name
_, self._var = py_utils.CreateVariable(
name=name,
params=py_utils.WeightParams([], py_utils.WeightInit.Constant(0),
tf.int64),
trainable=False)
self._value = self._var.value() + 0 # Makes a copy.
def __init__(self, params):
super(BatchNormLayerNoPadding, self).__init__(params)
p = self.params
assert p.name, 'Name of BatchNormLayerNoPadding is not set.'
p.fprop_dtype = None
# Skip L-P regularization for these variables.
collections = [
self.__class__.__name__ + '_vars', py_utils.SKIP_LP_REGULARIZATION
]
pc = py_utils.WeightParams(shape=[p.dim],
init=py_utils.WeightInit.Constant(0.0),
dtype=p.dtype,
collections=collections)
with tf.variable_scope(p.name):
self.CreateVariable('beta', pc)
# Note, The real gamma to use is 1 + gamma.
self.CreateVariable('gamma', pc, lambda x: 1.0 + x)
moving_collections = [
'moving_vars', tf.GraphKeys.MOVING_AVERAGE_VARIABLES,
self.__class__.__name__ + '_vars'
]
mva = py_utils.WeightParams(shape=[p.dim],
init=py_utils.WeightInit.Constant(0.0),
dtype=p.dtype,
collections=moving_collections)
# Two statistics computed from sufficient stats.
self.CreateVariable('moving_mean', mva, trainable=False)
mvv = py_utils.WeightParams(shape=[p.dim],
init=py_utils.WeightInit.Constant(1.0),
dtype=p.dtype,
collections=moving_collections)
self.CreateVariable('moving_variance', mvv, trainable=False)
# Accumulate bn sufficient stats over micro-batches.
dim = self.vars.beta.shape[0]
self.RegisterAccumulator('counts', AddingAccumulator([], p.dtype))
self.RegisterAccumulator('mean_ss', AddingAccumulator([dim], p.dtype))
self.RegisterAccumulator('variance_ss',
AddingAccumulator([dim], p.dtype))
def __init__(self, params):
super(BiasLayer, self).__init__(params)
p = self.params
with tf.variable_scope(p.name):
self.CreateVariable(
'b',
py_utils.WeightParams(
shape=[p.dims],
init=py_utils.WeightInit.Constant(0.0),
dtype=p.dtype,
collections=[self.__class__.__name__ + '_vars']))
def __init__(self, params):
super(LinearLayer, self).__init__(params)
p = self.params
with tf.variable_scope(p.name):
self.CreateVariable(
'w',
py_utils.WeightParams(
shape=[p.input_dims, p.output_dims],
init=p.params_init,
dtype=p.dtype,
collections=[self.__class__.__name__ + '_vars']))
def _Acc(vg):
"""Updating accumulators."""
v, g = vg
with tf.variable_scope(v.op.name):
_, a = py_utils.CreateVariable(
'grad_accumulator',
py_utils.WeightParams(v.get_shape(),
py_utils.WeightInit.Constant(0.0),
self.params.dtype),
trainable=False)
a = tf.assign_add(a, g)
return py_utils.VarGrad(v, a)
def __init__(self, params):
super(DevBasedSchedule, self).__init__(params)
p = self.params
with tf.variable_scope(p.name):
wp = py_utils.WeightParams(shape=[],
init=py_utils.WeightInit.Constant(1.0),
collections=['DevBasedSchedule_vars'],
dtype=tf.float32)
_, self._cur_factor, = py_utils.CreateVariable('cur_factor',
wp,
trainable=False)
wp = py_utils.WeightParams(shape=[],
init=py_utils.WeightInit.Constant(0),
collections=['DevBasedSchedule_vars'],
dtype=tf.int64)
_, self._ref_step, = py_utils.CreateVariable('ref_step',
wp,
trainable=False)
self._metric_history = early_stop.MetricHistory(p.metric_history)
self._best_step = ops.best_step(self._metric_history.hist_file,
p.tolerance)
def __init__(self, params):
super(DeterministicWeightsLayer, self).__init__(params)
p = self.params
if not p.name:
raise ValueError('Layer must have a specified name!')
assert p.num_sources > 0, ('Must specify num_sources > 0.')
params_init = py_utils.WeightInit.Constant(0.0)
# Weights to be learned.
pw = py_utils.WeightParams(
shape=[p.num_sources],
init=params_init,
dtype=p.dtype,
collections=[self.__class__.__name__ + '_vars'])
with tf.variable_scope(p.name):
self.CreateVariable('sum_weight', pw)
p.dropout_tpl.name = 'dropout'
self.CreateChild('weighted_merger_dropout', p.dropout_tpl)
def __init__(self, params):
super(SoftCondLayer, self).__init__(params)
p = self.params
assert p.name
assert p.num_experts
assert p.cond_dim
with tf.variable_scope(p.name):
# Create Variables for task weight mapping.
collections = [
self.__class__.__name__ + '_vars',
]
w_p = py_utils.WeightParams(
shape=[p.cond_dim, p.num_experts],
init=p.params_init, # TODO(huangyp): try zero init instead.
dtype=p.dtype,
collections=collections)
self.CreateVariable('w', w_p)
# Prepends p.num_experts to the tensor shape of every variable created
# by p.body.
with py_utils.VariableShapePrefixContext(p.num_experts):
self.CreateChild('body', p.body)