def _embedded_lattices(calibrated_input_tensor,
input_dim,
output_dim,
interpolation_type,
monotonic_num_lattices,
monotonic_lattice_rank,
monotonic_lattice_size,
non_monotonic_num_lattices,
non_monotonic_lattice_rank,
non_monotonic_lattice_size,
linear_embedding_calibration_min,
linear_embedding_calibration_max,
linear_embedding_calibration_num_keypoints,
is_monotone=None,
lattice_l1_reg=None,
lattice_l2_reg=None,
lattice_l1_torsion_reg=None,
lattice_l2_torsion_reg=None,
lattice_l1_laplacian_reg=None,
lattice_l2_laplacian_reg=None):
"""Creates an ensemble of lattices with a linear embedding.
This function constructs the following deep lattice network:
calibrated_input -> linear_embedding -> calibration -> ensemble of lattices.
Then ensemble of lattices' output are averaged and bias term is added to make
a final prediction.
ensemble of lattices is consists of two parts: monotonic lattices and
non-monotonic lattices. The input to the monotonic lattices is an output of
linear_embedding that contains both monotonic and non-monotonic
calibrated_input. All inputs to the monotonic lattices are set to be monotonic
to preserve end-to-end monotonicity in the monotonic feature.
The input to the non-monotonic lattices is an output of linear_embedding that
only contains non-monotonic calibrated_input. All inputs to the non-monotonic
lattices are set to be non-monotonic, since we do not need to guarantee
monotonicity.
Args:
calibrated_input_tensor: [batch_size, input_dim] tensor.
input_dim: (int) input dimnension.
output_dim: (int) output dimension.
interpolation_type: defines whether the lattice will interpolate using the
full hypercube or only the simplex ("hyper-triangle") around the point
being evaluated. Valid values: 'hypercube' or 'simplex'
monotonic_num_lattices: (int) number of monotonic lattices in the ensemble
lattices layer.
monotonic_lattice_rank: (int) number of inputs to each monotonic lattice in
the ensemble lattices layer.
monotonic_lattice_size: (int) lattice cell size for each monotonic lattice
in the ensemble lattices layer.
non_monotonic_num_lattices: (int) number of non monotonic lattices in the
ensemble lattices layer.
non_monotonic_lattice_rank: (int) number of inputs to each non monotonic
lattice in the ensemble lattices layer.
non_monotonic_lattice_size: (int) lattice cell size for each non monotonic
lattice in the ensemble lattices layer.
linear_embedding_calibration_min: (float) a minimum input keypoints value
for linear_embedding calibration.
linear_embedding_calibration_max: (float) a maximum input keypoints value
for linear_embedding calibration.
linear_embedding_calibration_num_keypoints: (int) a number of eypoints for
linear_embedding calibration.
is_monotone: (bool, list of booleans) is_monotone[k] == true then
calibrated_input_tensor[:, k] is considered to be a monotonic input.
lattice_l1_reg: (float) lattice l1 regularization amount.
lattice_l2_reg: (float) lattice l2 regularization amount.
lattice_l1_torsion_reg: (float) lattice l1 torsion regularization amount.
lattice_l2_torsion_reg: (float) lattice l2 torsion regularization amount.
lattice_l1_laplacian_reg: (float) lattice l1 laplacian regularization
amount.
lattice_l2_laplacian_reg: (float) lattice l2 laplacian regularization
amount.
Returns:
A tuple of (output_tensor, projection_ops, regularization).
Raises:
ValueError: If there is no non-monotonic inputs but
non_monotonic_num_lattices is not zero.
"""
projections = []
regularization = None
# Explictly assign number of lattices to zero for any empty cases.
if not monotonic_num_lattices:
monotonic_num_lattices = 0
if not non_monotonic_num_lattices:
non_monotonic_num_lattices = 0
# Step 1. Create a linear embedding.
if monotonic_num_lattices:
monotonic_embedding_dim = monotonic_num_lattices * monotonic_lattice_rank
else:
monotonic_num_lattices = 0
monotonic_embedding_dim = 0
if non_monotonic_num_lattices:
non_monotonic_embedding_dim = (non_monotonic_num_lattices *
non_monotonic_lattice_rank)
else:
non_monotonic_num_lattices = 0
non_monotonic_embedding_dim = 0
if is_monotone is not None:
is_monotone = tools.cast_to_list(is_monotone, input_dim, 'is_monotone')
with variable_scope.variable_scope('linear_embedding'):
packed_results = monotone_linear_layers.split_monotone_linear_layer(
calibrated_input_tensor,
input_dim,
monotonic_embedding_dim,
non_monotonic_embedding_dim,
is_monotone=is_monotone)
(monotonic_output, _, non_monotonic_output, _, proj,
_) = packed_results
if proj is not None:
projections.append(proj)
# Step 2. Create ensemble of monotonic lattices.
if monotonic_num_lattices == 0:
m_lattice_outputs = None
else:
with variable_scope.variable_scope('monotonic_lattices'):
m_lattice_outputs, projs, reg = _ensemble_lattices_layer(
monotonic_output,
monotonic_embedding_dim,
output_dim,
interpolation_type,
linear_embedding_calibration_min,
linear_embedding_calibration_max,
linear_embedding_calibration_num_keypoints,
monotonic_num_lattices,
monotonic_lattice_rank,
monotonic_lattice_size,
is_monotone=True,
l1_reg=lattice_l1_reg,
l2_reg=lattice_l2_reg,
l1_torsion_reg=lattice_l1_torsion_reg,
l2_torsion_reg=lattice_l2_torsion_reg,
l1_laplacian_reg=lattice_l1_laplacian_reg,
l2_laplacian_reg=lattice_l2_laplacian_reg)
if projs:
projections += projs
regularization = tools.add_if_not_none(regularization, reg)
# Step 3. Construct non-monotonic ensembles.
if non_monotonic_output is None and non_monotonic_num_lattices > 0:
raise ValueError(
'All input signals are monotonic but the number of non monotonic '
'lattices is not zero.')
if non_monotonic_num_lattices == 0:
n_lattice_outputs = None
else:
with variable_scope.variable_scope('non_monotonic_lattices'):
n_lattice_outputs, projs, reg = _ensemble_lattices_layer(
non_monotonic_output,
non_monotonic_embedding_dim,
output_dim,
interpolation_type,
linear_embedding_calibration_min,
linear_embedding_calibration_max,
linear_embedding_calibration_num_keypoints,
non_monotonic_num_lattices,
non_monotonic_lattice_rank,
non_monotonic_lattice_size,
is_monotone=False,
l1_reg=lattice_l1_reg,
l2_reg=lattice_l2_reg,
l1_torsion_reg=lattice_l1_torsion_reg,
l2_torsion_reg=lattice_l2_torsion_reg,
l1_laplacian_reg=lattice_l1_laplacian_reg,
l2_laplacian_reg=lattice_l2_laplacian_reg)
if projs:
projections += projs
regularization = tools.add_if_not_none(regularization, reg)
# Step 4. Take average to make a final prediction.
with variable_scope.variable_scope('ensemble_average'):
output = variable_scope.get_variable(
name='ensemble_bias',
initializer=[0.0] * output_dim,
dtype=calibrated_input_tensor.dtype)
if m_lattice_outputs:
output += math_ops.divide(math_ops.add_n(m_lattice_outputs),
monotonic_num_lattices)
if n_lattice_outputs is not None:
output += math_ops.divide(math_ops.add_n(n_lattice_outputs),
non_monotonic_num_lattices)
return (output, projections, regularization)
def calibration_layer(uncalibrated_tensor,
num_keypoints,
keypoints_initializers=None,
keypoints_initializer_fns=None,
bound=False,
monotonic=None,
missing_input_values=None,
missing_output_values=None,
name=None,
**regularizer_amounts):
"""Creates a calibration layer for uncalibrated values.
Returns a calibrated tensor of the same shape as the uncalibrated continuous
signals passed in, and a list of projection ops, that must be applied at
each step (or every so many steps) to project the model to a feasible space:
used for bounding the outputs or for imposing monotonicity -- the list will be
empty if bound and monotonic are not set.
Args:
uncalibrated_tensor: Tensor of shape [batch_size, ...] with uncalibrated
values.
num_keypoints: Number of keypoints to use. Either a scalar value that
will be used for every uncalibrated signal, or a list of n values,
per uncalibrated signal -- uncalibrated is first flattened (
see tf.contrib.layers.flatten) to [batch_size, n], and there should
be one value in the list per n. If a value of the list is 0 or None
the correspondent signal won't be calibrated.
keypoints_initializers: For evaluation or inference (or when resuming
training from a checkpoint) the values will be loaded from disk, so they
don't need to be given (leave it as None).
Otherwise provide either a tuple of two tensors of shape [num_keypoints],
or a list of n pairs of tensors, each of shape [num_keypoints]. In this
list there should be one pair per uncalibrated signal, just like
num_keypoints above. Notice that num_keypoints can be different per
signal.
keypoints_initializer_fns: Like keypoints_initializers but using lambda
initializers. They should be compatible with tf.get_variable. If this is
set, then keypoints_initializers must be None.
bound: boolean whether output of calibration must be bound. Alternatively
a list of n booleans, one per uncalibrated value, like num_keypoints
above.
monotonic: whether calibration is monotonic: None or 0 means no
monotonicity. Positive or negative values mean increasing or decreasing
monotonicity respectively. Alternatively a list of n monotonic values,
one per uncalibrated value, like num_keypoints above.
missing_input_values: If set, and if the input has this value it is assumed
to be missing and the output will either be calibrated to some value
between `[calibration_output_min, calibration_output_max]` or set to a
fixed value set by missing_output_value. Limitation: it only works for
scalars. Either one value for all inputs, or a list with one value per
uncalibrated value.
missing_output_values: Requires missing_input_value also to be set. If set
if will convert missing input to this value. Either one value for all
outputs, or a list with one value per uncalibrated value.
name: Name scope for operations.
**regularizer_amounts: Keyword args of regularization amounts passed to
regularizers.calibrator_regularization(). Keyword names should be among
supported regularizers.CALIBRATOR_REGULARIZERS and values should be
either float or list of floats. If float, then same value is applied to
all input signals.
Returns:
A tuple of:
* calibrated tensor of shape [batch_size, ...], the same shape as
uncalibrated.
* list of projection ops, that must be applied at each step (or every so
many steps) to project the model to a feasible space: used for bounding
the outputs or for imposing monotonicity. Empty if none are requested.
* None or tensor with regularization loss.
Raises:
ValueError: If dimensions don't match.
"""
with ops.name_scope(name or 'calibration_layer'):
# Flattening uncalibrated tensor [batch_Size, k1, k2, ..., kn] to
# [batch_size, k1 * k2 * ... * kn].
uncalibrated_shape = uncalibrated_tensor.get_shape().as_list()
n = 1
for non_batch_dim in uncalibrated_shape[1:]:
n *= non_batch_dim
flat_uncalibrated = array_ops.reshape(uncalibrated_tensor,
shape=[-1, n],
name='flat_uncalibrated')
num_keypoints = tools.cast_to_list(num_keypoints, n, 'num_keypoints')
keypoints_initializers = tools.cast_to_list(keypoints_initializers, n,
'keypoints_initializers')
keypoints_initializer_fns = tools.cast_to_list(
keypoints_initializer_fns, n, 'keypoints_initializer_fns')
bound = tools.cast_to_list(bound, n, 'bound')
monotonic = tools.cast_to_list(monotonic, n, 'monotonic')
missing_input_values = tools.cast_to_list(missing_input_values, n,
'missing_input_values')
missing_output_values = tools.cast_to_list(missing_output_values, n,
'missing_output_values')
regularizer_amounts = {
regularizer_name:
tools.cast_to_list(regularizer_amounts[regularizer_name], n,
regularizer_name)
for regularizer_name in regularizer_amounts
}
signal_names = ['signal_%d' % ii for ii in range(n)]
uncalibrated_splits = array_ops.unstack(flat_uncalibrated, axis=1)
calibrated_splits = []
projection_ops = []
total_regularization = None
for ii in range(n):
if not num_keypoints[ii]:
# No calibration for this signal.
calibrated_splits += [uncalibrated_splits[ii]]
else:
signal_regularizer_amounts = {
regularizer_name: regularizer_amounts[regularizer_name][ii]
for regularizer_name in regularizer_amounts
}
calibrated, projection, reg = one_dimensional_calibration_layer(
uncalibrated_splits[ii],
num_keypoints[ii],
signal_name=signal_names[ii],
keypoints_initializers=keypoints_initializers[ii],
keypoints_initializer_fns=keypoints_initializer_fns[ii],
bound=bound[ii],
monotonic=monotonic[ii],
missing_input_value=missing_input_values[ii],
missing_output_value=missing_output_values[ii],
**signal_regularizer_amounts)
calibrated_splits += [calibrated]
if projection is not None:
projection_ops += [projection]
total_regularization = tools.add_if_not_none(
total_regularization, reg)
flat_calibrated = array_ops.stack(calibrated_splits,
axis=1,
name='stack_calibrated')
reshaped_calibrated = array_ops.reshape(
flat_calibrated,
shape=array_ops.shape(uncalibrated_tensor),
name='reshape_calibrated')
return reshaped_calibrated, projection_ops, total_regularization
def monotone_linear_layer(input_tensor,
input_dim,
output_dim,
is_monotone=None,
init_weight_mean=2.0,
init_weight_stddev=0.5,
init_bias=None,
l1_reg=None,
l2_reg=None):
"""Creates a partially monotonic linear embedding layer.
Returns an output of partially monotonic linear embedding layer, weights in
the linear embedding layer, projection ops and regularizers.
output = input * weight' + bias
and the kth row is constrained to be non-negative, if is_monotone[k] == True.
weight is initialized to entrywise Normal random variable (init_weight_mean,
init_weight_stdev). If init_b is not provided, then the initial bias is
initialized to -1/2 * init_weight_mean * input_dim. This offset term is used
to make the initial mean to 0, assuming each input tensor is from the uniform
distribution [0, 1]:
E[output] = E[input * weight' + bias] = E[input] * E[weight] + bias
= 1/2 * init_weight_mean * input_dim + bias
= 0.
Args:
input_tensor: [batch_size, input_dim] tensor.
input_dim: (int) input dimension.
output_dim: (int) output dimension.
is_monotone: A list of input_dim booleans, a single boolean, or None.
If None or False, linear layer will not have monotonicity constraints.
If True, all of inputs are set to be monotonic. In the case of boolean
list, input_tensor[:, k] is set to be monotonic if is_monotone[k] == True.
init_weight_mean: (float) A mean for Normal random weight initializer.
init_weight_stddev: (float) A standard deviation for Normal random weight
initializer.
init_bias: (float) initial bias. If not provided,
-1/2 * init_weight_mean * input_dim is used.
l1_reg: (float) amount of l1 regularization.
l2_reg: (float) amount of l2 regularization.
Returns:
A tuple of:
* output tensor of shape [batch_size, output_dim]
* weight tensor of shape [output_dim, input_dim]
* None or projection ops, that must be applied at each
step (or every so many steps) to project the model to a feasible space:
used for bounding the outputs or for imposing monotonicity.
* None or a regularization loss, if regularization is configured.
Raises:
ValueError: If is_monotone is not None, but its length != input_dim.
"""
with variable_scope.variable_scope('monotone_linear'):
# We use [output_dim, input_dim] convention to use broadcasting in
# projeciton.
init_weights = random_ops.random_normal(
[output_dim, input_dim],
mean=init_weight_mean,
stddev=init_weight_stddev)
if init_bias is None:
init_biases = [-init_weight_mean * 0.5 * input_dim] * output_dim
else:
init_biases = [init_bias] * output_dim
w = variable_scope.get_variable(name='weight',
initializer=init_weights,
dtype=input_tensor.dtype)
b = variable_scope.get_variable(name='bias',
initializer=init_biases,
dtype=input_tensor.dtype)
output_tensor = math_ops.matmul(input_tensor, w, transpose_b=True) + b
# Constructing a projection op.
projection = None
if is_monotone:
with ops.name_scope('monotonic_projection'):
is_monotone = tools.cast_to_list(is_monotone, input_dim, 'is_monotone')
if input_dim != len(is_monotone):
raise ValueError('input_dim (%d) != is_monotone length (%d)' %
(input_dim, len(is_monotone)))
# Construct a multiplicative mask for monotonic dimension
# selection.
monotone_mask = array_ops.constant(
[1.0 if monotone else 0.0 for monotone in is_monotone],
dtype=w.dtype)
# Since input_dim is the last dimension of the weight, we can use
# broadcasting.
masked_w = math_ops.multiply(w, monotone_mask)
projected_w = math_ops.maximum(masked_w, 0.0)
diff = projected_w - masked_w
projection = w.assign_add(diff)
# Constructing a regularization op.
regularizer = None
if l1_reg is not None or l2_reg is not None:
with ops.name_scope('linear_regularization'):
regularizer = regularizers.linear_regularization(w, l1_reg, l2_reg)
return (output_tensor, w, projection, regularizer)
def _lattice_laplacian(lattice_param,
lattice_sizes,
l1_reg=None,
l2_reg=None,
name='lattice_laplacian'):
"""Returns a lattice laplacian regularization.
Laplacian regularizers penalize the difference between adjacent vertices in
multi-cell lattice. See Lattice Regression, NIPS, 2009 for the details, but
we provide a 2d example in here.
Consider a 3 x 2 lattice:
3-------4--------5
| | |
| | |
0-------1--------2
where the number at each node represents the parameter index.
In this case, the laplacian l1 regularizer is defined as
reg = l1_reg[0] * (|param[1] - param[0]| + |param[2] - param[1]|
+ |param[4] - param[3]| + |param[5] - param[4]|)
+ l1_reg[1] * (|param[3] - param[0]| + |param[4] - param[1]|
+ |param[5] - param[2]})
where param is a lattice_param tensor assuming one output.
In l2 case, the absolute value is replaced with a square.
If num_outputs > 1, the op is
total_reg = sum_{d=1}^{output_dim} reg(lattice_param[d, :])
i.e., a sum across all output dimensions.
Args:
lattice_param: (Rank-2 tensor with shape [num_outputs, num_parameters])
lattice model's parameter.
lattice_sizes: (list of integers) lattice size of each dimension.
l1_reg: (list of floats or float) l1 regularization amount per each
lattice dimension. If float, a same number will be accrossed to all
lattice dimensions.
l2_reg: (list of floats or float) l2 regularization amount per each
lattice dimension. If float, a same number will be accrossed to all
lattice dimensions.
name: name scope of lattice laplacian regularizer.
Returns:
A rank-0 tensor (scalar) that contains regularizer or None if there is no
regularization. This can happen if l1_reg and l2_reg amounts are not set.
Raises:
ValueError: * lattice_param is not rank-2 tensor.
* output_dim or param_dim is unknown.
"""
dims = lattice_param.shape.as_list()
if len(dims) != 2:
raise ValueError(
'lattice_laplacian expects lattice_param as a '
'rank-2 tensor but got dimensions: ', dims)
output_dim = dims[0]
param_dim = dims[1]
if output_dim is None or param_dim is None:
raise ValueError(
'lattice_laplacian expects all the dimensions in '
'lattice_param to be known, but got dimensions: ', dims)
l1_reg = tools.cast_to_list(l1_reg, len(lattice_sizes), 'laplacian_l1_reg')
l2_reg = tools.cast_to_list(l2_reg, len(lattice_sizes), 'laplacian_l2_reg')
# Collect all dimensions that has non-trivial regularization amount.
reg_dims = []
lattice_rank = len(lattice_sizes)
for dim in range(lattice_rank):
if l1_reg[dim] or l2_reg[dim]:
reg_dims.append(dim)
if not reg_dims:
return None
regularization = None
with tf.name_scope(name):
for dim in reg_dims:
slice_size = lattice_sizes[dim] - 1
per_dim_upper = tools.lattice_1d_slice(
lattice_param,
lattice_sizes=lattice_sizes,
lattice_axis=dim,
begin=1,
size=slice_size)
per_dim_lower = tools.lattice_1d_slice(
lattice_param,
lattice_sizes=lattice_sizes,
lattice_axis=dim,
begin=0,
size=slice_size)
per_dim_diff = per_dim_upper - per_dim_lower
if l1_reg[dim]:
regularization = tools.add_if_not_none(
regularization, l1_reg[dim] * tf.reduce_sum(tf.abs(per_dim_diff)))
if l2_reg[dim]:
regularization = tools.add_if_not_none(
regularization,
l2_reg[dim] * tf.reduce_sum(tf.square(per_dim_diff)))
return regularization
def ensemble_lattices_layer(input_tensor,
lattice_sizes,
structure_indices,
is_monotone=None,
output_dim=1,
interpolation_type='hypercube',
lattice_initializers=None,
l1_reg=None,
l2_reg=None,
l1_torsion_reg=None,
l2_torsion_reg=None,
l1_laplacian_reg=None,
l2_laplacian_reg=None):
"""Creates a ensemble of lattices layer.
Returns a list of output of lattices, lattice parameters, and projection ops.
Args:
input_tensor: [batch_size, input_dim] tensor.
lattice_sizes: A list of lattice sizes of each dimension.
structure_indices: A list of list of ints. structure_indices[k] is a list
of indices that belongs to kth lattices.
is_monotone: A list of input_dim booleans, boolean or None. If None or
False, lattice will not have monotonicity constraints. If
is_monotone[k] == True, then the lattice output has the non-decreasing
monotonicity with respect to input_tensor[?, k] (the kth coordinate). If
True, all the input coordinate will have the non-decreasing monotonicity.
output_dim: Number of outputs.
interpolation_type: 'hypercube' or 'simplex'.
lattice_initializers: (Optional) A list of initializer for each lattice
parameter vectors. lattice_initializer[k] is a 2D tensor
[output_dim, parameter_dim[k]], where parameter_dim[k] is the number of
parameter in the kth lattice. If None, lattice_param_as_linear initializer
will be used with
linear_weights=[1 if monotone else 0 for monotone in is_monotone].
l1_reg: (float) l1 regularization amount.
l2_reg: (float) l2 regularization amount.
l1_torsion_reg: (float) l1 torsion regularization amount.
l2_torsion_reg: (float) l2 torsion regularization amount.
l1_laplacian_reg: (list of floats or float) list of L1 Laplacian
regularization amount per each dimension. If a single float value is
provided, then all diemnsion will get the same value.
l2_laplacian_reg: (list of floats or float) list of L2 Laplacian
regularization amount per each dimension. If a single float value is
provided, then all diemnsion will get the same value.
Returns:
A tuple of:
* a list of output tensors, [batch_size, output_dim], with length
len(structure_indices), i.e., one for each lattice.
* a list of parameter tensors shape [output_dim, parameter_dim]
* None or projection ops, that must be applied at each
step (or every so many steps) to project the model to a feasible space:
used for bounding the outputs or for imposing monotonicity.
* None or a regularization loss, if regularization is configured.
"""
num_lattices = len(structure_indices)
lattice_initializers = tools.cast_to_list(lattice_initializers,
num_lattices,
'lattice initializers')
if l1_laplacian_reg is not None:
l1_laplacian_reg = tools.cast_to_list(l1_laplacian_reg,
len(lattice_sizes),
'l1_laplacian_reg')
if l2_laplacian_reg is not None:
l2_laplacian_reg = tools.cast_to_list(l2_laplacian_reg,
len(lattice_sizes),
'l2_laplacian_reg')
# input_slices[k] = input_tensor[:, k].
input_slices = array_ops.unstack(input_tensor, axis=1)
output_tensors = []
param_tensors = []
projections = []
regularization = None
if is_monotone:
is_monotone = tools.cast_to_list(is_monotone, len(lattice_sizes),
'is_monotone')
# Now iterate through structure_indices to construct lattices.
for (cnt, structure) in enumerate(structure_indices):
with variable_scope.variable_scope('lattice_%d' % cnt):
sub_lattice_sizes = [lattice_sizes[idx] for idx in structure]
sub_is_monotone = None
if is_monotone:
sub_is_monotone = [is_monotone[idx] for idx in structure]
sub_input_tensor_list = [input_slices[idx] for idx in structure]
sub_input_tensor = array_ops.stack(sub_input_tensor_list, axis=1)
if l1_laplacian_reg is not None:
sub_l1_laplacian_reg = [
l1_laplacian_reg[idx] for idx in structure
]
else:
sub_l1_laplacian_reg = None
if l2_laplacian_reg is not None:
sub_l2_laplacian_reg = [
l2_laplacian_reg[idx] for idx in structure
]
else:
sub_l2_laplacian_reg = None
packed_results = lattice_layer(
sub_input_tensor,
sub_lattice_sizes,
sub_is_monotone,
output_dim=output_dim,
interpolation_type=interpolation_type,
lattice_initializer=lattice_initializers[cnt],
l1_reg=l1_reg,
l2_reg=l2_reg,
l1_torsion_reg=l1_torsion_reg,
l2_torsion_reg=l2_torsion_reg,
l1_laplacian_reg=sub_l1_laplacian_reg,
l2_laplacian_reg=sub_l2_laplacian_reg)
(sub_output, sub_param, sub_proj, sub_reg) = packed_results
output_tensors.append(sub_output)
param_tensors.append(sub_param)
if sub_proj:
projections += sub_proj
regularization = tools.add_if_not_none(regularization, sub_reg)
return (output_tensors, param_tensors, projections, regularization)
def lattice_layer(input_tensor,
lattice_sizes,
is_monotone=None,
output_dim=1,
interpolation_type='hypercube',
lattice_initializer=None,
l1_reg=None,
l2_reg=None,
l1_torsion_reg=None,
l2_torsion_reg=None,
l1_laplacian_reg=None,
l2_laplacian_reg=None):
"""Creates a lattice layer.
Returns an output of lattice, lattice parameters, and projection ops.
Args:
input_tensor: [batch_size, input_dim] tensor.
lattice_sizes: A list of lattice sizes of each dimension.
is_monotone: A list of input_dim booleans, boolean or None. If None or
False, lattice will not have monotonicity constraints. If
is_monotone[k] == True, then the lattice output has the non-decreasing
monotonicity with respect to input_tensor[?, k] (the kth coordinate). If
True, all the input coordinate will have the non-decreasing monotonicity.
output_dim: Number of outputs.
interpolation_type: 'hypercube' or 'simplex'.
lattice_initializer: (Optional) Initializer for lattice parameter vectors,
a 2D tensor [output_dim, parameter_dim] (where parameter_dim ==
lattice_sizes[0] * ... * lattice_sizes[input_dim - 1]). If None,
lattice_param_as_linear initializer will be used with
linear_weights=[1 if monotone else 0 for monotone in is_monotone].
l1_reg: (float) l1 regularization amount.
l2_reg: (float) l2 regularization amount.
l1_torsion_reg: (float) l1 torsion regularization amount.
l2_torsion_reg: (float) l2 torsion regularization amount.
l1_laplacian_reg: (list of floats or float) list of L1 Laplacian
regularization amount per each dimension. If a single float value is
provided, then all diemnsion will get the same value.
l2_laplacian_reg: (list of floats or float) list of L2 Laplacian
regularization amount per each dimension. If a single float value is
provided, then all diemnsion will get the same value.
Returns:
A tuple of:
* output tensor of shape [batch_size, output_dim]
* parameter tensor of shape [output_dim, parameter_dim]
* None or projection ops, that must be applied at each
step (or every so many steps) to project the model to a feasible space:
used for bounding the outputs or for imposing monotonicity.
* None or a regularization loss, if regularization is configured.
Raises:
ValueError: for invalid parameters.
"""
if interpolation_type not in _VALID_INTERPOLATION_TYPES:
raise ValueError('interpolation_type should be one of {}'.format(
_VALID_INTERPOLATION_TYPES))
if lattice_initializer is None:
if is_monotone:
is_monotone = tools.cast_to_list(is_monotone, len(lattice_sizes),
'is_monotone')
linear_weights = [
1.0 if monotonic else 0.0 for monotonic in is_monotone
]
else:
linear_weights = [0.0] * len(lattice_sizes)
lattice_initializer = lattice_param_as_linear(
lattice_sizes, output_dim, linear_weights=linear_weights)
parameter_tensor = variable_scope.get_variable(
interpolation_type + '_lattice_parameters',
initializer=lattice_initializer)
output_tensor = lattice_ops.lattice(input_tensor,
parameter_tensor,
lattice_sizes,
interpolation_type=interpolation_type)
with ops.name_scope('lattice_monotonic_projection'):
if is_monotone:
is_monotone = tools.cast_to_list(is_monotone, len(lattice_sizes),
'is_monotone')
projected_parameter_tensor = monotone_lattice(
parameter_tensor,
lattice_sizes=lattice_sizes,
is_monotone=is_monotone)
delta = projected_parameter_tensor - parameter_tensor
projection_ops = [parameter_tensor.assign_add(delta)]
else:
projection_ops = None
with ops.name_scope('lattice_regularization'):
reg = regularizers.lattice_regularization(
parameter_tensor,
lattice_sizes,
l1_reg=l1_reg,
l2_reg=l2_reg,
l1_torsion_reg=l1_torsion_reg,
l2_torsion_reg=l2_torsion_reg,
l1_laplacian_reg=l1_laplacian_reg,
l2_laplacian_reg=l2_laplacian_reg)
return (output_tensor, parameter_tensor, projection_ops, reg)
def ensemble_lattices_layer(input_tensor,
lattice_sizes,
structure_indices,
is_monotone=None,
output_dim=1,
interpolation_type='hypercube',
lattice_initializers=None,
**regularizer_amounts):
"""Creates a ensemble of lattices layer.
Returns a list of output of lattices, lattice parameters, and projection ops.
Args:
input_tensor: [batch_size, input_dim] tensor.
lattice_sizes: A list of lattice sizes of each dimension.
structure_indices: A list of list of ints. structure_indices[k] is a list of
indices that belongs to kth lattices.
is_monotone: A list of input_dim booleans, boolean or None. If None or
False, lattice will not have monotonicity constraints. If is_monotone[k]
== True, then the lattice output has the non-decreasing monotonicity with
respect to input_tensor[?, k] (the kth coordinate). If True, all the input
coordinate will have the non-decreasing monotonicity.
output_dim: Number of outputs.
interpolation_type: 'hypercube' or 'simplex'.
lattice_initializers: (Optional) A list of initializer for each lattice
parameter vectors. lattice_initializer[k] is a 2D tensor [output_dim,
parameter_dim[k]], where parameter_dim[k] is the number of parameter in
the kth lattice. If None, lattice_param_as_linear initializer will be used
with linear_weights=[1 if monotone else 0 for monotone in is_monotone].
**regularizer_amounts: Keyword args of regularization amounts passed to
regularizers.lattice_regularization(). Keyword names should be among
regularizers.LATTICE_ONE_DIMENSIONAL_REGULARIZERS or
regularizers.LATTICE_MULTI_DIMENSIONAL_REGULARIZERS. For multi-dimensional
regularizers the value should be float. For one-dimensional regularizers
the values should be float or list of floats. If a single float value is
provided, then all dimensions will get the same value.
Returns:
A tuple of:
* a list of output tensors, [batch_size, output_dim], with length
len(structure_indices), i.e., one for each lattice.
* a list of parameter tensors shape [output_dim, parameter_dim]
* None or projection ops, that must be applied at each
step (or every so many steps) to project the model to a feasible space:
used for bounding the outputs or for imposing monotonicity.
* None or a regularization loss, if regularization is configured.
"""
num_lattices = len(structure_indices)
lattice_initializers = tools.cast_to_list(lattice_initializers,
num_lattices,
'lattice initializers')
one_dimensional_regularizers = \
regularizers.LATTICE_ONE_DIMENSIONAL_REGULARIZERS
for regularizer_name in regularizer_amounts:
if regularizer_name in one_dimensional_regularizers:
regularizer_amounts[regularizer_name] = tools.cast_to_list(
regularizer_amounts[regularizer_name], len(lattice_sizes),
regularizer_name)
# input_slices[k] = input_tensor[:, k].
input_slices = tf.unstack(input_tensor, axis=1)
output_tensors = []
param_tensors = []
projections = []
regularization = None
if is_monotone:
is_monotone = tools.cast_to_list(is_monotone, len(lattice_sizes),
'is_monotone')
# Now iterate through structure_indices to construct lattices.
get_indices = lambda indices, iterable: [
iterable[index] for index in indices
]
for (cnt, structure) in enumerate(structure_indices):
with tf.compat.v1.variable_scope('lattice_%d' % cnt):
sub = functools.partial(get_indices, structure)
sub_lattice_sizes = sub(lattice_sizes)
sub_is_monotone = None
if is_monotone:
sub_is_monotone = sub(is_monotone)
sub_input_tensor_list = sub(input_slices)
sub_input_tensor = tf.stack(sub_input_tensor_list, axis=1)
sub_regularizer_amounts = {}
for regularizer_name in regularizer_amounts:
if regularizer_name in one_dimensional_regularizers:
sub_regularizer_amounts[regularizer_name] = sub(
regularizer_amounts[regularizer_name])
else:
sub_regularizer_amounts[
regularizer_name] = regularizer_amounts[
regularizer_name]
packed_results = lattice_layer(
sub_input_tensor,
sub_lattice_sizes,
sub_is_monotone,
output_dim=output_dim,
interpolation_type=interpolation_type,
lattice_initializer=lattice_initializers[cnt],
**sub_regularizer_amounts)
(sub_output, sub_param, sub_proj, sub_reg) = packed_results
output_tensors.append(sub_output)
param_tensors.append(sub_param)
if sub_proj:
projections += sub_proj
regularization = tools.add_if_not_none(regularization, sub_reg)
return (output_tensors, param_tensors, projections, regularization)
def lattice_layer(input_tensor,
lattice_sizes,
is_monotone=None,
output_min=None,
output_max=None,
output_dim=1,
interpolation_type='hypercube',
lattice_initializer=None,
**regularizer_amounts):
"""Creates a lattice layer.
Returns an output of lattice, lattice parameters, and projection ops.
Args:
input_tensor: [batch_size, input_dim] tensor.
lattice_sizes: A list of lattice sizes of each dimension.
is_monotone: A list of input_dim booleans, boolean or None. If None or
False, lattice will not have monotonicity constraints. If is_monotone[k]
== True, then the lattice output has the non-decreasing monotonicity with
respect to input_tensor[?, k] (the kth coordinate). If True, all the input
coordinate will have the non-decreasing monotonicity.
output_min: Optional output lower bound.
output_max: Optional output upper bound.
output_dim: Number of outputs.
interpolation_type: 'hypercube' or 'simplex'.
lattice_initializer: (Optional) Initializer for lattice parameter vectors, a
2D tensor [output_dim, parameter_dim] (where parameter_dim ==
lattice_sizes[0] * ... * lattice_sizes[input_dim - 1]). If None,
lattice_param_as_linear initializer will be used with linear_weights=[1] *
len(lattice_sizes).
**regularizer_amounts: Keyword args of regularization amounts passed to
regularizers.lattice_regularization(). Keyword names should be among
regularizers.LATTICE_ONE_DIMENSIONAL_REGULARIZERS or
regularizers.LATTICE_MULTI_DIMENSIONAL_REGULARIZERS. For multi-dimensional
regularizers the value should be float. For one-dimensional regularizers
the values should be float or list of floats. If a single float value is
provided, then all dimensions will get the same value.
Returns:
A tuple of:
* output tensor of shape [batch_size, output_dim]
* parameter tensor of shape [output_dim, parameter_dim]
* None or projection ops, that must be applied at each
step (or every so many steps) to project the model to a feasible space:
used for bounding the outputs or for imposing monotonicity.
* None or a regularization loss, if regularization is configured.
Raises:
ValueError: for invalid parameters.
"""
if interpolation_type not in _VALID_INTERPOLATION_TYPES:
raise ValueError('interpolation_type should be one of {}'.format(
_VALID_INTERPOLATION_TYPES))
if lattice_initializer is None:
linear_weights = [1.0] * len(lattice_sizes)
lattice_initializer = lattice_param_as_linear(
lattice_sizes, output_dim, linear_weights=linear_weights)
parameter_tensor = tf.compat.v1.get_variable(
interpolation_type + '_lattice_parameters',
initializer=lattice_initializer)
output_tensor = lattice_ops.lattice(input_tensor,
parameter_tensor,
lattice_sizes,
interpolation_type=interpolation_type)
with tf.name_scope('lattice_monotonic_projection'):
if is_monotone or output_min is not None or output_max is not None:
projected_parameter_tensor = parameter_tensor
if is_monotone:
is_monotone = tools.cast_to_list(is_monotone,
len(lattice_sizes),
'is_monotone')
projected_parameter_tensor = monotone_lattice(
projected_parameter_tensor,
lattice_sizes=lattice_sizes,
is_monotone=is_monotone)
if output_min is not None:
projected_parameter_tensor = tf.maximum(
projected_parameter_tensor, output_min)
if output_max is not None:
projected_parameter_tensor = tf.minimum(
projected_parameter_tensor, output_max)
delta = projected_parameter_tensor - parameter_tensor
projection_ops = [parameter_tensor.assign_add(delta)]
else:
projection_ops = None
with tf.name_scope('lattice_regularization'):
reg = regularizers.lattice_regularization(parameter_tensor,
lattice_sizes,
**regularizer_amounts)
return (output_tensor, parameter_tensor, projection_ops, reg)
