def testTwodMultiOutputLatticeSlice(self):
lattice_sizes = [2, 2]
# first_param[0][0] = 0
# first_param[1][0] = 1
# first_param[0][1] = 2
# first_param[1][1] = 3
# second_param[0][0] = 3
# second_param[1][0] = 2
# second_param[0][1] = 1
# second_param[1][1] = 0
lattice_param_tensor = tf.constant(
[list(range(2 * 2)), list(range(2 * 2 - 1, -1, -1))])
# param[0][:] = [[0, 2], [3, 1]]
param_0_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=0, begin=0, size=1)
self._runAndCheckValues(param_0_x, expected_value=[[0, 2], [3, 1]])
# param[1][:] = [[1, 3], [2, 0]]
param_1_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=0, begin=1, size=1)
self._runAndCheckValues(param_1_x, expected_value=[[1, 3], [2, 0]])
# param[:][0] = [[0, 1], [3, 2]]
param_x_0 = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=0, size=1)
self._runAndCheckValues(param_x_0, expected_value=[[0, 1], [3, 2]])
# param[:][1] = [[2, 3], [1, 0]]
param_x_1 = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=1, size=1)
self._runAndCheckValues(param_x_1, expected_value=[[2, 3], [1, 0]])
def testThreedLatticeSlice(self):
lattice_sizes = [2, 3, 2]
# param[0][0][0] = 0
# param[1][0][0] = 1
# param[0][1][0] = 2
# param[1][1][0] = 3
# param[0][2][0] = 4
# param[1][2][0] = 5
# param[0][0][1] = 6
# param[1][0][1] = 7
# param[0][1][1] = 8
# param[1][1][1] = 9
# param[0][2][1] = 10
# param[1][2][1] = 11
lattice_param_tensor = tf.constant([list(range(2 * 3 * 2))])
# param[0][:][:] = [0, 2, 4, 6, 8, 10]
param_0_x_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=0, begin=0, size=1)
self._runAndCheckValues(param_0_x_x, expected_value=[[0, 2, 4, 6, 8, 10]])
# param[1][:][:] = [1, 3, 5, 7, 9, 11]
param_1_x_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=0, begin=1, size=1)
self._runAndCheckValues(param_1_x_x, expected_value=[[1, 3, 5, 7, 9, 11]])
# param[:][0][:] = [0, 1, 6, 7]
param_x_0_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=0, size=1)
self._runAndCheckValues(param_x_0_x, expected_value=[[0, 1, 6, 7]])
# param[:][1][:] = [2, 3, 8, 9]
param_x_1_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=1, size=1)
self._runAndCheckValues(param_x_1_x, expected_value=[[2, 3, 8, 9]])
# param[:][2][:] = [4, 5, 10, 11]
param_x_2_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=2, size=1)
self._runAndCheckValues(param_x_2_x, expected_value=[[4, 5, 10, 11]])
# param[:][0:1][:] = [0, 1, 2, 3, 6, 7, 8, 9]
param_x_01_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=0, size=2)
self._runAndCheckValues(
param_x_01_x, expected_value=[[0, 1, 2, 3, 6, 7, 8, 9]])
# param[:][1:2][:] = [2, 3, 4, 5, 8, 9, 10, 11]
param_x_12_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=1, size=2)
self._runAndCheckValues(
param_x_12_x, expected_value=[[2, 3, 4, 5, 8, 9, 10, 11]])
# param[:][:][0] = [0, 1, 2, 3, 4, 5]
param_x_x_0 = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=2, begin=0, size=1)
self._runAndCheckValues(param_x_x_0, expected_value=[[0, 1, 2, 3, 4, 5]])
# param[:][:][1] = [6, 7, 8, 9, 10, 11]
param_x_x_1 = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=2, begin=1, size=1)
self._runAndCheckValues(param_x_x_1, expected_value=[[6, 7, 8, 9, 10, 11]])
def testBeginSizeOutOfRangeExpectsError(self):
lattice_param_tensor = tf.compat.v1.placeholder(shape=(2, 4), dtype=tf.float32)
with self.assertRaises(ValueError):
_ = tools.lattice_1d_slice(
lattice_param_tensor,
lattice_sizes=[2, 2],
lattice_axis=0,
begin=1,
size=2)
def testWrongTensorShapeExpectsError(self):
lattice_param_tensor = tf.compat.v1.placeholder(shape=(2, 2, 2), dtype=tf.float32)
with self.assertRaises(ValueError):
_ = tools.lattice_1d_slice(
lattice_param_tensor,
lattice_sizes=[2],
lattice_axis=0,
begin=0,
size=1)
def testOutOfRangeAxisExpectsError(self):
lattice_param_tensor = array_ops.placeholder(
shape=(2, 4), dtype=dtypes.float32)
with self.assertRaises(ValueError):
_ = tools.lattice_1d_slice(
lattice_param_tensor,
lattice_sizes=[2, 2],
lattice_axis=3,
begin=0,
size=1)
def testTwodLatticeSlice(self):
lattice_sizes = [2, 3]
# param[0][0] = 0
# param[1][0] = 1
# param[0][1] = 2
# param[1][1] = 3
# param[0][2] = 4
# param[1][2] = 5
lattice_param_tensor = tf.constant([list(range(2 * 3))])
# param[0][:] = [0, 2, 4]
param_0_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=0, begin=0, size=1)
self._runAndCheckValues(param_0_x, expected_value=[[0, 2, 4]])
# param[1][:] = [1, 3, 5]
param_1_x = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=0, begin=1, size=1)
self._runAndCheckValues(param_1_x, expected_value=[[1, 3, 5]])
# param[:][0] = [0, 1]
param_x_0 = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=0, size=1)
self._runAndCheckValues(param_x_0, expected_value=[[0, 1]])
# param[:][1] = [2, 3]
param_x_1 = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=1, size=1)
self._runAndCheckValues(param_x_1, expected_value=[[2, 3]])
# param[:][2] = [4, 5]
param_x_2 = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=2, size=1)
self._runAndCheckValues(param_x_2, expected_value=[[4, 5]])
# param[:][0:1] = [0, 1, 2, 3]
param_x_01 = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=0, size=2)
self._runAndCheckValues(param_x_01, expected_value=[[0, 1, 2, 3]])
# param[:][1:2] = [2, 3, 4, 5]
param_x_12 = tools.lattice_1d_slice(
lattice_param_tensor, lattice_sizes, lattice_axis=1, begin=1, size=2)
self._runAndCheckValues(param_x_12, expected_value=[[2, 3, 4, 5]])
def _lattice_torsion(lattice_param,
lattice_sizes,
l1_reg=None,
l2_reg=None,
name='lattice_torsion'):
"""Returns a lattice torsion regularization.
Torsion regularizers penalizes how much the lattice function twists from
side-to-side, a non-linear interactions in each 2 x 2 cells. See
Monotonic Calibrated Interpolated Look-Up Tables, JMLR, 2016 for the details,
but we provide a 2d example in here.
Consider a 3 x 2 lattice:
3-------4--------5
| | |
| | |
0-------1--------2
where the number at each node represents the parameter index.
In this case, the torsion l2 regularizer is defined as
reg = l2_reg * ((param[4] + param[0] - param[3] - param[1]) ** 2
+ (param[5] + param[1] - param[4] - param[2]) ** 2
where param is a lattice_param tensor assuming one output.
In l1 case, the squared value is replaced with the absolte value.
If num_outputs > 1, the op is
total_reg = sum_{d=1}^{output_dim} reg(lattice_param[d, :])
i.e., a sum across all output dimensions.
Args:
lattice_param: (Rank-2 tensor with shape [num_outputs, num_parameters])
lattice model's parameter.
lattice_sizes: (list of integers) lattice size of each dimension.
l1_reg: (float) l1 regularization amount.
l2_reg: (float) l2 regularization amount.
name: name scope of lattice torsion regularizer.
Returns:
A rank-0 tensor (scalar) that contains regularizer or None if there is no
regularization. This can happen if l1_reg and l2_reg amounts are not set.
Raises:
ValueError: * lattice_param is not rank-2 tensor.
* output_dim or param_dim is unknown.
"""
dims = lattice_param.shape.as_list()
if len(dims) != 2:
raise ValueError(
'lattice_laplacian expects lattice_param as a '
'rank-2 tensor but got dimensions: ', dims)
output_dim = dims[0]
param_dim = dims[1]
lattice_rank = len(lattice_sizes)
if output_dim is None or param_dim is None:
raise ValueError(
'lattice_laplacian expects all the dimensions in '
'lattice_param to be known, but got dimensions: ', dims)
if l1_reg is None and l2_reg is None:
return None
regularization = None
with tf.name_scope(name):
for dim1 in range(lattice_rank - 1):
slice_size1 = lattice_sizes[dim1] - 1
param_0x = tools.lattice_1d_slice(
lattice_param,
lattice_sizes=lattice_sizes,
lattice_axis=dim1,
begin=0,
size=slice_size1)
param_1x = tools.lattice_1d_slice(
lattice_param,
lattice_sizes=lattice_sizes,
lattice_axis=dim1,
begin=1,
size=slice_size1)
resized_lattice_sizes = copy.deepcopy(lattice_sizes)
resized_lattice_sizes[dim1] -= 1
for dim2 in range(dim1 + 1, lattice_rank):
slice_size2 = resized_lattice_sizes[dim2] - 1
param_00 = tools.lattice_1d_slice(
param_0x,
lattice_sizes=resized_lattice_sizes,
lattice_axis=dim2,
begin=0,
size=slice_size2)
param_01 = tools.lattice_1d_slice(
param_0x,
lattice_sizes=resized_lattice_sizes,
lattice_axis=dim2,
begin=1,
size=slice_size2)
param_10 = tools.lattice_1d_slice(
param_1x,
lattice_sizes=resized_lattice_sizes,
lattice_axis=dim2,
begin=0,
size=slice_size2)
param_11 = tools.lattice_1d_slice(
param_1x,
lattice_sizes=resized_lattice_sizes,
lattice_axis=dim2,
begin=1,
size=slice_size2)
torsion = param_00 + param_11 - param_01 - param_10
if l1_reg:
regularization = tools.add_if_not_none(
regularization, l1_reg * tf.reduce_sum(tf.abs(torsion)))
if l2_reg:
regularization = tools.add_if_not_none(
regularization, l2_reg * tf.reduce_sum(tf.square(torsion)))
return regularization
def _lattice_laplacian(lattice_param,
lattice_sizes,
l1_reg=None,
l2_reg=None,
name='lattice_laplacian'):
"""Returns a lattice laplacian regularization.
Laplacian regularizers penalize the difference between adjacent vertices in
multi-cell lattice. See Lattice Regression, NIPS, 2009 for the details, but
we provide a 2d example in here.
Consider a 3 x 2 lattice:
3-------4--------5
| | |
| | |
0-------1--------2
where the number at each node represents the parameter index.
In this case, the laplacian l1 regularizer is defined as
reg = l1_reg[0] * (|param[1] - param[0]| + |param[2] - param[1]|
+ |param[4] - param[3]| + |param[5] - param[4]|)
+ l1_reg[1] * (|param[3] - param[0]| + |param[4] - param[1]|
+ |param[5] - param[2]})
where param is a lattice_param tensor assuming one output.
In l2 case, the absolute value is replaced with a square.
If num_outputs > 1, the op is
total_reg = sum_{d=1}^{output_dim} reg(lattice_param[d, :])
i.e., a sum across all output dimensions.
Args:
lattice_param: (Rank-2 tensor with shape [num_outputs, num_parameters])
lattice model's parameter.
lattice_sizes: (list of integers) lattice size of each dimension.
l1_reg: (list of floats or float) l1 regularization amount per each
lattice dimension. If float, a same number will be accrossed to all
lattice dimensions.
l2_reg: (list of floats or float) l2 regularization amount per each
lattice dimension. If float, a same number will be accrossed to all
lattice dimensions.
name: name scope of lattice laplacian regularizer.
Returns:
A rank-0 tensor (scalar) that contains regularizer or None if there is no
regularization. This can happen if l1_reg and l2_reg amounts are not set.
Raises:
ValueError: * lattice_param is not rank-2 tensor.
* output_dim or param_dim is unknown.
"""
dims = lattice_param.shape.as_list()
if len(dims) != 2:
raise ValueError(
'lattice_laplacian expects lattice_param as a '
'rank-2 tensor but got dimensions: ', dims)
output_dim = dims[0]
param_dim = dims[1]
if output_dim is None or param_dim is None:
raise ValueError(
'lattice_laplacian expects all the dimensions in '
'lattice_param to be known, but got dimensions: ', dims)
l1_reg = tools.cast_to_list(l1_reg, len(lattice_sizes), 'laplacian_l1_reg')
l2_reg = tools.cast_to_list(l2_reg, len(lattice_sizes), 'laplacian_l2_reg')
# Collect all dimensions that has non-trivial regularization amount.
reg_dims = []
lattice_rank = len(lattice_sizes)
for dim in range(lattice_rank):
if l1_reg[dim] or l2_reg[dim]:
reg_dims.append(dim)
if not reg_dims:
return None
regularization = None
with tf.name_scope(name):
for dim in reg_dims:
slice_size = lattice_sizes[dim] - 1
per_dim_upper = tools.lattice_1d_slice(
lattice_param,
lattice_sizes=lattice_sizes,
lattice_axis=dim,
begin=1,
size=slice_size)
per_dim_lower = tools.lattice_1d_slice(
lattice_param,
lattice_sizes=lattice_sizes,
lattice_axis=dim,
begin=0,
size=slice_size)
per_dim_diff = per_dim_upper - per_dim_lower
if l1_reg[dim]:
regularization = tools.add_if_not_none(
regularization, l1_reg[dim] * tf.reduce_sum(tf.abs(per_dim_diff)))
if l2_reg[dim]:
regularization = tools.add_if_not_none(
regularization,
l2_reg[dim] * tf.reduce_sum(tf.square(per_dim_diff)))
return regularization
