def testProjection(self): """Check projection operator.""" input_dim = 2 monotonic_output_dim = 2 non_monotonic_output_dim = 1 # First five is monotonic, and the last five is non-monotonic. is_monotone = [True, False] input_placeholder = array_ops.placeholder( dtype=dtypes.float32, shape=[None, input_dim]) packed_results = monotone_linear_layers.split_monotone_linear_layer( input_placeholder, input_dim=input_dim, monotonic_output_dim=monotonic_output_dim, non_monotonic_output_dim=non_monotonic_output_dim, is_monotone=is_monotone, init_weight_mean=-10.0, init_weight_stddev=0.0) (_, monotone_weights, _, non_monotone_weights, proj, _) = packed_results expected_pre_monotone_weights = [[-10.0, -10.0]] * 2 expected_pre_non_monotone_weights = [[-10.0]] expected_projected_monotone_weights = [[0.0, -10.0]] * 2 expected_projected_non_monotone_weights = [[-10.0]] with self.test_session() as sess: sess.run(variables.global_variables_initializer()) self.assertAllClose(expected_pre_monotone_weights, monotone_weights.eval()) self.assertAllClose(expected_pre_non_monotone_weights, non_monotone_weights.eval()) sess.run(proj) self.assertAllClose(expected_projected_monotone_weights, monotone_weights.eval()) self.assertAllClose(expected_projected_non_monotone_weights, non_monotone_weights.eval())
def testNoNonMonotonicInputsWithNonMonotonicOutputExpectFailure(self): input_dim = 2 monotonic_output_dim = 2 non_monotonic_output_dim = 2 is_monotone = [True, True] input_placeholder = array_ops.placeholder( dtype=dtypes.float32, shape=[None, input_dim]) with self.assertRaises(ValueError): _ = monotone_linear_layers.split_monotone_linear_layer( input_placeholder, input_dim=input_dim, monotonic_output_dim=monotonic_output_dim, non_monotonic_output_dim=non_monotonic_output_dim, is_monotone=is_monotone, init_weight_mean=-10.0, init_weight_stddev=0.0)
def testBooleanIsMonotoneExpectsError(self): """Test empty non monotonic output.""" input_dim = 2 monotonic_output_dim = 2 non_monotonic_output_dim = 1 is_monotone = True input_placeholder = array_ops.placeholder( dtype=dtypes.float32, shape=[None, input_dim]) with self.assertRaises(ValueError): _ = monotone_linear_layers.split_monotone_linear_layer( input_placeholder, input_dim=input_dim, monotonic_output_dim=monotonic_output_dim, non_monotonic_output_dim=non_monotonic_output_dim, is_monotone=is_monotone, init_weight_mean=-10.0, init_weight_stddev=0.0)
def testEvaluation(self): """Create a split monotone linear layer and check the results.""" batch_size = 5 input_dim = 10 monotonic_output_dim = 2 non_monotonic_output_dim = 3 # First five is monotonic, and the last five is non-monotonic. is_monotone = [True] * 5 + [False] * 5 input_placeholder = array_ops.placeholder( dtype=dtypes.float32, shape=[batch_size, input_dim]) packed_results = monotone_linear_layers.split_monotone_linear_layer( input_placeholder, input_dim=input_dim, monotonic_output_dim=monotonic_output_dim, non_monotonic_output_dim=non_monotonic_output_dim, is_monotone=is_monotone) (monotonic_output, _, non_monotonic_output, _, _, _) = packed_results # Check the shape of outputs. self.assertAllEqual(monotonic_output.shape, [batch_size, monotonic_output_dim]) self.assertAllEqual(non_monotonic_output.shape, [batch_size, non_monotonic_output_dim]) # Check monotonic inputs are not part of non_monotonic_output. # We do this by changing the first half of inputs and check whether it # changes the value or not. zero_input = [[0.0] * 10] * 5 identity_in_monotone_inputs = [ [1.0, 0.0, 0.0, 0.0, 0.0] + [0.0] * 5, [0.0, 1.0, 0.0, 0.0, 0.0] + [0.0] * 5, [0.0, 0.0, 1.0, 0.0, 0.0] + [0.0] * 5, [0.0, 0.0, 0.0, 1.0, 0.0] + [0.0] * 5, [0.0, 0.0, 0.0, 0.0, 1.0] + [0.0] * 5, ] with self.test_session() as sess: sess.run(variables.global_variables_initializer()) non_monotonic_output_at_zero = sess.run( non_monotonic_output, feed_dict={input_placeholder: zero_input}) non_monotonic_output_at_identity = sess.run( non_monotonic_output, feed_dict={input_placeholder: identity_in_monotone_inputs}) self.assertAllClose(non_monotonic_output_at_zero, non_monotonic_output_at_identity)
def testNoRegularizationExpectsNone(self): """Create a split monotone linear layer and check no regularization.""" input_dim = 2 monotonic_output_dim = 2 non_monotonic_output_dim = 2 is_monotone = [True, False] input_placeholder = array_ops.placeholder( dtype=dtypes.float32, shape=[None, input_dim]) # We set the initial_weight_mean to -10.0. (_, _, _, _, _, regularization) = monotone_linear_layers.split_monotone_linear_layer( input_placeholder, input_dim=input_dim, monotonic_output_dim=monotonic_output_dim, non_monotonic_output_dim=non_monotonic_output_dim, is_monotone=is_monotone, init_weight_mean=-10.0, init_weight_stddev=0.0) self.assertIsNone(regularization)
def testZeroNonMonotonicOutputExpectEmptyNonMonotonicOutput(self): """Test empty non monotonic output.""" input_dim = 2 monotonic_output_dim = 2 non_monotonic_output_dim = 0 is_monotone = [True, True] input_placeholder = array_ops.placeholder( dtype=dtypes.float32, shape=[None, input_dim]) packed_results = monotone_linear_layers.split_monotone_linear_layer( input_placeholder, input_dim=input_dim, monotonic_output_dim=monotonic_output_dim, non_monotonic_output_dim=non_monotonic_output_dim, is_monotone=is_monotone, init_weight_mean=-10.0, init_weight_stddev=0.0) (_, _, non_monotonic_outputs, non_monotonic_weights, _, _) = packed_results self.assertEqual(non_monotonic_outputs, None) self.assertEqual(non_monotonic_weights, None)
def testRegularization(self): """Create a split monotone linear layer and check regularization.""" input_dim = 2 monotonic_output_dim = 2 non_monotonic_output_dim = 2 is_monotone = [True, False] input_placeholder = array_ops.placeholder( dtype=dtypes.float32, shape=[None, input_dim]) # We set the initial_weight_mean to -10.0. (_, _, _, _, _, regularization) = monotone_linear_layers.split_monotone_linear_layer( input_placeholder, input_dim=input_dim, monotonic_output_dim=monotonic_output_dim, non_monotonic_output_dim=non_monotonic_output_dim, is_monotone=is_monotone, init_weight_mean=-10.0, init_weight_stddev=0.0, l1_reg=0.1, l2_reg=0.1) with self.test_session() as sess: sess.run(variables.global_variables_initializer()) self.assertAlmostEqual(66.0, sess.run(regularization), delta=1e-5)
def _embedded_lattices(calibrated_input_tensor, input_dim, output_dim, interpolation_type, monotonic_num_lattices, monotonic_lattice_rank, monotonic_lattice_size, non_monotonic_num_lattices, non_monotonic_lattice_rank, non_monotonic_lattice_size, linear_embedding_calibration_min, linear_embedding_calibration_max, linear_embedding_calibration_num_keypoints, is_monotone=None, lattice_l1_reg=None, lattice_l2_reg=None, lattice_l1_torsion_reg=None, lattice_l2_torsion_reg=None, lattice_l1_laplacian_reg=None, lattice_l2_laplacian_reg=None): """Creates an ensemble of lattices with a linear embedding. This function constructs the following deep lattice network: calibrated_input -> linear_embedding -> calibration -> ensemble of lattices. Then ensemble of lattices' output are averaged and bias term is added to make a final prediction. ensemble of lattices is consists of two parts: monotonic lattices and non-monotonic lattices. The input to the monotonic lattices is an output of linear_embedding that contains both monotonic and non-monotonic calibrated_input. All inputs to the monotonic lattices are set to be monotonic to preserve end-to-end monotonicity in the monotonic feature. The input to the non-monotonic lattices is an output of linear_embedding that only contains non-monotonic calibrated_input. All inputs to the non-monotonic lattices are set to be non-monotonic, since we do not need to guarantee monotonicity. Args: calibrated_input_tensor: [batch_size, input_dim] tensor. input_dim: (int) input dimnension. output_dim: (int) output dimension. interpolation_type: defines whether the lattice will interpolate using the full hypercube or only the simplex ("hyper-triangle") around the point being evaluated. Valid values: 'hypercube' or 'simplex' monotonic_num_lattices: (int) number of monotonic lattices in the ensemble lattices layer. monotonic_lattice_rank: (int) number of inputs to each monotonic lattice in the ensemble lattices layer. monotonic_lattice_size: (int) lattice cell size for each monotonic lattice in the ensemble lattices layer. non_monotonic_num_lattices: (int) number of non monotonic lattices in the ensemble lattices layer. non_monotonic_lattice_rank: (int) number of inputs to each non monotonic lattice in the ensemble lattices layer. non_monotonic_lattice_size: (int) lattice cell size for each non monotonic lattice in the ensemble lattices layer. linear_embedding_calibration_min: (float) a minimum input keypoints value for linear_embedding calibration. linear_embedding_calibration_max: (float) a maximum input keypoints value for linear_embedding calibration. linear_embedding_calibration_num_keypoints: (int) a number of eypoints for linear_embedding calibration. is_monotone: (bool, list of booleans) is_monotone[k] == true then calibrated_input_tensor[:, k] is considered to be a monotonic input. lattice_l1_reg: (float) lattice l1 regularization amount. lattice_l2_reg: (float) lattice l2 regularization amount. lattice_l1_torsion_reg: (float) lattice l1 torsion regularization amount. lattice_l2_torsion_reg: (float) lattice l2 torsion regularization amount. lattice_l1_laplacian_reg: (float) lattice l1 laplacian regularization amount. lattice_l2_laplacian_reg: (float) lattice l2 laplacian regularization amount. Returns: A tuple of (output_tensor, projection_ops, regularization). Raises: ValueError: If there is no non-monotonic inputs but non_monotonic_num_lattices is not zero. """ projections = [] regularization = None # Explictly assign number of lattices to zero for any empty cases. if not monotonic_num_lattices: monotonic_num_lattices = 0 if not non_monotonic_num_lattices: non_monotonic_num_lattices = 0 # Step 1. Create a linear embedding. if monotonic_num_lattices: monotonic_embedding_dim = monotonic_num_lattices * monotonic_lattice_rank else: monotonic_num_lattices = 0 monotonic_embedding_dim = 0 if non_monotonic_num_lattices: non_monotonic_embedding_dim = (non_monotonic_num_lattices * non_monotonic_lattice_rank) else: non_monotonic_num_lattices = 0 non_monotonic_embedding_dim = 0 if is_monotone is not None: is_monotone = tools.cast_to_list(is_monotone, input_dim, 'is_monotone') with variable_scope.variable_scope('linear_embedding'): packed_results = monotone_linear_layers.split_monotone_linear_layer( calibrated_input_tensor, input_dim, monotonic_embedding_dim, non_monotonic_embedding_dim, is_monotone=is_monotone) (monotonic_output, _, non_monotonic_output, _, proj, _) = packed_results if proj is not None: projections.append(proj) # Step 2. Create ensemble of monotonic lattices. if monotonic_num_lattices == 0: m_lattice_outputs = None else: with variable_scope.variable_scope('monotonic_lattices'): m_lattice_outputs, projs, reg = _ensemble_lattices_layer( monotonic_output, monotonic_embedding_dim, output_dim, interpolation_type, linear_embedding_calibration_min, linear_embedding_calibration_max, linear_embedding_calibration_num_keypoints, monotonic_num_lattices, monotonic_lattice_rank, monotonic_lattice_size, is_monotone=True, l1_reg=lattice_l1_reg, l2_reg=lattice_l2_reg, l1_torsion_reg=lattice_l1_torsion_reg, l2_torsion_reg=lattice_l2_torsion_reg, l1_laplacian_reg=lattice_l1_laplacian_reg, l2_laplacian_reg=lattice_l2_laplacian_reg) if projs: projections += projs regularization = tools.add_if_not_none(regularization, reg) # Step 3. Construct non-monotonic ensembles. if non_monotonic_output is None and non_monotonic_num_lattices > 0: raise ValueError( 'All input signals are monotonic but the number of non monotonic ' 'lattices is not zero.') if non_monotonic_num_lattices == 0: n_lattice_outputs = None else: with variable_scope.variable_scope('non_monotonic_lattices'): n_lattice_outputs, projs, reg = _ensemble_lattices_layer( non_monotonic_output, non_monotonic_embedding_dim, output_dim, interpolation_type, linear_embedding_calibration_min, linear_embedding_calibration_max, linear_embedding_calibration_num_keypoints, non_monotonic_num_lattices, non_monotonic_lattice_rank, non_monotonic_lattice_size, is_monotone=False, l1_reg=lattice_l1_reg, l2_reg=lattice_l2_reg, l1_torsion_reg=lattice_l1_torsion_reg, l2_torsion_reg=lattice_l2_torsion_reg, l1_laplacian_reg=lattice_l1_laplacian_reg, l2_laplacian_reg=lattice_l2_laplacian_reg) if projs: projections += projs regularization = tools.add_if_not_none(regularization, reg) # Step 4. Take average to make a final prediction. with variable_scope.variable_scope('ensemble_average'): output = variable_scope.get_variable( name='ensemble_bias', initializer=[0.0] * output_dim, dtype=calibrated_input_tensor.dtype) if m_lattice_outputs: output += math_ops.divide(math_ops.add_n(m_lattice_outputs), monotonic_num_lattices) if n_lattice_outputs is not None: output += math_ops.divide(math_ops.add_n(n_lattice_outputs), non_monotonic_num_lattices) return (output, projections, regularization)