def testMakeSparseSplitDefaultDirectionIsStable(self): """Tests default direction is stable when no sparsity.""" random.seed(1123) for _ in range(50): with self.test_session() as sess: grad = random.random() hessian = random.random() # The data looks like the following (divide by the num of steps 2). # Gradients | Partition | bucket ID | # (grad, hessian) | 0 | -1 | # And then 100 buckets of # (grad/100, hessian/100), so there is no sparsity. n_buckets = 100 # 1 for the overall sum, and 100 buckets. partition_ids = array_ops.constant( [0] * (n_buckets + 1), dtype=dtypes.int32) # We have only 1 dimension in our sparse feature column. bucket_ids = [-1] + [n for n in range(100)] bucket_ids = array_ops.constant(bucket_ids, dtype=dtypes.int64) dimension_ids = array_ops.constant( [0] * (n_buckets + 1), dtype=dtypes.int64) bucket_ids = array_ops.stack([bucket_ids, dimension_ids], axis=1) gradients = [grad] + [grad / n_buckets] * n_buckets gradients = array_ops.constant(gradients) hessians = [hessian] + [hessian / n_buckets] * n_buckets hessians = array_ops.constant(hessians) boundaries = [x * 1 for x in range(n_buckets + 1)] bucket_boundaries = array_ops.constant(boundaries, dtype=dtypes.float32) partitions, gains, splits = ( split_handler_ops.build_sparse_inequality_splits( num_minibatches=2, partition_ids=partition_ids, bucket_ids=bucket_ids, gradients=gradients, hessians=hessians, bucket_boundaries=bucket_boundaries, l1_regularization=0, l2_regularization=2, tree_complexity_regularization=0, min_node_weight=0, feature_column_group_id=0, bias_feature_id=-1, class_id=-1, multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS)) partitions, gains, splits = (sess.run([partitions, gains, splits])) self.assertAllEqual([0], partitions) self.assertEqual(1, len(splits)) split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) self.assertTrue( split_info.split_node.HasField( 'sparse_float_binary_split_default_left'))
def testMakeMulticlassSparseSplit(self): """Tests split handler op.""" with self.test_session() as sess: partition_ids = array_ops.constant([0, 0, 0, 1, 1], dtype=dtypes.int32) bucket_ids = array_ops.constant( [[-1, 0], [0, 0], [1, 0], [-1, 0], [1, 0]], dtype=dtypes.int64) gradients = array_ops.constant([[1.8, 3.5], [2.4, 1.0], [0.4, 4.0], [8.0, 3.1], [8.0, 0.8]]) hessian_0 = [[0.78, 1], [12, 1]] hessian_1 = [[0.4, 1], [1, 1]] hessian_2 = [[0.24, 1], [1, 1]] hessian_3 = [[0.26, 1], [1, 1]] hessian_4 = [[0.26, 1], [1, 1]] hessians = array_ops.constant( [hessian_0, hessian_1, hessian_2, hessian_3, hessian_4]) bucket_boundaries = array_ops.constant([0.3, 0.52]) partitions, gains, splits = ( split_handler_ops.build_sparse_inequality_splits( num_minibatches=2, partition_ids=partition_ids, bucket_ids=bucket_ids, gradients=gradients, hessians=hessians, bucket_boundaries=bucket_boundaries, l1_regularization=0, l2_regularization=2, tree_complexity_regularization=0, min_node_weight=0, feature_column_group_id=0, bias_feature_id=-1, class_id=-1, multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN)) partitions, gains, splits = (sess.run([partitions, gains, splits])) split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.sparse_float_binary_split_default_right # Each leaf has 2 element vector. self.assertEqual(2, len(left_child.value)) self.assertEqual(2, len(right_child.value)) self.assertEqual(0, split_node.split.feature_column) self.assertAllClose(0.52, split_node.split.threshold)
def testMakeMulticlassCategoricalEqualitySplit(self): """Tests split handler op for categorical equality split in multiclass.""" with self.cached_session() as sess: gradients = array_ops.constant([[1.8, 3.5], [2.4, 1.0], [0.4, 4.0], [9.0, 3.1], [3.0, 0.8]]) hessian_0 = [[0.78, 1], [12, 1]] hessian_1 = [[0.4, 1], [1, 1]] hessian_2 = [[0.24, 1], [1, 1]] hessian_3 = [[0.16, 2], [-1, 1]] hessian_4 = [[0.6, 1], [2, 1]] hessians = array_ops.constant( [hessian_0, hessian_1, hessian_2, hessian_3, hessian_4]) partition_ids = [0, 0, 0, 1, 1] feature_ids = array_ops.constant( [[-1, 0], [1, 0], [2, 0], [-1, 0], [1, 0]], dtype=dtypes.int64) partitions, gains, splits = ( split_handler_ops.build_categorical_equality_splits( num_minibatches=2, partition_ids=partition_ids, feature_ids=feature_ids, gradients=gradients, hessians=hessians, l1_regularization=0.1, l2_regularization=1, tree_complexity_regularization=0, min_node_weight=0, feature_column_group_id=0, bias_feature_id=-1, class_id=-1, multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN, weak_learner_type=learner_pb2.LearnerConfig. NORMAL_DECISION_TREE)) partitions, gains, splits = sess.run([partitions, gains, splits]) self.assertAllEqual([0, 1], partitions) split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.categorical_id_binary_split # Each leaf has 2 element vector. self.assertEqual(2, len(left_child.value)) self.assertEqual(2, len(right_child.value)) self.assertEqual(0, split_node.feature_column) self.assertEqual(1, split_node.feature_id)
def testMakeMulticlassDenseSplit(self): """Tests split handler op.""" with self.cached_session() as sess: partition_ids = array_ops.constant([0, 0, 1], dtype=dtypes.int32) bucket_ids = array_ops.constant([[0, 0], [1, 0], [1, 0]], dtype=dtypes.int64) gradients = array_ops.constant([[2.4, 3.0], [-0.6, 0.1], [8.0, 1.0]]) hessians = array_ops.constant([[[0.4, 1], [1, 1]], [[0.38, 1], [1, 1]], [[0.26, 1], [1, 1]]]) bucket_boundaries = [0.3, 0.52] partitions, gains, splits = ( split_handler_ops.build_dense_inequality_splits( num_minibatches=2, partition_ids=partition_ids, bucket_ids=bucket_ids, gradients=gradients, hessians=hessians, bucket_boundaries=bucket_boundaries, l1_regularization=0, l2_regularization=1, tree_complexity_regularization=0, min_node_weight=0, class_id=-1, feature_column_group_id=0, multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN, weak_learner_type=learner_pb2.LearnerConfig. NORMAL_DECISION_TREE)) partitions, gains, splits = sess.run([partitions, gains, splits]) self.assertAllEqual([0, 1], partitions) split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.dense_float_binary_split # Each leaf has 2 element vector. self.assertEqual(2, len(left_child.value)) self.assertEqual(2, len(right_child.value)) self.assertEqual(0, split_node.feature_column) self.assertAllClose(0.3, split_node.threshold, 1e-6)
def testGenerateFeatureSplitCandidatesSumReduction(self): with self.test_session() as sess: # The data looks like the following: # Example | Gradients | Partition | Feature ID | # i0 | (0.2, 0.12) | 0 | 1,2 | # i1 | (-0.5, 0.07) | 0 | | # i2 | (1.2, 0.2) | 0 | 2 | # i3 | (4.0, 0.13) | 1 | 1 | gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) partition_ids = [0, 0, 0, 1] indices = [[0, 0], [0, 1], [2, 0], [3, 0]] values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64) gradient_shape = tensor_shape.scalar() hessian_shape = tensor_shape.scalar() class_id = -1 split_handler = categorical_split_handler.EqualitySplitHandler( l1_regularization=0.1, l2_regularization=1, tree_complexity_regularization=0, min_node_weight=0, sparse_int_column=sparse_tensor.SparseTensor( indices, values, [4, 1]), feature_column_group_id=0, gradient_shape=gradient_shape, hessian_shape=hessian_shape, multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, init_stamp_token=0, loss_uses_sum_reduction=True) resources.initialize_resources(resources.shared_resources()).run() empty_gradients, empty_hessians = get_empty_tensors( gradient_shape, hessian_shape) example_weights = array_ops.ones([4, 1], dtypes.float32) update_1 = split_handler.update_stats_sync( 0, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) update_2 = split_handler.update_stats_sync( 0, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_1, update_2]): are_splits_ready, partitions, gains, splits = ( split_handler.make_splits(0, 1, class_id)) are_splits_ready, partitions, gains, splits = (sess.run( [are_splits_ready, partitions, gains, splits])) self.assertTrue(are_splits_ready) self.assertAllEqual([0, 1], partitions) # Check the split on partition 0. # -(0.4 + 2.4 - 0.1) / (0.24 + 0.4 + 1) expected_left_weight = -1.6463414634146338 # (0.4 + 2.4 - 0.1) ** 2 / (0.24 + 0.4 + 1) expected_left_gain = 4.445121951219511 # -(-1 + 0.1) / (0.14 + 1) expected_right_weight = 0.789473684211 # (-1 + 0.1) ** 2 / (0.14 + 1) expected_right_gain = 0.710526315789 # (0.4 + -1 + 2.4 - 0.1) ** 2 / (0.24 + 0.14 + 0.4 + 1) expected_bias_gain = 1.6235955056179772 split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.categorical_id_binary_split self.assertEqual(0, split_node.feature_column) self.assertEqual(2, split_node.feature_id) self.assertAllClose( expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) # Check the split on partition 1. # (-8 + 0.1) / (0.26 + 1) expected_left_weight = -6.26984126984 # (-8 + 0.1) ** 2 / (0.26 + 1) expected_left_gain = 49.5317460317 expected_right_weight = 0 expected_right_gain = 0 # (-8 + 0.1) ** 2 / (0.26 + 1) expected_bias_gain = 49.5317460317 # Verify candidate for partition 1, there's only one active feature here # so zero gain is expected. split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.categorical_id_binary_split self.assertAllClose(0.0, gains[1], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) self.assertEqual(0, split_node.feature_column) self.assertEqual(1, split_node.feature_id)
def testMakeCategoricalEqualitySplit(self): """Tests split handler op for categorical equality split.""" with self.test_session() as sess: # The data looks like the following after dividing by number of steps (2). # Gradients | Partition | Feature ID | # (0.9, 0.39) | 0 | -1 | # (0.2, 0.12) | 0 | 1 | # (1.4, 0.32) | 0 | 2 | # (4.0, 0.13) | 1 | -1 | # (4.0, 0.13) | 1 | 1 | gradients = [1.8, 0.4, 2.8, 8.0, 8.0] hessians = [0.78, 0.24, 0.64, 0.26, 0.26] partition_ids = [0, 0, 0, 1, 1] feature_ids = array_ops.constant( [[-1, 0], [1, 0], [2, 0], [-1, 0], [1, 0]], dtype=dtypes.int64) partitions, gains, splits = ( split_handler_ops.build_categorical_equality_splits( num_minibatches=2, partition_ids=partition_ids, feature_ids=feature_ids, gradients=gradients, hessians=hessians, l1_regularization=0.1, l2_regularization=1, tree_complexity_regularization=0, min_node_weight=0, feature_column_group_id=0, bias_feature_id=-1, class_id=-1, multiclass_strategy=learner_pb2.LearnerConfig. TREE_PER_CLASS)) partitions, gains, splits = sess.run([partitions, gains, splits]) self.assertAllEqual([0, 1], partitions) # Check the split on partition 0. # -(0.2 + 1.2 - 0.1) / (0.12 + 0.2 + 1) expected_left_weight = -0.9848484848484846 # (0.2 + 1.2 - 0.1) ** 2 / (0.12 + 0.2 + 1) expected_left_gain = 1.2803030303030298 # -(-0.5 + 0.1) / (0.07 + 1) expected_right_weight = 0.37383177570093457 # (-0.5 + 0.1) ** 2 / (0.07 + 1) expected_right_gain = 0.14953271028037385 # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) expected_bias_gain = 0.46043165467625885 split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.categorical_id_binary_split self.assertEqual(0, split_node.feature_column) self.assertEqual(2, split_node.feature_id) self.assertAllClose( expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) # Check the split on partition 1. # (-4 + 0.1) / (0.13 + 1) expected_left_weight = -3.4513274336283186 # (-4 + 0.1) ** 2 / (0.13 + 1) expected_left_gain = 13.460176991150442 expected_right_weight = 0 expected_right_gain = 0 # (-4 + 0.1) ** 2 / (0.13 + 1) expected_bias_gain = 13.460176991150442 # Verify candidate for partition 1, there's only one active feature here # so zero gain is expected. split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.categorical_id_binary_split self.assertAllClose(0.0, gains[1], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) self.assertEqual(0, split_node.feature_column) self.assertEqual(1, split_node.feature_id)
def testMakeDenseSplit(self): """Tests split handler op.""" with self.test_session() as sess: # The data looks like the following after dividing by number of steps (2). # Gradients | Partition | Dense Quantile | # (1.2, 0.2) | 0 | 0 | # (-0.3, 0.19) | 0 | 1 | # (4.0, 0.13) | 1 | 1 | partition_ids = array_ops.constant([0, 0, 1], dtype=dtypes.int32) bucket_ids = array_ops.constant([[0, 0], [1, 0], [1, 0]], dtype=dtypes.int64) gradients = array_ops.constant([2.4, -0.6, 8.0]) hessians = array_ops.constant([0.4, 0.38, 0.26]) bucket_boundaries = [0.3, 0.52] partitions, gains, splits = ( split_handler_ops.build_dense_inequality_splits( num_minibatches=2, partition_ids=partition_ids, bucket_ids=bucket_ids, gradients=gradients, hessians=hessians, bucket_boundaries=bucket_boundaries, l1_regularization=0.1, l2_regularization=1, tree_complexity_regularization=0, min_node_weight=0, class_id=-1, feature_column_group_id=0, multiclass_strategy=learner_pb2.LearnerConfig. TREE_PER_CLASS)) partitions, gains, splits = sess.run([partitions, gains, splits]) self.assertAllEqual([0, 1], partitions) # Check the split on partition 0. # -(1.2 - 0.1) / (0.2 + 1) expected_left_weight = -0.91666 # expected_left_weight * -(1.2 - 0.1) expected_left_gain = 1.0083333333333331 # (-0.3 + 0.1) / (0.19 + 1) expected_right_weight = 0.1680672 # expected_right_weight * -(-0.3 + 0.1) expected_right_gain = 0.033613445378151252 # (-0.3 + 1.2 - 0.1) ** 2 / (0.19 + 0.2 + 1) expected_bias_gain = 0.46043165467625885 split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.dense_float_binary_split self.assertAllClose( expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) self.assertEqual(0, split_node.feature_column) self.assertAllClose(0.3, split_node.threshold, 0.00001) # Check the split on partition 1. # (-4 + 0.1) / (0.13 + 1) expected_left_weight = -3.4513274336283186 expected_right_weight = 0 split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.dense_float_binary_split # There's only one active bucket here so zero gain is expected. self.assertAllClose(0.0, gains[1], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) self.assertEqual(0, split_node.feature_column) self.assertAllClose(0.52, split_node.threshold, 0.00001)
def testMakeSparseMultidimensionalSplit(self): """Tests split handler op.""" with self.test_session() as sess: # Num of steps is 2. # The feature column is three dimensional. # First dimension has bias bucket only, the second has bias bucket and # two valid buckets, the third has just one bias bucket and one valid # bucket. # Gradients | Partition | Dimension | bucket ID | # (0.9, 0.39) | 0 | 0 | -1 | # (1.2, 0.2) | 0 | 1 | 0 | # (0.2, 0.12) | 0 | 1 | 2 | # (0.1, 0.1) | 0 | 2 | 3 | # Now second node - nothing interesting there, just one dimension. # Second node has the same bucket ids for all dimensions. # (4.0, 0.13) | 1 | 0 | -1 | # (4.0, 0.13) | 1 | 2 | 3 | # Tree node ids. partition_ids = array_ops.constant([0, 0, 0, 0, 1, 1], dtype=dtypes.int32) dimension_ids = array_ops.constant([0, 1, 1, 2, 0, 2], dtype=dtypes.int64) bucket_ids = array_ops.constant([-1, 0, 2, 3, -1, 3], dtype=dtypes.int64) bucket_ids = array_ops.stack([bucket_ids, dimension_ids], axis=1) gradients = array_ops.constant([1.8, 2.4, 0.4, 0.2, 8.0, 8.0]) hessians = array_ops.constant([0.78, 0.4, 0.24, 0.2, 0.26, 0.26]) bucket_boundaries = array_ops.constant([0.3, 0.52, 0.58, 0.6]) partitions, gains, splits = ( split_handler_ops.build_sparse_inequality_splits( num_minibatches=2, partition_ids=partition_ids, bucket_ids=bucket_ids, gradients=gradients, hessians=hessians, bucket_boundaries=bucket_boundaries, l1_regularization=0, l2_regularization=2, tree_complexity_regularization=0, min_node_weight=0, feature_column_group_id=0, bias_feature_id=-1, class_id=-1, multiclass_strategy=learner_pb2.LearnerConfig. TREE_PER_CLASS)) partitions, gains, splits = (sess.run([partitions, gains, splits])) self.assertAllEqual([0, 1], partitions) self.assertEqual(2, len(splits)) # Check the split on node 0 - it should split on second dimension # -(0.2 + 1.2) / (0.12 + 0.2 + 2) expected_left_weight = -0.603448275862069 # (0.2 + 1.2) ** 2 / (0.12 + 0.2 + 2) expected_left_gain = 0.8448275862068965 # 0.5 / (0.07 + 2) expected_right_weight = 0.24154589371980678 # 0.5 ** 2 / (0.07 + 2) expected_right_gain = 0.12077294685990339 # (0.2 + 1.2 - 0.5) ** 2 / (0.12 + 0.2 + 0.07 + 2) expected_bias_gain = 0.3389121338912133 split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.sparse_float_binary_split_default_right self.assertAllClose( expected_left_gain + expected_right_gain - expected_bias_gain, gains[0]) self.assertAllClose([expected_left_weight], left_child.value) self.assertAllClose([expected_right_weight], right_child.value) self.assertEqual(0, split_node.split.feature_column) # Split happened on second dimension. self.assertEqual(1, split_node.split.dimension_id) self.assertAllClose(0.58, split_node.split.threshold) # Check the split on partition 1. expected_left_weight = -1.8779342723004695 expected_right_weight = 0 # Verify candidate for partition 1, there's only one active bucket here # so zero gain is expected. split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.sparse_float_binary_split_default_left self.assertAllClose(0.0, gains[1]) self.assertAllClose([expected_left_weight], left_child.value) self.assertAllClose([expected_right_weight], right_child.value) self.assertEqual(0, split_node.split.feature_column) self.assertEqual(2, split_node.split.dimension_id) self.assertAllClose(0.6, split_node.split.threshold)
def testMakeSparseSplit(self): """Tests split handler op.""" with self.test_session() as sess: # The data looks like the following after dividing by number of steps (2). # Gradients | Partition | bucket ID | # (0.9, 0.39) | 0 | -1 | # (1.2, 0.2) | 0 | 0 | # (0.2, 0.12) | 0 | 1 | # (4.0, 0.13) | 1 | -1 | # (4.0, 0.13) | 1 | 1 | partition_ids = array_ops.constant([0, 0, 0, 1, 1], dtype=dtypes.int32) # We have only 1 dimension in our sparse feature column. bucket_ids = array_ops.constant([-1, 0, 1, -1, 1], dtype=dtypes.int64) dimension_ids = array_ops.constant([0, 0, 0, 0, 0], dtype=dtypes.int64) bucket_ids = array_ops.stack([bucket_ids, dimension_ids], axis=1) gradients = array_ops.constant([1.8, 2.4, 0.4, 8.0, 8.0]) hessians = array_ops.constant([0.78, 0.4, 0.24, 0.26, 0.26]) bucket_boundaries = array_ops.constant([0.3, 0.52]) partitions, gains, splits = ( split_handler_ops.build_sparse_inequality_splits( num_minibatches=2, partition_ids=partition_ids, bucket_ids=bucket_ids, gradients=gradients, hessians=hessians, bucket_boundaries=bucket_boundaries, l1_regularization=0, l2_regularization=2, tree_complexity_regularization=0, min_node_weight=0, feature_column_group_id=0, bias_feature_id=-1, class_id=-1, multiclass_strategy=learner_pb2.LearnerConfig. TREE_PER_CLASS)) partitions, gains, splits = (sess.run([partitions, gains, splits])) self.assertAllEqual([0, 1], partitions) self.assertEqual(2, len(splits)) # Check the split on partition 0. # -(0.2 + 1.2) / (0.12 + 0.2 + 2) expected_left_weight = -0.603448275862069 # (0.2 + 1.2) ** 2 / (0.12 + 0.2 + 2) expected_left_gain = 0.8448275862068965 # 0.5 / (0.07 + 2) expected_right_weight = 0.24154589371980678 # 0.5 ** 2 / (0.07 + 2) expected_right_gain = 0.12077294685990339 # (0.2 + 1.2 - 0.5) ** 2 / (0.12 + 0.2 + 0.07 + 2) expected_bias_gain = 0.3389121338912133 split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.sparse_float_binary_split_default_right self.assertAllClose( expected_left_gain + expected_right_gain - expected_bias_gain, gains[0]) self.assertAllClose([expected_left_weight], left_child.value) self.assertAllClose([expected_right_weight], right_child.value) self.assertEqual(0, split_node.split.feature_column) # Sparse is one dimensional. self.assertEqual(0, split_node.split.dimension_id) self.assertAllClose(0.52, split_node.split.threshold) # Check the split on partition 1. expected_left_weight = -1.8779342723004695 expected_right_weight = 0 # Verify candidate for partition 1, there's only one active bucket here # so zero gain is expected. split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.sparse_float_binary_split_default_left self.assertAllClose(0.0, gains[1]) self.assertAllClose([expected_left_weight], left_child.value) self.assertAllClose([expected_right_weight], right_child.value) self.assertEqual(0, split_node.split.feature_column) # Sparse is one dimensional. self.assertEqual(0, split_node.split.dimension_id) self.assertAllClose(0.52, split_node.split.threshold)
def testGenerateFeatureSplitCandidates(self): with self.test_session() as sess: # The data looks like the following: # Example | Gradients | Partition | Sparse Quantile | # i0 | (0.2, 0.12) | 0 | 1 | # i1 | (-0.5, 0.07) | 0 | N/A | # i2 | (1.2, 0.2) | 0 | 0 | # i3 | (4.0, 0.13) | 1 | 1 | gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) example_partitions = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) indices = array_ops.constant([[0, 0], [2, 0], [3, 0]], dtype=dtypes.int64) values = array_ops.constant([0.52, 0.3, 0.52]) sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1]) gradient_shape = tensor_shape.scalar() hessian_shape = tensor_shape.scalar() class_id = -1 split_handler = ordinal_split_handler.SparseSplitHandler( l1_regularization=0, l2_regularization=2, tree_complexity_regularization=0, min_node_weight=0, epsilon=0.01, num_quantiles=2, feature_column_group_id=0, sparse_float_column=sparse_column, init_stamp_token=0, gradient_shape=gradient_shape, hessian_shape=hessian_shape, multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) resources.initialize_resources(resources.shared_resources()).run() empty_gradients, empty_hessians = get_empty_tensors( gradient_shape, hessian_shape) example_weights = array_ops.ones([4, 1], dtypes.float32) update_1 = split_handler.update_stats_sync( 0, example_partitions, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_1]): are_splits_ready = split_handler.make_splits(0, 1, class_id)[0] with ops.control_dependencies([are_splits_ready]): update_2 = split_handler.update_stats_sync( 1, example_partitions, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_2]): are_splits_ready2, partitions, gains, splits = ( split_handler.make_splits(1, 2, class_id)) are_splits_ready, are_splits_ready2, partitions, gains, splits = ( sess.run([ are_splits_ready, are_splits_ready2, partitions, gains, splits ])) # During the first iteration, inequality split handlers are not going to # have any splits. Make sure that we return not_ready in that case. self.assertFalse(are_splits_ready) self.assertTrue(are_splits_ready2) self.assertAllEqual([0, 1], partitions) # Check the split on partition 0. # -(0.2 + 1.2) / (0.12 + 0.2 + 2) expected_left_weight = -0.603448275862069 # (0.2 + 1.2) ** 2 / (0.12 + 0.2 + 2) expected_left_gain = 0.8448275862068965 # 0.5 / (0.07 + 2) expected_right_weight = 0.24154589371980678 # 0.5 ** 2 / (0.07 + 2) expected_right_gain = 0.12077294685990339 # (0.2 + 1.2 - 0.5) ** 2 / (0.12 + 0.2 + 0.07 + 2) expected_bias_gain = 0.3389121338912133 split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.sparse_float_binary_split_default_right self.assertAllClose( expected_left_gain + expected_right_gain - expected_bias_gain, gains[0]) self.assertAllClose([expected_left_weight], left_child.value) self.assertAllClose([expected_right_weight], right_child.value) self.assertEqual(0, split_node.split.feature_column) self.assertAllClose(0.52, split_node.split.threshold) # Check the split on partition 1. expected_left_weight = -1.8779342723004695 expected_right_weight = 0 # Verify candidate for partition 1, there's only one active bucket here # so zero gain is expected. split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.sparse_float_binary_split_default_left self.assertAllClose(0.0, gains[1]) self.assertAllClose([expected_left_weight], left_child.value) self.assertAllClose([expected_right_weight], right_child.value) self.assertEqual(0, split_node.split.feature_column) self.assertAllClose(0.52, split_node.split.threshold)
def testGenerateFeatureSplitCandidatesWithMinNodeWeight(self): with self.test_session() as sess: # The data looks like the following: # Example | Gradients | Partition | Dense Quantile | # i0 | (0.2, 0.12) | 0 | 1 | # i1 | (-0.5, 0.07) | 0 | 1 | # i2 | (1.2, 0.2) | 0 | 0 | # i3 | (4.0, 2.0) | 1 | 1 | dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) hessians = array_ops.constant([0.12, 0.07, 0.2, 2]) partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) gradient_shape = tensor_shape.scalar() hessian_shape = tensor_shape.scalar() class_id = -1 split_handler = ordinal_split_handler.DenseSplitHandler( l1_regularization=0.1, l2_regularization=1, tree_complexity_regularization=0.5, min_node_weight=1.5, epsilon=0.001, num_quantiles=10, feature_column_group_id=0, dense_float_column=dense_column, init_stamp_token=0, gradient_shape=gradient_shape, hessian_shape=hessian_shape, multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) resources.initialize_resources(resources.shared_resources()).run() empty_gradients, empty_hessians = get_empty_tensors( gradient_shape, hessian_shape) example_weights = array_ops.ones([4, 1], dtypes.float32) update_1 = split_handler.update_stats_sync( 0, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_1]): are_splits_ready = split_handler.make_splits(0, 1, class_id)[0] with ops.control_dependencies([are_splits_ready]): update_2 = split_handler.update_stats_sync( 1, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_2]): are_splits_ready2, partitions, gains, splits = ( split_handler.make_splits(1, 2, class_id)) are_splits_ready, are_splits_ready2, partitions, gains, splits = ( sess.run([ are_splits_ready, are_splits_ready2, partitions, gains, splits ])) # During the first iteration, inequality split handlers are not going to # have any splits. Make sure that we return not_ready in that case. self.assertFalse(are_splits_ready) self.assertTrue(are_splits_ready2) self.assertAllEqual([0, 1], partitions) # Check the gain on partition 0 to be -0.5. split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.dense_float_binary_split # Make sure the gain is subtracted by the tree complexity regularization. self.assertAllClose(-0.5, gains[0], 0.00001) self.assertEqual(0, split_node.feature_column) # Check the split on partition 1. # (-4 + 0.1) / (2 + 1) expected_left_weight = -1.3 expected_right_weight = 0 # Verify candidate for partition 1, there's only one active bucket here # so -0.5 gain is expected (because of tree complexity. split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.dense_float_binary_split self.assertAllClose(-0.5, gains[1], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) self.assertEqual(0, split_node.feature_column) self.assertAllClose(0.52, split_node.threshold, 0.00001)
def testGenerateFeatureSplitCandidates(self): with self.test_session() as sess: # The data looks like the following: # Example | Gradients | Partition | Dense Quantile | # i0 | (0.2, 0.12) | 0 | 1 | # i1 | (-0.5, 0.07) | 0 | 1 | # i2 | (1.2, 0.2) | 0 | 0 | # i3 | (4.0, 0.13) | 1 | 1 | dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) class_id = -1 gradient_shape = tensor_shape.scalar() hessian_shape = tensor_shape.scalar() split_handler = ordinal_split_handler.DenseSplitHandler( l1_regularization=0.1, l2_regularization=1, tree_complexity_regularization=0, min_node_weight=0, epsilon=0.001, num_quantiles=10, feature_column_group_id=0, dense_float_column=dense_column, init_stamp_token=0, gradient_shape=gradient_shape, hessian_shape=hessian_shape, multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) resources.initialize_resources(resources.shared_resources()).run() empty_gradients, empty_hessians = get_empty_tensors( gradient_shape, hessian_shape) example_weights = array_ops.ones([4, 1], dtypes.float32) update_1 = split_handler.update_stats_sync( 0, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_1]): are_splits_ready = split_handler.make_splits(0, 1, class_id)[0] with ops.control_dependencies([are_splits_ready]): update_2 = split_handler.update_stats_sync( 1, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_2]): are_splits_ready2, partitions, gains, splits = ( split_handler.make_splits(1, 2, class_id)) are_splits_ready, are_splits_ready2, partitions, gains, splits = ( sess.run([ are_splits_ready, are_splits_ready2, partitions, gains, splits ])) # During the first iteration, inequality split handlers are not going to # have any splits. Make sure that we return not_ready in that case. self.assertFalse(are_splits_ready) self.assertTrue(are_splits_ready2) self.assertAllEqual([0, 1], partitions) # Check the split on partition 0. # -(1.2 - 0.1) / (0.2 + 1) expected_left_weight = -0.91666 # expected_left_weight * -(1.2 - 0.1) expected_left_gain = 1.0083333333333331 # (-0.5 + 0.2 + 0.1) / (0.19 + 1) expected_right_weight = 0.1680672 # expected_right_weight * -(-0.5 + 0.2 + 0.1)) expected_right_gain = 0.033613445378151252 # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) expected_bias_gain = 0.46043165467625885 split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.dense_float_binary_split self.assertAllClose( expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) self.assertEqual(0, split_node.feature_column) self.assertAllClose(0.3, split_node.threshold, 0.00001) # Check the split on partition 1. # (-4 + 0.1) / (0.13 + 1) expected_left_weight = -3.4513274336283186 # (-4 + 0.1) ** 2 / (0.13 + 1) expected_left_gain = 13.460176991150442 expected_right_weight = 0 expected_right_gain = 0 # (-4 + 0.1) ** 2 / (0.13 + 1) expected_bias_gain = 13.460176991150442 # Verify candidate for partition 1, there's only one active bucket here # so zero gain is expected. split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.dense_float_binary_split self.assertAllClose(0.0, gains[1], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) self.assertEqual(0, split_node.feature_column) self.assertAllClose(0.52, split_node.threshold, 0.00001)
def testGenerateFeatureSplitCandidatesMulticlassDiagonalHessian(self): with self.test_session() as sess: dense_column = array_ops.constant([0.52, 0.52, 0.3, 0.52]) # Batch size is 4, 2 gradients per each instance. gradients = array_ops.constant( [[0.2, 0.1], [-0.5, 0.2], [1.2, 3.4], [4.0, -3.5]], shape=[4, 2]) # Each hessian is a diagonal of a full hessian matrix. hessian_0 = [0.12, 0.11] hessian_1 = [0.07, 0.2] hessian_2 = [0.2, 0.9] hessian_3 = [0.13, 2.2] hessians = array_ops.constant( [hessian_0, hessian_1, hessian_2, hessian_3]) partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) class_id = -1 gradient_shape = tensor_shape.TensorShape([2]) hessian_shape = tensor_shape.TensorShape([2]) split_handler = ordinal_split_handler.DenseSplitHandler( l1_regularization=0, l2_regularization=1, tree_complexity_regularization=0, min_node_weight=0, epsilon=0.001, num_quantiles=3, feature_column_group_id=0, dense_float_column=dense_column, init_stamp_token=0, gradient_shape=gradient_shape, hessian_shape=hessian_shape, multiclass_strategy=learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) resources.initialize_resources(resources.shared_resources()).run() empty_gradients, empty_hessians = get_empty_tensors( gradient_shape, hessian_shape) example_weights = array_ops.ones([4, 1], dtypes.float32) update_1 = split_handler.update_stats_sync( 0, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_1]): are_splits_ready = split_handler.make_splits(0, 1, class_id)[0] with ops.control_dependencies([are_splits_ready]): update_2 = split_handler.update_stats_sync( 1, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_2]): are_splits_ready2, partitions, gains, splits = ( split_handler.make_splits(1, 2, class_id)) are_splits_ready, are_splits_ready2, partitions, gains, splits = ( sess.run([ are_splits_ready, are_splits_ready2, partitions, gains, splits ])) # During the first iteration, inequality split handlers are not going to # have any splits. Make sure that we return not_ready in that case. self.assertFalse(are_splits_ready) self.assertTrue(are_splits_ready2) split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.dense_float_binary_split # Each leaf has 2 element vector. self.assertEqual(2, len(left_child.value)) self.assertEqual(2, len(right_child.value)) self.assertEqual(0, split_node.feature_column) self.assertAllClose(0.3, split_node.threshold, 1e-6)
def testDegenerativeCase(self): with self.test_session() as sess: # One data example only, one leaf and thus one quantile bucket.The same # situation is when all examples have the same values. This case was # causing before a failure. gradients = array_ops.constant([0.2]) hessians = array_ops.constant([0.12]) example_partitions = array_ops.constant([1], dtype=dtypes.int32) indices = array_ops.constant([[0, 0]], dtype=dtypes.int64) values = array_ops.constant([0.58]) sparse_column = sparse_tensor.SparseTensor(indices, values, [1, 1]) gradient_shape = tensor_shape.scalar() hessian_shape = tensor_shape.scalar() class_id = -1 split_handler = ordinal_split_handler.SparseSplitHandler( l1_regularization=0, l2_regularization=2, tree_complexity_regularization=0, min_node_weight=0, epsilon=0.01, num_quantiles=2, feature_column_group_id=0, sparse_float_column=sparse_column, init_stamp_token=0, gradient_shape=gradient_shape, hessian_shape=hessian_shape, multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS) resources.initialize_resources(resources.shared_resources()).run() empty_gradients, empty_hessians = get_empty_tensors( gradient_shape, hessian_shape) example_weights = array_ops.ones([1, 1], dtypes.float32) update_1 = split_handler.update_stats_sync( 0, example_partitions, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_1]): are_splits_ready = split_handler.make_splits(0, 1, class_id)[0] with ops.control_dependencies([are_splits_ready]): update_2 = split_handler.update_stats_sync( 1, example_partitions, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_2]): are_splits_ready2, partitions, gains, splits = ( split_handler.make_splits(1, 2, class_id)) are_splits_ready, are_splits_ready2, partitions, gains, splits = ( sess.run([ are_splits_ready, are_splits_ready2, partitions, gains, splits ])) # During the first iteration, inequality split handlers are not going to # have any splits. Make sure that we return not_ready in that case. self.assertFalse(are_splits_ready) self.assertTrue(are_splits_ready2) self.assertAllEqual([1], partitions) self.assertAllEqual([0.0], gains) split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) split_node = split_info.split_node.sparse_float_binary_split_default_left self.assertEqual(0, split_node.split.feature_column) self.assertAllClose(0.58, split_node.split.threshold)
def testGenerateFeatureSplitCandidatesMulticlass(self): with self.test_session() as sess: # Batch size is 4, 2 gradients per each instance. gradients = array_ops.constant( [[0.2, 0.1], [-0.5, 0.2], [1.2, 3.4], [4.0, -3.5]], shape=[4, 2]) # 2x2 matrix for each instance hessian_0 = [[0.12, 0.02], [0.3, 0.11]] hessian_1 = [[0.07, -0.2], [-0.5, 0.2]] hessian_2 = [[0.2, -0.23], [-0.8, 0.9]] hessian_3 = [[0.13, -0.3], [-1.5, 2.2]] hessians = array_ops.constant( [hessian_0, hessian_1, hessian_2, hessian_3]) partition_ids = [0, 0, 0, 1] indices = [[0, 0], [0, 1], [2, 0], [3, 0]] values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64) hessians = array_ops.constant( [hessian_0, hessian_1, hessian_2, hessian_3]) partition_ids = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) gradient_shape = tensor_shape.TensorShape([2]) hessian_shape = tensor_shape.TensorShape([2, 2]) class_id = -1 split_handler = categorical_split_handler.EqualitySplitHandler( l1_regularization=0.1, l2_regularization=1, tree_complexity_regularization=0, min_node_weight=0, sparse_int_column=sparse_tensor.SparseTensor( indices, values, [4, 1]), feature_column_group_id=0, gradient_shape=gradient_shape, hessian_shape=hessian_shape, multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN, init_stamp_token=0) resources.initialize_resources(resources.shared_resources()).run() empty_gradients, empty_hessians = get_empty_tensors( gradient_shape, hessian_shape) example_weights = array_ops.ones([4, 1], dtypes.float32) update_1 = split_handler.update_stats_sync( 0, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_1]): are_splits_ready, partitions, gains, splits = ( split_handler.make_splits(0, 1, class_id)) are_splits_ready, partitions, gains, splits = (sess.run( [are_splits_ready, partitions, gains, splits])) self.assertTrue(are_splits_ready) self.assertAllEqual([0, 1], partitions) split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.categorical_id_binary_split # Each leaf has 2 element vector. self.assertEqual(2, len(left_child.value)) self.assertEqual(2, len(right_child.value)) self.assertEqual(1, split_node.feature_id) split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.categorical_id_binary_split self.assertEqual(2, len(left_child.value)) self.assertEqual(0, len(right_child.value)) self.assertEqual(1, split_node.feature_id)
def testGenerateFeatureSplitCandidates(self): with self.test_session() as sess: # The data looks like the following: # Example | Gradients | Partition | Feature ID | # i0 | (0.2, 0.12) | 0 | 1,2 | # i1 | (-0.5, 0.07) | 0 | | # i2 | (1.2, 0.2) | 0 | 2 | # i3 | (4.0, 0.13) | 1 | 1 | gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0]) hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13]) partition_ids = [0, 0, 0, 1] indices = [[0, 0], [0, 1], [2, 0], [3, 0]] values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64) gradient_shape = tensor_shape.scalar() hessian_shape = tensor_shape.scalar() class_id = -1 split_handler = categorical_split_handler.EqualitySplitHandler( l1_regularization=0.1, l2_regularization=1, tree_complexity_regularization=0, min_node_weight=0, sparse_int_column=sparse_tensor.SparseTensor( indices, values, [4, 1]), feature_column_group_id=0, gradient_shape=gradient_shape, hessian_shape=hessian_shape, multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS, init_stamp_token=0) resources.initialize_resources(resources.shared_resources()).run() empty_gradients, empty_hessians = get_empty_tensors( gradient_shape, hessian_shape) example_weights = array_ops.ones([4, 1], dtypes.float32) update_1 = split_handler.update_stats_sync( 0, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) update_2 = split_handler.update_stats_sync( 0, partition_ids, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_1, update_2]): are_splits_ready, partitions, gains, splits = ( split_handler.make_splits(0, 1, class_id)) are_splits_ready, partitions, gains, splits = (sess.run( [are_splits_ready, partitions, gains, splits])) self.assertTrue(are_splits_ready) self.assertAllEqual([0, 1], partitions) # Check the split on partition 0. # -(0.2 + 1.2 - 0.1) / (0.12 + 0.2 + 1) expected_left_weight = -0.9848484848484846 # (0.2 + 1.2 - 0.1) ** 2 / (0.12 + 0.2 + 1) expected_left_gain = 1.2803030303030298 # -(-0.5 + 0.1) / (0.07 + 1) expected_right_weight = 0.37383177570093457 # (-0.5 + 0.1) ** 2 / (0.07 + 1) expected_right_gain = 0.14953271028037385 # (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1) expected_bias_gain = 0.46043165467625885 split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.categorical_id_binary_split self.assertEqual(0, split_node.feature_column) self.assertEqual(2, split_node.feature_id) self.assertAllClose( expected_left_gain + expected_right_gain - expected_bias_gain, gains[0], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) # Check the split on partition 1. # (-4 + 0.1) / (0.13 + 1) expected_left_weight = -3.4513274336283186 # (-4 + 0.1) ** 2 / (0.13 + 1) expected_left_gain = 13.460176991150442 expected_right_weight = 0 expected_right_gain = 0 # (-4 + 0.1) ** 2 / (0.13 + 1) expected_bias_gain = 13.460176991150442 # Verify candidate for partition 1, there's only one active feature here # so zero gain is expected. split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.categorical_id_binary_split self.assertAllClose(0.0, gains[1], 0.00001) self.assertAllClose([expected_left_weight], left_child.value, 0.00001) self.assertAllClose([expected_right_weight], right_child.value, 0.00001) self.assertEqual(0, split_node.feature_column) self.assertEqual(1, split_node.feature_id)
def testGenerateFeatureSplitCandidatesMulticlassDiagonalHessian(self): with self.test_session() as sess: # Batch is 4, 2 classes gradients = array_ops.constant( [[0.2, 1.4], [-0.5, 0.1], [1.2, 3], [4.0, -3]]) # Each hessian is a diagonal from a full hessian matrix. hessian_0 = [0.12, 0.11] hessian_1 = [0.07, 0.2] hessian_2 = [0.2, 0.9] hessian_3 = [0.13, 2.2] hessians = array_ops.constant( [hessian_0, hessian_1, hessian_2, hessian_3]) example_partitions = array_ops.constant([0, 0, 0, 1], dtype=dtypes.int32) indices = array_ops.constant([[0, 0], [2, 0], [3, 0]], dtype=dtypes.int64) values = array_ops.constant([0.52, 0.3, 0.52]) sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1]) gradient_shape = tensor_shape.TensorShape([2]) hessian_shape = tensor_shape.TensorShape([2]) class_id = -1 split_handler = ordinal_split_handler.SparseSplitHandler( l1_regularization=0, l2_regularization=2, tree_complexity_regularization=0, min_node_weight=0, epsilon=0.01, num_quantiles=2, feature_column_group_id=0, sparse_float_column=sparse_column, init_stamp_token=0, gradient_shape=gradient_shape, hessian_shape=hessian_shape, multiclass_strategy=learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) resources.initialize_resources(resources.shared_resources()).run() empty_gradients, empty_hessians = get_empty_tensors( gradient_shape, hessian_shape) example_weights = array_ops.ones([4, 1], dtypes.float32) update_1 = split_handler.update_stats_sync( 0, example_partitions, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_1]): are_splits_ready = split_handler.make_splits(0, 1, class_id)[0] with ops.control_dependencies([are_splits_ready]): update_2 = split_handler.update_stats_sync( 1, example_partitions, gradients, hessians, empty_gradients, empty_hessians, example_weights, is_active=array_ops.constant([True, True])) with ops.control_dependencies([update_2]): are_splits_ready2, partitions, gains, splits = ( split_handler.make_splits(1, 2, class_id)) are_splits_ready, are_splits_ready2, partitions, gains, splits = ( sess.run([ are_splits_ready, are_splits_ready2, partitions, gains, splits ])) self.assertFalse(are_splits_ready) self.assertTrue(are_splits_ready2) split_info = split_info_pb2.SplitInfo() split_info.ParseFromString(splits[0]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.sparse_float_binary_split_default_right # Each leaf has 2 element vector. self.assertEqual(2, len(left_child.value)) self.assertEqual(2, len(right_child.value)) self.assertEqual(0, split_node.split.feature_column) self.assertAllClose(0.52, split_node.split.threshold) split_info.ParseFromString(splits[1]) left_child = split_info.left_child.vector right_child = split_info.right_child.vector split_node = split_info.split_node.sparse_float_binary_split_default_left self.assertEqual(2, len(left_child.value)) self.assertEqual(0, split_node.split.feature_column) self.assertAllClose(0.52, split_node.split.threshold)