def loss_fn(labels, logits): result = losses.per_example_maxent_loss( labels=labels, logits=logits, weights=weight_column, num_classes=n_classes) return result[0]
def loss_fn(labels, logits, weights=None): result = losses.per_example_maxent_loss( labels=labels, logits=logits, weights=weights, num_classes=n_classes) return math_ops.reduce_mean(result[0])
def loss_fn(labels, logits): result = losses.per_example_maxent_loss( # Don't pass the weights: head already multiplies by them. labels=labels, logits=logits, weights=None, num_classes=n_classes) return result[0]
def tree_loss_fn(labels, logits): result = bt_losses.per_example_maxent_loss( labels=labels, logits=logits, num_classes=n_classes, weights=None) return result[0]
def testTrainFnMulticlassTreePerClass(self): """Tests the GBDT train for multiclass tree per class strategy.""" with self.test_session() as sess: ensemble_handle = model_ops.tree_ensemble_variable( stamp_token=0, tree_ensemble_config="", name="tree_ensemble") learner_config = learner_pb2.LearnerConfig() learner_config.learning_rate_tuner.fixed.learning_rate = 1 # Use full hessian multiclass strategy. learner_config.multi_class_strategy = ( learner_pb2.LearnerConfig.TREE_PER_CLASS) learner_config.num_classes = 5 learner_config.regularization.l1 = 0 # To make matrix inversible. learner_config.regularization.l2 = 1e-5 learner_config.constraints.max_tree_depth = 1 learner_config.constraints.min_node_weight = 0 features = { "dense_float": array_ops.constant( [[1.0], [1.5], [2.0]], dtypes.float32), } gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( is_chief=True, num_ps_replicas=0, center_bias=False, ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, features=features) batch_size = 3 predictions = array_ops.constant( [[0.0, -1.0, 0.5, 1.2, 3.1], [1.0, 0.0, 0.8, 0.3, 1.0], [0.0, 0.0, 0.0, 2.0, 1.2]], dtype=dtypes.float32) labels = array_ops.constant([[2], [2], [3]], dtype=dtypes.float32) weights = array_ops.ones([batch_size, 1], dtypes.float32) partition_ids = array_ops.zeros([batch_size], dtypes.int32) ensemble_stamp = variables.Variable( initial_value=0, name="ensemble_stamp", trainable=False, dtype=dtypes.int64) predictions_dict = { "predictions": predictions, "predictions_no_dropout": predictions, "partition_ids": partition_ids, "ensemble_stamp": ensemble_stamp, # This should result in a tree built for a class 2. "num_trees": 13, } # Create train op. train_op = gbdt_model.train( loss=math_ops.reduce_mean( losses.per_example_maxent_loss( labels, weights, predictions, num_classes=learner_config.num_classes)[0]), predictions_dict=predictions_dict, labels=labels) variables.global_variables_initializer().run() resources.initialize_resources(resources.shared_resources()).run() # On first run, expect no splits to be chosen because the quantile # buckets will not be ready. train_op.run() stamp_token, serialized = model_ops.tree_ensemble_serialize( ensemble_handle) output = tree_config_pb2.DecisionTreeEnsembleConfig() output.ParseFromString(serialized.eval()) self.assertEqual(len(output.trees), 0) self.assertEqual(len(output.tree_weights), 0) self.assertEqual(stamp_token.eval(), 1) # Update the stamp to be able to run a second time. sess.run([ensemble_stamp.assign_add(1)]) # On second run, expect a trivial split to be chosen to basically # predict the average. train_op.run() output = tree_config_pb2.DecisionTreeEnsembleConfig() output.ParseFromString(serialized.eval()) stamp_token, serialized = model_ops.tree_ensemble_serialize( ensemble_handle) output.ParseFromString(serialized.eval()) self.assertEqual(len(output.trees), 1) self.assertAllClose(output.tree_weights, [1]) self.assertEqual(stamp_token.eval(), 2) # One node for a split, two children nodes. self.assertEqual(3, len(output.trees[0].nodes)) # Leafs will have a sparse vector for class 3. self.assertEqual(1, len(output.trees[0].nodes[1].leaf.sparse_vector.index)) self.assertEqual(3, output.trees[0].nodes[1].leaf.sparse_vector.index[0]) self.assertAlmostEqual( -1.13134455681, output.trees[0].nodes[1].leaf.sparse_vector.value[0]) self.assertEqual(1, len(output.trees[0].nodes[2].leaf.sparse_vector.index)) self.assertEqual(3, output.trees[0].nodes[2].leaf.sparse_vector.index[0]) self.assertAlmostEqual( 0.893284678459, output.trees[0].nodes[2].leaf.sparse_vector.value[0])
def testTrainFnMulticlassDiagonalHessian(self): """Tests the GBDT train for multiclass diagonal hessian.""" with self.test_session() as sess: ensemble_handle = model_ops.tree_ensemble_variable( stamp_token=0, tree_ensemble_config="", name="tree_ensemble") learner_config = learner_pb2.LearnerConfig() learner_config.learning_rate_tuner.fixed.learning_rate = 1 # Use full hessian multiclass strategy. learner_config.multi_class_strategy = ( learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) learner_config.num_classes = 5 learner_config.regularization.l1 = 0 # To make matrix inversible. learner_config.regularization.l2 = 1e-5 learner_config.constraints.max_tree_depth = 1 learner_config.constraints.min_node_weight = 0 batch_size = 3 features = {} features["dense_float"] = array_ops.constant( [0.3, 1.5, 1.1], dtype=dtypes.float32) gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( is_chief=True, num_ps_replicas=0, center_bias=False, ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, features=features) predictions = array_ops.constant( [[0.0, -1.0, 0.5, 1.2, 3.1], [1.0, 0.0, 0.8, 0.3, 1.0], [0.0, 0.0, 0.0, 0.0, 1.2]], dtype=dtypes.float32) labels = array_ops.constant([[2], [2], [3]], dtype=dtypes.float32) weights = array_ops.ones([batch_size, 1], dtypes.float32) partition_ids = array_ops.zeros([batch_size], dtypes.int32) ensemble_stamp = variables.Variable( initial_value=0, name="ensemble_stamp", trainable=False, dtype=dtypes.int64) predictions_dict = { "predictions": predictions, "predictions_no_dropout": predictions, "partition_ids": partition_ids, "ensemble_stamp": ensemble_stamp, "num_trees": 0, } # Create train op. train_op = gbdt_model.train( loss=math_ops.reduce_mean( losses.per_example_maxent_loss( labels, weights, predictions, num_classes=learner_config.num_classes)[0]), predictions_dict=predictions_dict, labels=labels) variables.global_variables_initializer().run() resources.initialize_resources(resources.shared_resources()).run() # On first run, expect no splits to be chosen because the quantile # buckets will not be ready. train_op.run() stamp_token, serialized = model_ops.tree_ensemble_serialize( ensemble_handle) output = tree_config_pb2.DecisionTreeEnsembleConfig() output.ParseFromString(serialized.eval()) self.assertEqual(len(output.trees), 0) self.assertEqual(len(output.tree_weights), 0) self.assertEqual(stamp_token.eval(), 1) # Update the stamp to be able to run a second time. sess.run([ensemble_stamp.assign_add(1)]) # On second run, expect a trivial split to be chosen to basically # predict the average. train_op.run() output = tree_config_pb2.DecisionTreeEnsembleConfig() output.ParseFromString(serialized.eval()) stamp_token, serialized = model_ops.tree_ensemble_serialize( ensemble_handle) output.ParseFromString(serialized.eval()) self.assertEqual(len(output.trees), 1) # We got 3 nodes: one parent and 2 leafs. self.assertEqual(len(output.trees[0].nodes), 3) self.assertAllClose(output.tree_weights, [1]) self.assertEqual(stamp_token.eval(), 2) # Leafs should have a dense vector of size 5. expected_leaf_1 = [-1.0354, -1.0107, 17.2976, -1.1313, -4.5023] expected_leaf_2 = [-1.2924, -1.1376, 2.2042, 3.1052, -1.6269] self.assertArrayNear(expected_leaf_1, output.trees[0].nodes[1].leaf.vector.value, 1e-3) self.assertArrayNear(expected_leaf_2, output.trees[0].nodes[2].leaf.vector.value, 1e-3)
def testTrainFnMulticlassTreePerClass(self): """Tests the GBDT train for multiclass tree per class strategy.""" with self.test_session() as sess: ensemble_handle = model_ops.tree_ensemble_variable( stamp_token=0, tree_ensemble_config="", name="tree_ensemble") learner_config = learner_pb2.LearnerConfig() learner_config.learning_rate_tuner.fixed.learning_rate = 1 # Use full hessian multiclass strategy. learner_config.multi_class_strategy = ( learner_pb2.LearnerConfig.TREE_PER_CLASS) learner_config.num_classes = 5 learner_config.regularization.l1 = 0 # To make matrix inversible. learner_config.regularization.l2 = 1e-5 learner_config.constraints.max_tree_depth = 1 learner_config.constraints.min_node_weight = 0 features = { "dense_float": array_ops.constant([[1.0], [1.5], [2.0]], dtypes.float32), } gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( is_chief=True, num_ps_replicas=0, center_bias=False, ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, features=features) batch_size = 3 predictions = array_ops.constant( [[0.0, -1.0, 0.5, 1.2, 3.1], [1.0, 0.0, 0.8, 0.3, 1.0], [0.0, 0.0, 0.0, 2.0, 1.2]], dtype=dtypes.float32) labels = array_ops.constant([[2], [2], [3]], dtype=dtypes.float32) weights = array_ops.ones([batch_size, 1], dtypes.float32) partition_ids = array_ops.zeros([batch_size], dtypes.int32) ensemble_stamp = variables.Variable(initial_value=0, name="ensemble_stamp", trainable=False, dtype=dtypes.int64) predictions_dict = { "predictions": predictions, "predictions_no_dropout": predictions, "partition_ids": partition_ids, "ensemble_stamp": ensemble_stamp, # This should result in a tree built for a class 2. "num_trees": 13, } # Create train op. train_op = gbdt_model.train(loss=math_ops.reduce_mean( losses.per_example_maxent_loss( labels, weights, predictions, num_classes=learner_config.num_classes)[0]), predictions_dict=predictions_dict, labels=labels) variables.global_variables_initializer().run() resources.initialize_resources(resources.shared_resources()).run() # On first run, expect no splits to be chosen because the quantile # buckets will not be ready. train_op.run() stamp_token, serialized = model_ops.tree_ensemble_serialize( ensemble_handle) output = tree_config_pb2.DecisionTreeEnsembleConfig() output.ParseFromString(serialized.eval()) self.assertEqual(len(output.trees), 0) self.assertEqual(len(output.tree_weights), 0) self.assertEqual(stamp_token.eval(), 1) # Update the stamp to be able to run a second time. sess.run([ensemble_stamp.assign_add(1)]) # On second run, expect a trivial split to be chosen to basically # predict the average. train_op.run() output = tree_config_pb2.DecisionTreeEnsembleConfig() output.ParseFromString(serialized.eval()) stamp_token, serialized = model_ops.tree_ensemble_serialize( ensemble_handle) output.ParseFromString(serialized.eval()) self.assertEqual(len(output.trees), 1) self.assertAllClose(output.tree_weights, [1]) self.assertEqual(stamp_token.eval(), 2) # One node for a split, two children nodes. self.assertEqual(3, len(output.trees[0].nodes)) # Leafs will have a sparse vector for class 3. self.assertEqual( 1, len(output.trees[0].nodes[1].leaf.sparse_vector.index)) self.assertEqual( 3, output.trees[0].nodes[1].leaf.sparse_vector.index[0]) self.assertAlmostEqual( -1.13134455681, output.trees[0].nodes[1].leaf.sparse_vector.value[0]) self.assertEqual( 1, len(output.trees[0].nodes[2].leaf.sparse_vector.index)) self.assertEqual( 3, output.trees[0].nodes[2].leaf.sparse_vector.index[0]) self.assertAlmostEqual( 0.893284678459, output.trees[0].nodes[2].leaf.sparse_vector.value[0])
def testTrainFnMulticlassDiagonalHessian(self): """Tests the GBDT train for multiclass diagonal hessian.""" with self.test_session() as sess: ensemble_handle = model_ops.tree_ensemble_variable( stamp_token=0, tree_ensemble_config="", name="tree_ensemble") learner_config = learner_pb2.LearnerConfig() learner_config.learning_rate_tuner.fixed.learning_rate = 1 # Use full hessian multiclass strategy. learner_config.multi_class_strategy = ( learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) learner_config.num_classes = 5 learner_config.regularization.l1 = 0 # To make matrix inversible. learner_config.regularization.l2 = 1e-5 learner_config.constraints.max_tree_depth = 1 learner_config.constraints.min_node_weight = 0 batch_size = 3 features = {} features["dense_float"] = array_ops.ones([batch_size, 1], dtypes.float32) gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel( is_chief=True, num_ps_replicas=0, center_bias=False, ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, features=features) predictions = array_ops.constant( [[0.0, -1.0, 0.5, 1.2, 3.1], [1.0, 0.0, 0.8, 0.3, 1.0], [0.0, 0.0, 0.0, 0.0, 1.2]], dtype=dtypes.float32) labels = array_ops.constant([[2], [2], [3]], dtype=dtypes.float32) weights = array_ops.ones([batch_size, 1], dtypes.float32) partition_ids = array_ops.zeros([batch_size], dtypes.int32) ensemble_stamp = variables.Variable(initial_value=0, name="ensemble_stamp", trainable=False, dtype=dtypes.int64) predictions_dict = { "predictions": predictions, "predictions_no_dropout": predictions, "partition_ids": partition_ids, "ensemble_stamp": ensemble_stamp, "num_trees": 0, } # Create train op. train_op = gbdt_model.train(loss=math_ops.reduce_mean( losses.per_example_maxent_loss( labels, weights, predictions, num_classes=learner_config.num_classes)[0]), predictions_dict=predictions_dict, labels=labels) variables.global_variables_initializer().run() resources.initialize_resources(resources.shared_resources()).run() # On first run, expect no splits to be chosen because the quantile # buckets will not be ready. train_op.run() stamp_token, serialized = model_ops.tree_ensemble_serialize( ensemble_handle) output = tree_config_pb2.DecisionTreeEnsembleConfig() output.ParseFromString(serialized.eval()) self.assertEqual(len(output.trees), 0) self.assertEqual(len(output.tree_weights), 0) self.assertEqual(stamp_token.eval(), 1) # Update the stamp to be able to run a second time. sess.run([ensemble_stamp.assign_add(1)]) # On second run, expect a trivial split to be chosen to basically # predict the average. train_op.run() output = tree_config_pb2.DecisionTreeEnsembleConfig() output.ParseFromString(serialized.eval()) stamp_token, serialized = model_ops.tree_ensemble_serialize( ensemble_handle) output.ParseFromString(serialized.eval()) self.assertEqual(len(output.trees), 1) self.assertAllClose(output.tree_weights, [1]) self.assertEqual(stamp_token.eval(), 2) # Leaf should have a dense vector of size 5. expected = [ -1.26767396927, -1.13043296337, 4.58542203903, 1.81428349018, -2.43038392067 ] for i in range(learner_config.num_classes): self.assertAlmostEqual( expected[i], output.trees[0].nodes[1].leaf.vector.value[i])