def _get_train_op_and_ensemble(self, head, config, is_classification,
train_in_memory):
"""Calls bt_model_fn() and returns the train_op and ensemble_serialzed."""
features, labels = _make_train_input_fn(is_classification)()
estimator_spec = boosted_trees._bt_model_fn( # pylint:disable=protected-access
features=features,
labels=labels,
mode=model_fn.ModeKeys.TRAIN,
head=head,
feature_columns=self._feature_columns,
tree_hparams=self._tree_hparams,
example_id_column_name=EXAMPLE_ID_COLUMN,
n_batches_per_layer=1,
config=config,
train_in_memory=train_in_memory)
resources.initialize_resources(resources.shared_resources()).run()
variables.global_variables_initializer().run()
variables.local_variables_initializer().run()
# Gets the train_op and serialized proto of the ensemble.
shared_resources = resources.shared_resources()
self.assertEqual(1, len(shared_resources))
train_op = estimator_spec.train_op
with ops.control_dependencies([train_op]):
_, ensemble_serialized = (
gen_boosted_trees_ops.boosted_trees_serialize_ensemble(
shared_resources[0].handle))
return train_op, ensemble_serialized
def testBiasEnsembleMultiClass(self):
with self.test_session():
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
tree = tree_ensemble_config.trees.add()
tree_ensemble_config.tree_metadata.add().is_finalized = True
leaf = tree.nodes.add().leaf
_append_to_leaf(leaf, 0, -0.4)
_append_to_leaf(leaf, 1, 0.9)
tree_ensemble_config.tree_weights.append(1.0)
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="multiclass")
resources.initialize_resources(resources.shared_resources()).run()
# Prepare learner config.
learner_config = learner_pb2.LearnerConfig()
learner_config.num_classes = 3
result, dropout_info = self._get_predictions(
tree_ensemble_handle,
learner_config=learner_config.SerializeToString(),
reduce_dim=True)
self.assertAllClose([[-0.4, 0.9], [-0.4, 0.9]], result.eval())
# Empty dropout.
self.assertAllEqual([[], []], dropout_info.eval())
def testTreeFinalized(self):
with self.test_session():
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
# Depth 3 tree.
tree1 = tree_ensemble_config.trees.add()
_set_float_split(tree1.nodes.add().dense_float_binary_split, 0, 9.0, 1, 2)
_set_float_split(tree1.nodes.add()
.sparse_float_binary_split_default_left.split, 0, -20.0,
3, 4)
_append_to_leaf(tree1.nodes.add().leaf, 0, 0.2)
_append_to_leaf(tree1.nodes.add().leaf, 0, 0.3)
_set_categorical_id_split(tree1.nodes.add().categorical_id_binary_split,
0, 9, 5, 6)
_append_to_leaf(tree1.nodes.add().leaf, 0, 0.5)
_append_to_leaf(tree1.nodes.add().leaf, 0, 0.6)
tree_ensemble_config.tree_weights.append(1.0)
tree_ensemble_config.tree_metadata.add().is_finalized = True
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="full_ensemble")
resources.initialize_resources(resources.shared_resources()).run()
result = prediction_ops.gradient_trees_partition_examples(
tree_ensemble_handle, [self._dense_float_tensor], [
self._sparse_float_indices1, self._sparse_float_indices2
], [self._sparse_float_values1, self._sparse_float_values2],
[self._sparse_float_shape1,
self._sparse_float_shape2], [self._sparse_int_indices1],
[self._sparse_int_values1], [self._sparse_int_shape1])
self.assertAllEqual([0, 0], result.eval())
def testBasicQuantileBucketsMultipleResources(self):
with self.test_session() as sess:
quantile_accumulator_handle_0 = self.create_resource("float_0", self.eps,
self.max_elements)
quantile_accumulator_handle_1 = self.create_resource("float_1", self.eps,
self.max_elements)
resources.initialize_resources(resources.shared_resources()).run()
summaries = boosted_trees_ops.make_quantile_summaries(
[self._feature_0, self._feature_1], self._example_weights,
epsilon=self.eps)
summary_op_0 = boosted_trees_ops.quantile_add_summaries(
quantile_accumulator_handle_0,
[summaries[0]])
summary_op_1 = boosted_trees_ops.quantile_add_summaries(
quantile_accumulator_handle_1,
[summaries[1]])
flush_op_0 = boosted_trees_ops.quantile_flush(
quantile_accumulator_handle_0, self.num_quantiles)
flush_op_1 = boosted_trees_ops.quantile_flush(
quantile_accumulator_handle_1, self.num_quantiles)
bucket_0 = boosted_trees_ops.get_bucket_boundaries(
quantile_accumulator_handle_0, num_features=1)
bucket_1 = boosted_trees_ops.get_bucket_boundaries(
quantile_accumulator_handle_1, num_features=1)
quantiles = boosted_trees_ops.boosted_trees_bucketize(
[self._feature_0, self._feature_1], bucket_0 + bucket_1)
sess.run([summary_op_0, summary_op_1])
sess.run([flush_op_0, flush_op_1])
self.assertAllClose(self._feature_0_boundaries, bucket_0[0].eval())
self.assertAllClose(self._feature_1_boundaries, bucket_1[0].eval())
self.assertAllClose(self._feature_0_quantiles, quantiles[0].eval())
self.assertAllClose(self._feature_1_quantiles, quantiles[1].eval())
def testAverageMoreThanNumTreesExist(self):
with self.test_session():
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
adjusted_tree_ensemble_config = (
tree_config_pb2.DecisionTreeEnsembleConfig())
# When we say to average over more trees than possible, it is averaging
# across all trees.
total_num = 100
for i in range(0, total_num):
tree = tree_ensemble_config.trees.add()
_append_to_leaf(tree.nodes.add().leaf, 0, -0.4)
tree_ensemble_config.tree_metadata.add().is_finalized = True
tree_ensemble_config.tree_weights.append(1.0)
# This is how the weight will look after averaging
copy_tree = adjusted_tree_ensemble_config.trees.add()
_append_to_leaf(copy_tree.nodes.add().leaf, 0, -0.4)
adjusted_tree_ensemble_config.tree_metadata.add().is_finalized = True
adjusted_tree_ensemble_config.tree_weights.append(
1.0 * (total_num - i) / total_num)
# Prepare learner config WITH AVERAGING.
learner_config = learner_pb2.LearnerConfig()
learner_config.num_classes = 2
# We have only 100 trees but we ask to average over 250.
learner_config.averaging_config.average_last_n_trees = 250
# No averaging config.
learner_config_no_averaging = learner_pb2.LearnerConfig()
learner_config_no_averaging.num_classes = 2
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="existing")
# This is how our ensemble will "look" during averaging
adjusted_tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=adjusted_tree_ensemble_config.SerializeToString(
),
name="adjusted")
resources.initialize_resources(resources.shared_resources()).run()
result, dropout_info = self._get_predictions(
tree_ensemble_handle,
learner_config.SerializeToString(),
apply_averaging=True,
reduce_dim=True)
pattern_result, pattern_dropout_info = self._get_predictions(
adjusted_tree_ensemble_handle,
learner_config_no_averaging.SerializeToString(),
apply_averaging=False,
reduce_dim=True)
self.assertAllEqual(result.eval(), pattern_result.eval())
self.assertAllEqual(dropout_info.eval(), pattern_dropout_info.eval())
def testCreate(self):
with self.cached_session():
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
tree = tree_ensemble_config.trees.add()
_append_to_leaf(tree.nodes.add().leaf, 0, -0.4)
tree_ensemble_config.tree_weights.append(1.0)
# Prepare learner config.
learner_config = learner_pb2.LearnerConfig()
learner_config.num_classes = 2
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=3,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="create_tree")
resources.initialize_resources(resources.shared_resources()).run()
result, _ = prediction_ops.gradient_trees_prediction(
tree_ensemble_handle,
self._seed, [self._dense_float_tensor], [
self._sparse_float_indices1, self._sparse_float_indices2
], [self._sparse_float_values1, self._sparse_float_values2],
[self._sparse_float_shape1,
self._sparse_float_shape2], [self._sparse_int_indices1],
[self._sparse_int_values1], [self._sparse_int_shape1],
learner_config=learner_config.SerializeToString(),
apply_dropout=False,
apply_averaging=False,
center_bias=False,
reduce_dim=True)
self.assertAllClose(result.eval(), [[-0.4], [-0.4]])
stamp_token = model_ops.tree_ensemble_stamp_token(tree_ensemble_handle)
self.assertEqual(stamp_token.eval(), 3)
def testSaveRestoreBeforeFlush(self):
save_dir = os.path.join(self.get_temp_dir(), "save_restore")
save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
with self.cached_session() as sess:
accumulator = boosted_trees_ops.QuantileAccumulator(
num_streams=2, num_quantiles=3, epsilon=self.eps, name="q0")
save = saver.Saver()
resources.initialize_resources(resources.shared_resources()).run()
summaries = accumulator.add_summaries([self._feature_0, self._feature_1],
self._example_weights)
self.evaluate(summaries)
buckets = accumulator.get_bucket_boundaries()
self.assertAllClose([], buckets[0].eval())
self.assertAllClose([], buckets[1].eval())
save.save(sess, save_path)
self.evaluate(accumulator.flush())
self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
with self.session(graph=ops.Graph()) as sess:
accumulator = boosted_trees_ops.QuantileAccumulator(
num_streams=2, num_quantiles=3, epsilon=self.eps, name="q0")
save = saver.Saver()
save.restore(sess, save_path)
buckets = accumulator.get_bucket_boundaries()
self.assertAllClose([], buckets[0].eval())
self.assertAllClose([], buckets[1].eval())
def testCachedPredictionOnEmptyEnsemble(self):
"""Tests that prediction on a dummy ensemble does not fail."""
with self.cached_session() as session:
# Create a dummy ensemble.
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble', serialized_proto='')
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
# No previous cached values.
cached_tree_ids = [0, 0]
cached_node_ids = [0, 0]
# We have two features: 0 and 1. Values don't matter here on a dummy
# ensemble.
feature_0_values = [67, 5]
feature_1_values = [9, 17]
# Grow tree ensemble.
predict_op = boosted_trees_ops.training_predict(
tree_ensemble_handle,
cached_tree_ids=cached_tree_ids,
cached_node_ids=cached_node_ids,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
# Nothing changed.
self.assertAllClose(cached_tree_ids, new_tree_ids)
self.assertAllClose(cached_node_ids, new_node_ids)
self.assertAllClose([[0], [0]], logits_updates)
def _testStreamingQuantileBucketsHelper(
self, inputs, num_quantiles=3, expected_buckets=None):
"""Helper to test quantile buckets on different inputs."""
# set generate_quantiles to True since the test will generate fewer
# boundaries otherwise.
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=num_quantiles,
epsilon=0.001, name="q1", generate_quantiles=True)
resources.initialize_resources(resources.shared_resources()).run()
input_column = array_ops.placeholder(dtypes.float32)
weights = array_ops.placeholder(dtypes.float32)
update = accumulator.add_summary(
stamp_token=0,
column=input_column,
example_weights=weights)
with self.test_session() as sess:
sess.run(update,
{input_column: inputs,
weights: [1] * len(inputs)})
with self.test_session() as sess:
sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1))
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run(
[buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
# By default, use 3 quantiles, 4 boundaries for simplicity.
self.assertEqual(num_quantiles + 1, len(buckets))
if expected_buckets:
self.assertAllEqual(buckets, expected_buckets)
def testStreamingQuantileBuckets(self):
"""Sets up the quantile summary op test as follows.
100 batches of data is added to the accumulator. The batches are in form:
[0 1 .. 99]
[100 101 .. 200]
...
[9900 9901 .. 9999]
All the batches have 1 for all the example weights.
"""
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=3, epsilon=0.01, name="q1")
resources.initialize_resources(resources.shared_resources()).run()
weight_placeholder = array_ops.placeholder(dtypes.float32)
dense_placeholder = array_ops.placeholder(dtypes.float32)
update = accumulator.add_summary(
stamp_token=0,
column=dense_placeholder,
example_weights=weight_placeholder)
with self.test_session() as sess:
for i in range(100):
dense_float = np.linspace(
i * 100, (i + 1) * 100 - 1, num=100).reshape(-1, 1)
sess.run(update, {
dense_placeholder: dense_float,
weight_placeholder: np.ones(shape=(100, 1), dtype=np.float32)
})
with self.test_session() as sess:
sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1))
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([0, 3335., 6671., 9999.], buckets)
def testContribsForOnlyABiasNode(self):
"""Tests case when, after training, only left with a bias node.
For example, this could happen if the final ensemble contains one tree that
got pruned up to the root.
"""
with self.cached_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
leaf {
scalar: 1.72
}
}
}
tree_weights: 0.1
tree_metadata: {
num_layers_grown: 0
}
""", tree_ensemble_config)
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
# All features are unused.
feature_0_values = [36, 32]
feature_1_values = [13, -29]
feature_2_values = [11, 27]
# Expected logits are computed by traversing the logit path and
# subtracting child logits from parent logits.
bias = 1.72 * 0.1 # Root node of tree_0.
expected_feature_ids = ((), ())
expected_logits_paths = ((bias,), (bias,))
bucketized_features = [
feature_0_values, feature_1_values, feature_2_values
]
debug_op = boosted_trees_ops.example_debug_outputs(
tree_ensemble_handle,
bucketized_features=bucketized_features,
logits_dimension=1)
serialized_examples_debug_outputs = session.run(debug_op)
feature_ids = []
logits_paths = []
for example in serialized_examples_debug_outputs:
example_debug_outputs = boosted_trees_pb2.DebugOutput()
example_debug_outputs.ParseFromString(example)
feature_ids.append(example_debug_outputs.feature_ids)
logits_paths.append(example_debug_outputs.logits_path)
self.assertAllClose(feature_ids, expected_feature_ids)
self.assertAllClose(logits_paths, expected_logits_paths)
def testDropout(self):
with self.test_session():
# Empty tree ensenble.
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
# Add 1000 trees with some weights.
for i in range(0, 999):
tree = tree_ensemble_config.trees.add()
tree_ensemble_config.tree_metadata.add().is_finalized = True
_append_to_leaf(tree.nodes.add().leaf, 0, -0.4)
tree_ensemble_config.tree_weights.append(i + 1)
# Prepare learner/dropout config.
learner_config = learner_pb2.LearnerConfig()
learner_config.learning_rate_tuner.dropout.dropout_probability = 0.5
learner_config.learning_rate_tuner.dropout.learning_rate = 1.0
learner_config.num_classes = 2
# Apply dropout.
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="existing")
resources.initialize_resources(resources.shared_resources()).run()
result, dropout_info = self._get_predictions(
tree_ensemble_handle,
learner_config=learner_config.SerializeToString(),
apply_dropout=True,
apply_averaging=False,
center_bias=False,
reduce_dim=True)
# We expect approx 500 trees were dropped.
dropout_info = dropout_info.eval()
self.assertIn(dropout_info[0].size, range(400, 601))
self.assertEqual(dropout_info[0].size, dropout_info[1].size)
for i in range(dropout_info[0].size):
dropped_index = dropout_info[0][i]
dropped_weight = dropout_info[1][i]
# We constructed the trees so tree number + 1 is the tree weight, so
# we can check here the weights for dropped trees.
self.assertEqual(dropped_index + 1, dropped_weight)
# Don't apply dropout.
result_no_dropout, no_dropout_info = self._get_predictions(
tree_ensemble_handle,
learner_config=learner_config.SerializeToString(),
apply_dropout=False,
apply_averaging=False,
center_bias=False,
reduce_dim=True)
self.assertEqual(result.eval().size, result_no_dropout.eval().size)
for i in range(result.eval().size):
self.assertNotEqual(result.eval()[i], result_no_dropout.eval()[i])
# We expect none of the trees were dropped.
self.assertAllEqual([[], []], no_dropout_info.eval())
def testWithExistingEnsembleAndShrinkage(self):
with self.test_session():
# Add shrinkage config.
learning_rate = 0.0001
tree_ensemble = tree_config_pb2.DecisionTreeEnsembleConfig()
# Add 10 trees with some weights.
for i in range(0, 5):
tree = tree_ensemble.trees.add()
_append_to_leaf(tree.nodes.add().leaf, 0, -0.4)
tree_ensemble.tree_weights.append(i + 1)
meta = tree_ensemble.tree_metadata.add()
meta.num_tree_weight_updates = 1
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble.SerializeToString(),
name="existing")
# Create non-zero feature importance.
feature_usage_counts = variables.Variable(
initial_value=np.array([4, 7], np.int64),
name="feature_usage_counts",
trainable=False)
feature_gains = variables.Variable(
initial_value=np.array([0.2, 0.8], np.float32),
name="feature_gains",
trainable=False)
resources.initialize_resources(resources.shared_resources()).run()
variables.initialize_all_variables().run()
output_ensemble = tree_config_pb2.DecisionTreeEnsembleConfig()
with ops.control_dependencies([
ensemble_optimizer_ops.add_trees_to_ensemble(
tree_ensemble_handle,
self._ensemble_to_add.SerializeToString(),
feature_usage_counts, [1, 2],
feature_gains, [0.5, 0.3], [[], []],
learning_rate=learning_rate)
]):
output_ensemble.ParseFromString(
model_ops.tree_ensemble_serialize(tree_ensemble_handle)[1].eval())
# The weights of previous trees stayed the same, new tree (LAST) is added
# with shrinkage weight.
self.assertAllClose([1.0, 2.0, 3.0, 4.0, 5.0, learning_rate],
output_ensemble.tree_weights)
# Check that all number of updates are equal to 1 (e,g, no old tree weight
# got adjusted.
for i in range(0, 6):
self.assertEqual(
1, output_ensemble.tree_metadata[i].num_tree_weight_updates)
# Ensure feature importance was aggregated correctly.
self.assertAllEqual([5, 9], feature_usage_counts.eval())
self.assertArrayNear(
[0.2 + 0.5 * learning_rate, 0.8 + 0.3 * learning_rate],
feature_gains.eval(), 1e-6)
def testCreate(self):
with self.test_session():
ensemble = boosted_trees_ops.TreeEnsemble('ensemble')
resources.initialize_resources(resources.shared_resources()).run()
stamp_token = ensemble.get_stamp_token()
self.assertEqual(0, stamp_token.eval())
(_, num_trees, num_finalized_trees,
num_attempted_layers) = ensemble.get_states()
self.assertEqual(0, num_trees.eval())
self.assertEqual(0, num_finalized_trees.eval())
self.assertEqual(0, num_attempted_layers.eval())
def testCreate(self):
with self.cached_session():
ensemble = boosted_trees_ops.TreeEnsemble('ensemble')
resources.initialize_resources(resources.shared_resources()).run()
stamp_token = ensemble.get_stamp_token()
self.assertEqual(0, self.evaluate(stamp_token))
(_, num_trees, num_finalized_trees, num_attempted_layers,
nodes_range) = ensemble.get_states()
self.assertEqual(0, self.evaluate(num_trees))
self.assertEqual(0, self.evaluate(num_finalized_trees))
self.assertEqual(0, self.evaluate(num_attempted_layers))
self.assertAllEqual([0, 1], self.evaluate(nodes_range))
def testPredictFn(self):
"""Tests the predict function."""
with self.test_session() as sess:
# Create ensemble with one bias node.
ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
text_format.Merge("""
trees {
nodes {
leaf {
vector {
value: 0.25
}
}
}
}
tree_weights: 1.0
tree_metadata {
num_tree_weight_updates: 1
num_layers_grown: 1
is_finalized: true
}""", ensemble_config)
ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=3,
tree_ensemble_config=ensemble_config.SerializeToString(),
name="tree_ensemble")
resources.initialize_resources(resources.shared_resources()).run()
learner_config = learner_pb2.LearnerConfig()
learner_config.learning_rate_tuner.fixed.learning_rate = 0.1
learner_config.num_classes = 2
learner_config.regularization.l1 = 0
learner_config.regularization.l2 = 0
learner_config.constraints.max_tree_depth = 1
learner_config.constraints.min_node_weight = 0
features = {}
features["dense_float"] = array_ops.ones([4, 1], dtypes.float32)
gbdt_model = gbdt_batch.GradientBoostedDecisionTreeModel(
is_chief=False,
num_ps_replicas=0,
center_bias=True,
ensemble_handle=ensemble_handle,
examples_per_layer=1,
learner_config=learner_config,
features=features)
# Create predict op.
mode = model_fn.ModeKeys.EVAL
predictions_dict = sess.run(gbdt_model.predict(mode))
self.assertEquals(predictions_dict["ensemble_stamp"], 3)
self.assertAllClose(predictions_dict["predictions"], [[0.25], [0.25],
[0.25], [0.25]])
self.assertAllClose(predictions_dict["partition_ids"], [0, 0, 0, 0])
def testSaveRestoreBeforeFlush(self):
save_dir = os.path.join(self.get_temp_dir(), "save_restore")
save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
with self.test_session(graph=ops.Graph()) as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=3, epsilon=0.33, name="q0")
save = saver.Saver()
resources.initialize_resources(resources.shared_resources()).run()
sparse_indices_0 = constant_op.constant(
[[1, 0], [2, 1], [3, 0], [4, 2], [5, 0]], dtype=dtypes.int64)
sparse_values_0 = constant_op.constant(
[2.0, 3.0, 4.0, 5.0, 6.0], dtype=dtypes.float32)
sparse_shape_0 = constant_op.constant([6, 3], dtype=dtypes.int64)
example_weights = constant_op.constant(
[10, 1, 1, 1, 1, 1], dtype=dtypes.float32, shape=[6, 1])
update = accumulator.add_summary(
stamp_token=0,
column=sparse_tensor.SparseTensor(sparse_indices_0, sparse_values_0,
sparse_shape_0),
example_weights=example_weights)
update.run()
save.save(sess, save_path)
reset = accumulator.flush(stamp_token=0, next_stamp_token=1)
with ops.control_dependencies([reset]):
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([2, 4, 6.], buckets)
with self.test_session(graph=ops.Graph()) as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=3, epsilon=0.33, name="q0")
save = saver.Saver()
# Restore the saved values in the parameter nodes.
save.restore(sess, save_path)
are_ready_noflush = accumulator.get_buckets(stamp_token=0)[0]
with ops.control_dependencies([are_ready_noflush]):
reset = accumulator.flush(stamp_token=0, next_stamp_token=1)
with ops.control_dependencies([reset]):
are_ready_flush, buckets = accumulator.get_buckets(stamp_token=1)
buckets, are_ready_flush, are_ready_noflush = (sess.run(
[buckets, are_ready_flush, are_ready_noflush]))
self.assertFalse(are_ready_noflush)
self.assertTrue(are_ready_flush)
self.assertAllEqual([2, 4, 6.], buckets)
def _init_graph(self):
# Initialize all weights
if not self._is_initialized:
self.saver = tf.train.Saver()
init_vars = tf.group(tf.global_variables_initializer(),
resources.initialize_resources(
resources.shared_resources()))
self.session.run(init_vars)
self._is_initialized = True
# Restore weights if needed
if self._to_be_restored:
self.saver = tf.train.Saver()
self.saver.restore(self.session, self._to_be_restored)
self._to_be_restored = False
def run_feeds_iter(output_dict, feed_dicts, restore_checkpoint_path=None):
"""Run `output_dict` tensors with each input in `feed_dicts`.
If `restore_checkpoint_path` is supplied, restore from checkpoint. Otherwise,
init all variables.
Args:
output_dict: A `dict` mapping string names to `Tensor` objects to run.
Tensors must all be from the same graph.
feed_dicts: Iterable of `dict` objects of input values to feed.
restore_checkpoint_path: A string containing the path to a checkpoint to
restore.
Yields:
A sequence of dicts of values read from `output_dict` tensors, one item
yielded for each item in `feed_dicts`. Keys are the same as `output_dict`,
values are the results read from the corresponding `Tensor` in
`output_dict`.
Raises:
ValueError: if `output_dict` or `feed_dicts` is None or empty.
"""
if not output_dict:
raise ValueError('output_dict is invalid: %s.' % output_dict)
if not feed_dicts:
raise ValueError('feed_dicts is invalid: %s.' % feed_dicts)
graph = contrib_ops.get_graph_from_inputs(output_dict.values())
with graph.as_default() as g:
with tf_session.Session('') as session:
session.run(
resources.initialize_resources(resources.shared_resources() +
resources.local_resources()))
if restore_checkpoint_path:
_restore_from_checkpoint(session, g, restore_checkpoint_path)
else:
session.run(variables.global_variables_initializer())
session.run(variables.local_variables_initializer())
session.run(data_flow_ops.initialize_all_tables())
coord = coordinator.Coordinator()
threads = None
try:
threads = queue_runner.start_queue_runners(session, coord=coord)
for f in feed_dicts:
yield session.run(output_dict, f)
finally:
coord.request_stop()
if threads:
coord.join(threads, stop_grace_period_secs=120)
def testMetadataMissing(self):
# Sometimes we want to do prediction on trees that are not added to ensemble
# (for example in
with self.test_session():
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
# Bias tree.
tree1 = tree_ensemble_config.trees.add()
_append_to_leaf(tree1.nodes.add().leaf, 0, -0.4)
# Depth 3 tree.
tree2 = tree_ensemble_config.trees.add()
# We are not setting the tree_ensemble_config.tree_metadata in this test.
_set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 1, 2)
_set_float_split(tree2.nodes.add()
.sparse_float_binary_split_default_left.split, 0, -20.0,
3, 4)
_append_to_leaf(tree2.nodes.add().leaf, 0, 0.5)
_append_to_leaf(tree2.nodes.add().leaf, 0, 1.2)
_set_categorical_id_split(tree2.nodes.add().categorical_id_binary_split,
0, 9, 5, 6)
_append_to_leaf(tree2.nodes.add().leaf, 0, -0.9)
_append_to_leaf(tree2.nodes.add().leaf, 0, 0.7)
tree_ensemble_config.tree_weights.append(1.0)
tree_ensemble_config.tree_weights.append(1.0)
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="full_ensemble")
resources.initialize_resources(resources.shared_resources()).run()
# Prepare learner config.
learner_config = learner_pb2.LearnerConfig()
learner_config.num_classes = 2
result, dropout_info = self._get_predictions(
tree_ensemble_handle,
learner_config=learner_config.SerializeToString(),
reduce_dim=True)
# The first example will get bias -0.4 from first tree and
# leaf 4 payload of -0.9 hence -1.3, the second example will
# get the same bias -0.4 and leaf 3 payload (sparse feature missing)
# of 1.2 hence 0.8.
self.assertAllClose([[-1.3], [0.8]], result.eval())
# Empty dropout.
self.assertAllEqual([[], []], dropout_info.eval())
def testUsedHandlers(self):
with self.cached_session():
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
tree_ensemble_config.growing_metadata.used_handler_ids.append(1)
tree_ensemble_config.growing_metadata.used_handler_ids.append(5)
stamp_token = 3
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=stamp_token,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="create_tree")
resources.initialize_resources(resources.shared_resources()).run()
result = model_ops.tree_ensemble_used_handlers(
tree_ensemble_handle, stamp_token, num_all_handlers=6)
self.assertAllEqual([0, 1, 0, 0, 0, 1], result.used_handlers_mask.eval())
self.assertEqual(2, result.num_used_handlers.eval())
def testInactive(self):
with self.test_session() as sess:
gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0])
hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13])
partition_ids = [0, 0, 0, 1]
indices = [[0, 0], [0, 1], [2, 0], [3, 0]]
values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64)
gradient_shape = tensor_shape.scalar()
hessian_shape = tensor_shape.scalar()
class_id = -1
split_handler = categorical_split_handler.EqualitySplitHandler(
l1_regularization=0.1,
l2_regularization=1,
tree_complexity_regularization=0,
min_node_weight=0,
sparse_int_column=sparse_tensor.SparseTensor(indices, values, [4, 1]),
feature_column_group_id=0,
gradient_shape=gradient_shape,
hessian_shape=hessian_shape,
multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS,
init_stamp_token=0)
resources.initialize_resources(resources.shared_resources()).run()
empty_gradients, empty_hessians = get_empty_tensors(
gradient_shape, hessian_shape)
example_weights = array_ops.ones([4, 1], dtypes.float32)
update_1 = split_handler.update_stats_sync(
0,
partition_ids,
gradients,
hessians,
empty_gradients,
empty_hessians,
example_weights,
is_active=array_ops.constant([False, False]))
with ops.control_dependencies([update_1]):
are_splits_ready, partitions, gains, splits = (
split_handler.make_splits(0, 1, class_id))
are_splits_ready, partitions, gains, splits = (sess.run(
[are_splits_ready, partitions, gains, splits]))
self.assertTrue(are_splits_ready)
self.assertEqual(len(partitions), 0)
self.assertEqual(len(gains), 0)
self.assertEqual(len(splits), 0)
def testExcludeNonFinalTree(self):
with self.test_session():
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
# Bias tree.
tree1 = tree_ensemble_config.trees.add()
tree_ensemble_config.tree_metadata.add().is_finalized = True
_append_to_leaf(tree1.nodes.add().leaf, 0, -0.4)
# Depth 3 tree.
tree2 = tree_ensemble_config.trees.add()
tree_ensemble_config.tree_metadata.add().is_finalized = False
_set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 1, 2)
_set_float_split(tree2.nodes.add()
.sparse_float_binary_split_default_left.split, 0, -20.0,
3, 4)
_append_to_leaf(tree2.nodes.add().leaf, 0, 0.5)
_append_to_leaf(tree2.nodes.add().leaf, 0, 1.2)
_set_categorical_id_split(tree2.nodes.add().categorical_id_binary_split,
0, 9, 5, 6)
_append_to_leaf(tree2.nodes.add().leaf, 0, -0.9)
_append_to_leaf(tree2.nodes.add().leaf, 0, 0.7)
tree_ensemble_config.tree_weights.append(1.0)
tree_ensemble_config.tree_weights.append(1.0)
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="full_ensemble")
resources.initialize_resources(resources.shared_resources()).run()
# Prepare learner config.
learner_config = learner_pb2.LearnerConfig()
learner_config.num_classes = 2
learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE
result, dropout_info = self._get_predictions(
tree_ensemble_handle,
learner_config=learner_config.SerializeToString(),
reduce_dim=True)
# All the examples should get only the bias since the second tree is
# non-finalized
self.assertAllClose([[-0.4], [-0.4]], result.eval())
# Empty dropout.
self.assertAllEqual([[], []], dropout_info.eval())
def testStreamingQuantileBucketsLowPrecisionInput(self):
"""Tests inputs that simulate low precision float16 values."""
num_quantiles = 3
# set generate_quantiles to True since the test will generate fewer
# boundaries otherwise.
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=num_quantiles,
epsilon=0.001, name="q1", generate_quantiles=True)
resources.initialize_resources(resources.shared_resources()).run()
input_column = array_ops.placeholder(dtypes.float32)
weights = array_ops.placeholder(dtypes.float32)
update = accumulator.add_summary(
stamp_token=0,
column=input_column,
example_weights=weights)
with self.test_session() as sess:
# This input is generated by integer in the range [2030, 2060]
# but represented by with float16 precision. Integers <= 2048 are
# exactly represented, whereas numbers > 2048 are rounded; and hence
# numbers > 2048 are repeated. For precision loss / rounding, see:
# https://en.wikipedia.org/wiki/Half-precision_floating-point_format.
#
# The intent of the test is not handling of float16 values, but to
# validate the number of buckets is returned, in cases where the input
# may contain repeated values.
inputs = [
2030.0, 2031.0, 2032.0, 2033.0, 2034.0, 2035.0, 2036.0, 2037.0,
2038.0, 2039.0, 2040.0, 2041.0, 2042.0, 2043.0, 2044.0, 2045.0,
2046.0, 2047.0, 2048.0, 2048.0, 2050.0, 2052.0, 2052.0, 2052.0,
2054.0, 2056.0, 2056.0, 2056.0, 2058.0, 2060.0
]
sess.run(update,
{input_column: inputs,
weights: [1] * len(inputs)})
with self.test_session() as sess:
sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1))
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run(
[buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertEqual(num_quantiles + 1, len(buckets))
self.assertAllEqual([2030, 2040, 2050, 2060], buckets)
def testFullEnsembleMultiNotClassTreePerClassStrategyDenseVector(self):
with self.test_session():
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
# Bias tree only for second class.
tree1 = tree_ensemble_config.trees.add()
tree_ensemble_config.tree_metadata.add().is_finalized = True
_append_multi_values_to_dense_leaf(tree1.nodes.add().leaf, [0, -0.2, -2])
# Depth 2 tree.
tree2 = tree_ensemble_config.trees.add()
tree_ensemble_config.tree_metadata.add().is_finalized = True
_set_float_split(tree2.nodes.add()
.sparse_float_binary_split_default_right.split, 1, 4.0,
1, 2)
_set_float_split(tree2.nodes.add().dense_float_binary_split, 0, 9.0, 3, 4)
_append_multi_values_to_dense_leaf(tree2.nodes.add().leaf, [0.5, 0, 0])
_append_multi_values_to_dense_leaf(tree2.nodes.add().leaf, [0, 1.2, -0.7])
_append_multi_values_to_dense_leaf(tree2.nodes.add().leaf, [-0.9, 0, 0])
tree_ensemble_config.tree_weights.append(1.0)
tree_ensemble_config.tree_weights.append(1.0)
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="ensemble_multi_class")
resources.initialize_resources(resources.shared_resources()).run()
# Prepare learner config.
learner_config = learner_pb2.LearnerConfig()
learner_config.num_classes = 3
learner_config.multi_class_strategy = (
learner_pb2.LearnerConfig.FULL_HESSIAN)
result, dropout_info = self._get_predictions(
tree_ensemble_handle,
learner_config=learner_config.SerializeToString(),
reduce_dim=False)
# The first example will get bias class 1 -0.2 and -2 for class 2 from
# first tree and leaf 2 payload (sparse feature missing) of 0.5 hence
# 0.5, -0.2], the second example will get the same bias and leaf 3 payload
# of class 1 1.2 and class 2-0.7 hence [0.0, 1.0, -2.7].
self.assertAllClose([[0.5, -0.2, -2.0], [0, 1.0, -2.7]], result.eval())
# Empty dropout.
self.assertAllEqual([[], []], dropout_info.eval())
def testWithExistingEnsemble(self):
with self.test_session():
# Create existing tree ensemble.
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=self._tree_ensemble.SerializeToString(),
name="existing")
# Create non-zero feature importance.
feature_usage_counts = variables.Variable(
initial_value=np.array([0, 4, 1], np.int64),
name="feature_usage_counts",
trainable=False)
feature_gains = variables.Variable(
initial_value=np.array([0.0, 0.3, 0.05], np.float32),
name="feature_gains",
trainable=False)
resources.initialize_resources(resources.shared_resources()).run()
variables.initialize_all_variables().run()
output_ensemble = tree_config_pb2.DecisionTreeEnsembleConfig()
with ops.control_dependencies([
ensemble_optimizer_ops.add_trees_to_ensemble(
tree_ensemble_handle,
self._ensemble_to_add.SerializeToString(),
feature_usage_counts, [1, 2, 0],
feature_gains, [0.02, 0.1, 0.0], [[], []],
learning_rate=1)
]):
output_ensemble.ParseFromString(
model_ops.tree_ensemble_serialize(tree_ensemble_handle)[1].eval())
# Output.
self.assertEqual(3, len(output_ensemble.trees))
self.assertProtoEquals(self._tree_to_add, output_ensemble.trees[2])
self.assertAllEqual([1.0, 1.0, 1.0], output_ensemble.tree_weights)
self.assertEqual(2,
output_ensemble.tree_metadata[0].num_tree_weight_updates)
self.assertEqual(3,
output_ensemble.tree_metadata[1].num_tree_weight_updates)
self.assertEqual(1,
output_ensemble.tree_metadata[2].num_tree_weight_updates)
self.assertAllEqual([1, 6, 1], feature_usage_counts.eval())
self.assertArrayNear([0.02, 0.4, 0.05], feature_gains.eval(), 1e-6)
def testStreamingQuantileBucketsWithVaryingBatch(self):
"""Sets up the quantile summary op test as follows.
Creates batches examples with different number of inputs in each batch.
The input values are dense in the range [1 ... N]
The data looks like this:
| Batch | Start | InputList
| 1 | 1 | [1]
| 2 | 2 | [2, 3]
| 3 | 4 | [4, 5, 6]
| 4 | 7 | [7, 8, 9, 10]
| 5 | 11 | [11, 12, 13, 14, 15]
| 6 | 16 | [16, 17, 18, 19, 20, 21]
"""
num_quantiles = 3
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=num_quantiles,
epsilon=0.001, name="q1")
resources.initialize_resources(resources.shared_resources()).run()
input_column = array_ops.placeholder(dtypes.float32)
weights = array_ops.placeholder(dtypes.float32)
update = accumulator.add_summary(
stamp_token=0,
column=input_column,
example_weights=weights)
with self.test_session() as sess:
for i in range(1, 23):
# start = 1, 2, 4, 7, 11, 16 ... (see comment above)
start = int((i * (i-1) / 2) + 1)
sess.run(update,
{input_column: range(start, start+i),
weights: [1] * i})
with self.test_session() as sess:
sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1))
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run(
[buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertEqual(num_quantiles + 1, len(buckets))
self.assertAllEqual([1, 86., 170., 253.], buckets)
def testEnsembleEmpty(self):
with self.test_session():
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble_config.SerializeToString(),
name="full_ensemble")
resources.initialize_resources(resources.shared_resources()).run()
result = prediction_ops.gradient_trees_partition_examples(
tree_ensemble_handle, [self._dense_float_tensor], [
self._sparse_float_indices1, self._sparse_float_indices2
], [self._sparse_float_values1, self._sparse_float_values2],
[self._sparse_float_shape1,
self._sparse_float_shape2], [self._sparse_int_indices1],
[self._sparse_int_values1], [self._sparse_int_shape1])
self.assertAllEqual([0, 0], result.eval())
def testWithEmptyEnsemble(self):
with self.test_session():
# Create an empty ensemble.
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0, tree_ensemble_config="", name="empty")
# Create zero feature importance.
feature_usage_counts = variables.Variable(
initial_value=array_ops.zeros([1], dtypes.int64),
name="feature_usage_counts",
trainable=False)
feature_gains = variables.Variable(
initial_value=array_ops.zeros([1], dtypes.float32),
name="feature_gains",
trainable=False)
resources.initialize_resources(resources.shared_resources()).run()
variables.initialize_all_variables().run()
with ops.control_dependencies([
ensemble_optimizer_ops.add_trees_to_ensemble(
tree_ensemble_handle,
self._ensemble_to_add.SerializeToString(),
feature_usage_counts, [2],
feature_gains, [0.4], [[]],
learning_rate=1.0)
]):
result = model_ops.tree_ensemble_serialize(tree_ensemble_handle)[1]
# Output.
output_ensemble = tree_config_pb2.DecisionTreeEnsembleConfig()
output_ensemble.ParseFromString(result.eval())
self.assertProtoEquals(self._tree_to_add, output_ensemble.trees[0])
self.assertEqual(1, len(output_ensemble.trees))
self.assertAllEqual([1.0], output_ensemble.tree_weights)
self.assertEqual(1,
output_ensemble.tree_metadata[0].num_tree_weight_updates)
self.assertAllEqual([2], feature_usage_counts.eval())
self.assertArrayNear([0.4], feature_gains.eval(), 1e-6)
def testWithEmptyEnsembleAndShrinkage(self):
with self.test_session():
# Add shrinkage config.
learning_rate = 0.0001
tree_ensemble = tree_config_pb2.DecisionTreeEnsembleConfig()
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config=tree_ensemble.SerializeToString(),
name="existing")
# Create zero feature importance.
feature_usage_counts = variables.Variable(
initial_value=np.array([0, 0], np.int64),
name="feature_usage_counts",
trainable=False)
feature_gains = variables.Variable(
initial_value=np.array([0.0, 0.0], np.float32),
name="feature_gains",
trainable=False)
resources.initialize_resources(resources.shared_resources()).run()
variables.initialize_all_variables().run()
output_ensemble = tree_config_pb2.DecisionTreeEnsembleConfig()
with ops.control_dependencies([
ensemble_optimizer_ops.add_trees_to_ensemble(
tree_ensemble_handle,
self._ensemble_to_add.SerializeToString(),
feature_usage_counts, [1, 2],
feature_gains, [0.5, 0.3], [[], []],
learning_rate=learning_rate)
]):
output_ensemble.ParseFromString(
model_ops.tree_ensemble_serialize(tree_ensemble_handle)[1].eval())
# New tree is added with shrinkage weight.
self.assertAllClose([learning_rate], output_ensemble.tree_weights)
self.assertEqual(1,
output_ensemble.tree_metadata[0].num_tree_weight_updates)
self.assertAllEqual([1, 2], feature_usage_counts.eval())
self.assertArrayNear([0.5 * learning_rate, 0.3 * learning_rate],
feature_gains.eval(), 1e-6)
def testStreamingQuantileBuckets(self):
"""Sets up the quantile summary op test as follows.
Create a batch of 6 examples having a dense and sparse features.
The data looks like this
| Instance | instance weights | Dense 0
| 0 | 10 | 1
| 1 | 1 | 2
| 2 | 1 | 3
| 3 | 1 | 4
| 4 | 1 | 4
| 5 | 1 | 5
"""
dense_float_tensor_0 = np.array([1, 2, 3, 4, 4, 5])
example_weights = np.array([10, 1, 1, 1, 1, 1])
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(init_stamp_token=0,
num_quantiles=3,
epsilon=0.33,
name="q1")
resources.initialize_resources(resources.shared_resources()).run()
are_ready_noflush, _, = (accumulator.get_buckets(stamp_token=0))
update = accumulator.add_summary(stamp_token=0,
column=dense_float_tensor_0,
example_weights=example_weights)
with ops.control_dependencies([are_ready_noflush, update]):
reset = accumulator.flush(stamp_token=0, next_stamp_token=1)
with ops.control_dependencies([reset]):
are_ready_flush, buckets = (accumulator.get_buckets(
stamp_token=1))
buckets, are_ready_noflush, are_ready_flush = (sess.run(
[buckets, are_ready_noflush, are_ready_flush]))
self.assertEqual(False, are_ready_noflush)
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([1, 3, 5], buckets)
def testSaveRestoreAfterFlush(self):
save_dir = os.path.join(self.get_temp_dir(), "save_restore")
save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
with self.test_session(graph=ops.Graph()) as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=3, epsilon=0.33, name="q0")
save = saver.Saver()
resources.initialize_resources(resources.shared_resources()).run()
example_weights = constant_op.constant(
[10, 1, 1, 1, 1, 1], dtype=dtypes.float32, shape=[6, 1])
dense_float_tensor_0 = constant_op.constant(
[1, 2, 3, 4, 4, 5], dtype=dtypes.float32, shape=[6, 1])
update = accumulator.add_summary(
stamp_token=0,
column=dense_float_tensor_0,
example_weights=example_weights)
update.run()
reset = accumulator.flush(stamp_token=0, next_stamp_token=1)
with ops.control_dependencies([reset]):
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([1, 3, 5], buckets)
save.save(sess, save_path)
with self.test_session(graph=ops.Graph()) as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=3, epsilon=0.33, name="q0")
save = saver.Saver()
# Restore the saved values in the parameter nodes.
save.restore(sess, save_path)
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([1, 3, 5], buckets)
def testBasicQuantileBucketsSingleResource(self):
with self.cached_session() as sess:
quantile_accumulator_handle = self.create_resource("floats", self.eps,
self.max_elements, 2)
resources.initialize_resources(resources.shared_resources()).run()
summaries = boosted_trees_ops.make_quantile_summaries(
[self._feature_0, self._feature_1], self._example_weights,
epsilon=self.eps)
summary_op = boosted_trees_ops.quantile_add_summaries(
quantile_accumulator_handle, summaries)
flush_op = boosted_trees_ops.quantile_flush(
quantile_accumulator_handle, self.num_quantiles)
buckets = boosted_trees_ops.get_bucket_boundaries(
quantile_accumulator_handle, num_features=2)
quantiles = boosted_trees_ops.boosted_trees_bucketize(
[self._feature_0, self._feature_1], buckets)
sess.run(summary_op)
sess.run(flush_op)
self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
self.assertAllClose(self._feature_0_quantiles, quantiles[0].eval())
self.assertAllClose(self._feature_1_quantiles, quantiles[1].eval())
def set_parameter(self, param):
for name in self.default_param:
if name not in param:
param[name] = self.default_param[name]
self.build_model()
num_trees = param['num_trees']
max_nodes = param['max_nodes']
# Random Forest Parameters
self.hparams = tensor_forest.ForestHParams(
num_classes=self.class_num,
num_features=self.feature_num,
num_trees=num_trees,
max_nodes=max_nodes).fill()
# Build the Random Forest
self.forest_graph = tensor_forest.RandomForestGraphs(self.hparams)
# Get training graph and loss
self.train_op = self.forest_graph.training_graph(
self.inputs, self.labels)
self.loss = self.forest_graph.training_loss(self.inputs, self.labels)
# Measure the accuracy
self.infer_op, _, _ = self.forest_graph.inference_graph(self.inputs)
self.correct_prediction = tf.equal(tf.argmax(self.infer_op, 1),
tf.cast(self.labels, tf.int64))
self.accuracy = tf.reduce_mean(
tf.cast(self.correct_prediction, tf.float32))
#metrics = [self.get_metric(metric) for metric in param["metrics"]]
#self.metrics = [metric_fun(self.output, self.ground_truth) for metric_fun in metrics]
self.init_vars = tf.group(
tf.global_variables_initializer(),
resources.initialize_resources(resources.shared_resources()))
self.batch_size = param["batch_size"]
self.num_epochs = param["num_epochs"]
def testSaveRestoreAfterFlush(self):
save_dir = os.path.join(self.get_temp_dir(), "save_restore")
save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
with self.cached_session() as sess:
accumulator = boosted_trees_ops.QuantileAccumulator(
num_streams=2,
num_quantiles=self.num_quantiles,
epsilon=self.eps,
name="q0")
save = saver.Saver()
resources.initialize_resources(resources.shared_resources()).run()
buckets = accumulator.get_bucket_boundaries()
self.assertAllClose([], buckets[0].eval())
self.assertAllClose([], buckets[1].eval())
summaries = accumulator.add_summaries(
[self._feature_0, self._feature_1], self._example_weights)
with ops.control_dependencies([summaries]):
flush = accumulator.flush()
self.evaluate(flush)
self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
save.save(sess, save_path)
with self.session(graph=ops.Graph()) as sess:
accumulator = boosted_trees_ops.QuantileAccumulator(
num_streams=2,
num_quantiles=self.num_quantiles,
epsilon=self.eps,
name="q0")
save = saver.Saver()
save.restore(sess, save_path)
buckets = accumulator.get_bucket_boundaries()
self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
def _testStreamingQuantileBucketsHelper(self,
inputs,
num_quantiles=3,
expected_buckets=None):
"""Helper to test quantile buckets on different inputs."""
# set generate_quantiles to True since the test will generate fewer
# boundaries otherwise.
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0,
num_quantiles=num_quantiles,
epsilon=0.001,
name="q1",
generate_quantiles=True)
resources.initialize_resources(resources.shared_resources()).run()
input_column = array_ops.placeholder(dtypes.float32)
weights = array_ops.placeholder(dtypes.float32)
update = accumulator.add_summary(stamp_token=0,
column=input_column,
example_weights=weights)
with self.test_session() as sess:
sess.run(update, {
input_column: inputs,
weights: [1] * len(inputs)
})
with self.test_session() as sess:
sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1))
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
# By default, use 3 quantiles, 4 boundaries for simplicity.
self.assertEqual(num_quantiles + 1, len(buckets))
if expected_buckets:
self.assertAllEqual(buckets, expected_buckets)
# Build the Random Forest
forest_graph = tensor_forest.RandomForestGraphs(hparams)
# Get training graph and loss
train_op = forest_graph.training_graph(X, Y)
loss_op = forest_graph.training_loss(X, Y)
# Measure the accuracy
infer_op, _, _ = forest_graph.inference_graph(X)
correct_prediction = tf.equal(tf.argmax(infer_op, 1), tf.cast(Y, tf.int64))
accuracy_op = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# Initialize the variables (i.e. assign their default value) and forest resources
init_vars = tf.group(
tf.global_variables_initializer(),
resources.initialize_resources(resources.shared_resources()))
# Start TensorFlow session
sess = tf.train.MonitoredSession()
# Run the initializer
sess.run(init_vars)
# Training
for i in range(1, num_steps + 1):
# Prepare Data
# Get the next batch of MNIST data (only images are needed, not labels)
# batch_x, batch_y = mnist.train.next_batch(batch_size)
_, l = sess.run([train_op, loss_op], feed_dict={X: input_x, Y: input_y})
if i % 50 == 0 or i == 1:
acc = sess.run(accuracy_op, feed_dict={X: X_train, Y: y_train})
def testPredictionMultipleTree(self):
"""Tests the predictions work when we have multiple trees."""
with self.test_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
bucketized_split {
feature_id: 1
threshold: 28
left_id: 1
right_id: 2
}
metadata {
gain: 7.62
}
}
nodes {
leaf {
scalar: 1.14
}
}
nodes {
leaf {
scalar: 8.79
}
}
}
trees {
nodes {
bucketized_split {
feature_id: 1
threshold: 26
left_id: 1
right_id: 2
}
}
nodes {
bucketized_split {
feature_id: 0
threshold: 50
left_id: 3
right_id: 4
}
}
nodes {
leaf {
scalar: 7.0
}
}
nodes {
leaf {
scalar: 5.0
}
}
nodes {
leaf {
scalar: 6.0
}
}
}
trees {
nodes {
bucketized_split {
feature_id: 0
threshold: 34
left_id: 1
right_id: 2
}
}
nodes {
leaf {
scalar: -7.0
}
}
nodes {
leaf {
scalar: 5.0
}
}
}
tree_weights: 0.1
tree_weights: 0.2
tree_weights: 1.0
""", tree_ensemble_config)
# Create existing ensemble with one root split
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
feature_0_values = [36, 32]
feature_1_values = [11, 27]
# Example 1: tree 0: 1.14, tree 1: 5.0, tree 2: 5.0 = >
# logit = 0.1*5.0+0.2*5.0+1*5
# Example 2: tree 0: 1.14, tree 1: 7.0, tree 2: -7 = >
# logit= 0.1*1.14+0.2*7.0-1*7.0
expected_logits = [[6.114], [-5.486]]
# Do with parallelization, e.g. EVAL
predict_op = boosted_trees_ops.predict(
tree_ensemble_handle,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits = session.run(predict_op)
self.assertAllClose(expected_logits, logits)
# Do without parallelization, e.g. INFER - the result is the same
predict_op = boosted_trees_ops.predict(
tree_ensemble_handle,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits = session.run(predict_op)
self.assertAllClose(expected_logits, logits)
def testCachedPredictionTheWholeTreeWasPruned(self):
"""Tests that prediction based on previous node in the tree works."""
with self.test_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
leaf {
scalar: 0.00
}
}
}
tree_weights: 1.0
tree_metadata {
num_layers_grown: 1
is_finalized: true
post_pruned_nodes_meta {
new_node_id: 0
logit_change: 0.0
}
post_pruned_nodes_meta {
new_node_id: 0
logit_change: -6.0
}
post_pruned_nodes_meta {
new_node_id: 0
logit_change: 5.0
}
}
growing_metadata {
num_trees_attempted: 1
num_layers_attempted: 1
}
""", tree_ensemble_config)
# Create existing ensemble.
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
cached_tree_ids = [
0,
0,
]
# The predictions were cached in 1 and 2, both were pruned to the root.
cached_node_ids = [1, 2]
# We have two features: 0 and 1.These are not going to be used anywhere.
feature_0_values = [12, 17]
feature_1_values = [12, 12]
# Grow tree ensemble.
predict_op = boosted_trees_ops.training_predict(
tree_ensemble_handle,
cached_tree_ids=cached_tree_ids,
cached_node_ids=cached_node_ids,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits_updates, new_tree_ids, new_node_ids = session.run(
predict_op)
# We are in the last tree.
self.assertAllClose([0, 0], new_tree_ids)
self.assertAllClose([0, 0], new_node_ids)
self.assertAllClose([[-6.0], [5.0]], logits_updates)
def testNoCachedPredictionButTreeExists(self):
"""Tests that predictions are updated once trees are added."""
with self.test_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
bucketized_split {
feature_id: 0
threshold: 15
left_id: 1
right_id: 2
}
metadata {
gain: 7.62
}
}
nodes {
leaf {
scalar: 1.14
}
}
nodes {
leaf {
scalar: 8.79
}
}
}
tree_weights: 0.1
tree_metadata {
is_finalized: true
num_layers_grown: 1
}
growing_metadata {
num_trees_attempted: 1
num_layers_attempted: 2
}
""", tree_ensemble_config)
# Create existing ensemble with one root split
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
# Two examples, none were cached before.
cached_tree_ids = [0, 0]
cached_node_ids = [0, 0]
feature_0_values = [67, 5]
# Grow tree ensemble.
predict_op = boosted_trees_ops.training_predict(
tree_ensemble_handle,
cached_tree_ids=cached_tree_ids,
cached_node_ids=cached_node_ids,
bucketized_features=[feature_0_values],
logits_dimension=1)
logits_updates, new_tree_ids, new_node_ids = session.run(
predict_op)
# We are in the first tree.
self.assertAllClose([0, 0], new_tree_ids)
self.assertAllClose([2, 1], new_node_ids)
self.assertAllClose([[0.1 * 8.79], [0.1 * 1.14]], logits_updates)
def testCategoricalSplits(self):
"""Tests the training prediction work for categorical splits."""
with self.cached_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
categorical_split {
feature_id: 1
value: 2
left_id: 1
right_id: 2
}
}
nodes {
categorical_split {
feature_id: 0
value: 13
left_id: 3
right_id: 4
}
}
nodes {
leaf {
scalar: 7.0
}
}
nodes {
leaf {
scalar: 5.0
}
}
nodes {
leaf {
scalar: 6.0
}
}
}
tree_weights: 1.0
tree_metadata {
is_finalized: true
}
""", tree_ensemble_config)
# Create existing ensemble with one root split
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
feature_0_values = [13, 1, 3]
feature_1_values = [2, 2, 1]
# No previous cached values.
cached_tree_ids = [0, 0, 0]
cached_node_ids = [0, 0, 0]
# Grow tree ensemble.
predict_op = boosted_trees_ops.training_predict(
tree_ensemble_handle,
cached_tree_ids=cached_tree_ids,
cached_node_ids=cached_node_ids,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
self.assertAllClose([0, 0, 0], new_tree_ids)
self.assertAllClose([3, 4, 2], new_node_ids)
self.assertAllClose([[5.], [6.], [7.]], logits_updates)
def testRestore(self):
# Calling self.test_session() without a graph specified results in
# TensorFlowTestCase caching the session and returning the same one
# every time. In this test, we need to create two different sessions
# which is why we also create a graph and pass it to self.test_session()
# to ensure no caching occurs under the hood.
save_path = os.path.join(self.get_temp_dir(), "restore-test")
with ops.Graph().as_default() as graph:
with self.test_session(graph) as sess:
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig(
)
tree = tree_ensemble_config.trees.add()
tree_ensemble_config.tree_metadata.add().is_finalized = True
tree_ensemble_config.tree_weights.append(1.0)
_append_to_leaf(tree.nodes.add().leaf, 0, -0.1)
tree_ensemble_config2 = tree_config_pb2.DecisionTreeEnsembleConfig(
)
tree2 = tree_ensemble_config2.trees.add()
tree_ensemble_config.tree_weights.append(1.0)
_append_to_leaf(tree2.nodes.add().leaf, 0, -1.0)
tree_ensemble_config3 = tree_config_pb2.DecisionTreeEnsembleConfig(
)
tree3 = tree_ensemble_config3.trees.add()
tree_ensemble_config.tree_weights.append(1.0)
_append_to_leaf(tree3.nodes.add().leaf, 0, -10.0)
# Prepare learner config.
learner_config = learner_pb2.LearnerConfig()
learner_config.num_classes = 2
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=3,
tree_ensemble_config=tree_ensemble_config.
SerializeToString(),
name="restore_tree")
feature_usage_counts = variables.Variable(
initial_value=array_ops.zeros([1], dtypes.int64),
name="feature_usage_counts",
trainable=False)
feature_gains = variables.Variable(
initial_value=array_ops.zeros([1], dtypes.float32),
name="feature_gains",
trainable=False)
resources.initialize_resources(
resources.shared_resources()).run()
variables.initialize_all_variables().run()
my_saver = saver.Saver()
with ops.control_dependencies([
ensemble_optimizer_ops.add_trees_to_ensemble(
tree_ensemble_handle,
tree_ensemble_config2.SerializeToString(),
feature_usage_counts, [0],
feature_gains, [0], [[]],
learning_rate=1)
]):
result, _, _ = prediction_ops.gradient_trees_prediction(
tree_ensemble_handle,
self._seed, [self._dense_float_tensor], [
self._sparse_float_indices1,
self._sparse_float_indices2
], [
self._sparse_float_values1,
self._sparse_float_values2
],
[self._sparse_float_shape1, self._sparse_float_shape2],
[self._sparse_int_indices1],
[self._sparse_int_values1], [self._sparse_int_shape1],
learner_config=learner_config.SerializeToString(),
apply_dropout=False,
apply_averaging=False,
center_bias=False,
reduce_dim=True)
self.assertAllClose([[-1.1], [-1.1]], result.eval())
# Save before adding other trees.
val = my_saver.save(sess, save_path)
self.assertEqual(save_path, val)
# Add more trees after saving.
with ops.control_dependencies([
ensemble_optimizer_ops.add_trees_to_ensemble(
tree_ensemble_handle,
tree_ensemble_config3.SerializeToString(),
feature_usage_counts, [0],
feature_gains, [0], [[]],
learning_rate=1)
]):
result, _, _ = prediction_ops.gradient_trees_prediction(
tree_ensemble_handle,
self._seed, [self._dense_float_tensor], [
self._sparse_float_indices1,
self._sparse_float_indices2
], [
self._sparse_float_values1,
self._sparse_float_values2
],
[self._sparse_float_shape1, self._sparse_float_shape2],
[self._sparse_int_indices1],
[self._sparse_int_values1], [self._sparse_int_shape1],
learner_config=learner_config.SerializeToString(),
apply_dropout=False,
apply_averaging=False,
center_bias=False,
reduce_dim=True)
self.assertAllClose(result.eval(), [[-11.1], [-11.1]])
# Start a second session. In that session the parameter nodes
# have not been initialized either.
with ops.Graph().as_default() as graph:
with self.test_session(graph) as sess:
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=0,
tree_ensemble_config="",
name="restore_tree")
my_saver = saver.Saver()
my_saver.restore(sess, save_path)
result, _, _ = prediction_ops.gradient_trees_prediction(
tree_ensemble_handle,
self._seed, [self._dense_float_tensor],
[self._sparse_float_indices1, self._sparse_float_indices2],
[self._sparse_float_values1, self._sparse_float_values2],
[self._sparse_float_shape1, self._sparse_float_shape2],
[self._sparse_int_indices1], [self._sparse_int_values1],
[self._sparse_int_shape1],
learner_config=learner_config.SerializeToString(),
apply_dropout=False,
apply_averaging=False,
center_bias=False,
reduce_dim=True)
# Make sure we only have the first and second tree.
# The third tree was added after the save.
self.assertAllClose(result.eval(), [[-1.1], [-1.1]])
hparams = tensor_forest.ForestHParams(num_classes=n_classes, num_features=n_features, num_trees=n_trees,
max_nodes=max_nodes, split_after_samples=30).fill()
forest_graph = tensor_forest.RandomForestGraphs(hparams)
train_op = forest_graph.training_graph(x, y)
loss_op = forest_graph.training_loss(x, y)
infer_op, _, _ = forest_graph.inference_graph(x)
auc = tf.metrics.auc(tf.cast(y, tf.int64), infer_op[:, 1])[1]
init_vars = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer(), resources.initialize_resources(resources.shared_resources()))
sess = tf.Session()
sess.run(init_vars)
batch_size = 1000
import numpy as np
indices = list(range(n_train))
def gen_batch(indices):
np.random.shuffle(indices)
for batch_i in range(int(n_train / batch_size)):
batch_index = indices[batch_i*batch_size: (batch_i+1)*batch_size]
yield X_train_enc[batch_index], Y_train[batch_index]
def initialize_local_state(self, tf_config=None):
"""Called by the CombineFnWrapper's __init__ method.
This can be used to set non-pickleable local state. It is used in
conjunction with overriding __reduce__ so this state is not pickled. This
method must be called prior to any other method.
Args:
tf_config: (optional) A tf.ConfigProto
"""
# stamp_token is used to commit the state of the qaccumulator. In
# this case, the qaccumulator state is completely returned and stored
# as part of quantile_state/summary in the combiner fn (i.e the summary is
# extracted and stored outside the qaccumulator). So we don't use
# the timestamp mechanism to signify progress in the qaccumulator state.
stamp_token = 0
# Create a new session with a new graph for quantile ops.
self._session = tf.Session(graph=tf.Graph(), config=tf_config)
with self._session.graph.as_default():
with self._session.as_default():
self._qaccumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=stamp_token,
num_quantiles=self._num_quantiles,
epsilon=self._epsilon,
name='qaccumulator')
resources.initialize_resources(resources.shared_resources()).run()
# Create placeholder that will be used to provide input the
# QuantileAccumulator. Has shape (1, None) as this is what the
# QuantileAccumulator accepts.
self._add_summary_input = tf.placeholder(
dtype=self._bucket_numpy_dtype, shape=[1, None])
# Create op to update the accumulator with new input fed from
# self._qaccumulator_input.
self._add_summary_op = self._qaccumulator.add_summary(
stamp_token=stamp_token,
column=self._add_summary_input,
# All weights are equal, and the weight vector is the
# same length as the input.
example_weights=tf.ones_like(self._add_summary_input))
# Create op to add a prebuilt summary to the accumulator, and a
# placeholder tensor to provide the input for this op.
self._prebuilt_summary_input = tf.placeholder(
dtype=tf.string, shape=[])
self._add_prebuilt_summary_op = self._qaccumulator.add_prebuilt_summary(
stamp_token=stamp_token,
summary=self._prebuilt_summary_input)
# Create op to flush summaries and return a summary representing the
# summaries that were added the accumulator so far.
self._flush_summary_op = self._qaccumulator.flush_summary(
stamp_token=stamp_token,
next_stamp_token=stamp_token)
# Create ops to flush the accumulator and return approximate boundaries.
self._flush_op = self._qaccumulator.flush(
stamp_token=stamp_token,
next_stamp_token=stamp_token)
_, self._buckets_op = self._qaccumulator.get_buckets(
stamp_token=stamp_token)
# We generate an empty summary by calling self._flush_summary_op.
# We cache this as some implementations may call create_accumulator for
# every input, and it can be cached since it will always be the same and
# immutable.
self._empty_summary = self._session.run(self._flush_summary_op)
def testInfrenceFromRestoredModel(self):
input_data = [
[-1., 0.],
[-1., 2.], # node 1
[1., 0.],
[1., -2.]
] # node 2
expected_prediction = [[0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0]]
hparams = tensor_forest.ForestHParams(num_classes=2,
num_features=2,
num_trees=1,
max_nodes=1000,
split_after_samples=25).fill()
tree_weight = {
'decisionTree': {
'nodes': [{
'binaryNode': {
'rightChildId': 2,
'leftChildId': 1,
'inequalityLeftChildTest': {
'featureId': {
'id': '0'
},
'threshold': {
'floatValue': 0
}
}
}
}, {
'leaf': {
'vector': {
'value': [{
'floatValue': 0.0
}, {
'floatValue': 1.0
}]
}
},
'nodeId': 1
}, {
'leaf': {
'vector': {
'value': [{
'floatValue': 0.0
}, {
'floatValue': 1.0
}]
}
},
'nodeId': 2
}]
}
}
restored_tree_param = ParseDict(
tree_weight, _tree_proto.Model()).SerializeToString()
graph_builder = tensor_forest.RandomForestGraphs(
hparams, [restored_tree_param])
probs, paths, var = graph_builder.inference_graph(input_data)
self.assertTrue(isinstance(probs, ops.Tensor))
self.assertTrue(isinstance(paths, ops.Tensor))
self.assertTrue(isinstance(var, ops.Tensor))
with self.cached_session():
variables.global_variables_initializer().run()
resources.initialize_resources(resources.shared_resources()).run()
self.assertEquals(probs.eval().shape, (4, 2))
self.assertEquals(probs.eval().tolist(), expected_prediction)
def testCategoricalSplits(self):
"""Tests the predictions work for categorical splits."""
with self.cached_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
categorical_split {
feature_id: 1
value: 2
left_id: 1
right_id: 2
}
}
nodes {
categorical_split {
feature_id: 0
value: 13
left_id: 3
right_id: 4
}
}
nodes {
leaf {
scalar: 7.0
}
}
nodes {
leaf {
scalar: 5.0
}
}
nodes {
leaf {
scalar: 6.0
}
}
}
tree_weights: 1.0
""", tree_ensemble_config)
# Create existing ensemble with one root split
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
feature_0_values = [13, 1, 3]
feature_1_values = [2, 2, 1]
expected_logits = [[5.], [6.], [7.]]
# Prediction should work fine.
predict_op = boosted_trees_ops.predict(
tree_ensemble_handle,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits = session.run(predict_op)
self.assertAllClose(expected_logits, logits)
def run_LAmbDA2(gamma, delta, tau, prc_cut, bs_prc, num_trees, max_nodes):
global X, Y, Gnp, Dnp, train, test, prt, cv
D = tf.cast(Dnp, tf.float32)
G = tf.cast(Gnp, tf.float32)
#optunity_it = optunity_it+1;
num_trees = int(num_trees)
max_nodes = int(max_nodes)
prc_cut = int(np.ceil(prc_cut))
print(
"gamma=%.4f, delta=%.4f, tau=%.4f, prc_cut=%i, bs_prc=%.4f, num_trees=%i, max_nodes=%i"
% (gamma, delta, tau, prc_cut, bs_prc, num_trees, max_nodes))
input_feats = X.shape[1]
num_labls = G.shape.as_list()
output_feats = num_labls[1]
#print(output_feats)
num_labls = num_labls[0]
rowsums = np.sum(Gnp, axis=1)
train2 = resample(prc_cut, Y, Gnp, train, gamma)
# Bug??
bs = int(np.ceil(bs_prc * train2.size))
xs = tf.placeholder(tf.float32, [None, input_feats])
#ys = tf.placeholder(tf.float32, [None,num_labls])
yin = tf.placeholder(tf.int32, [None])
print("Vars loaded xs and ys created")
hparams = tensor_forest.ForestHParams(num_classes=output_feats,
num_features=input_feats,
num_trees=num_trees,
max_nodes=max_nodes).fill()
print("Tensor forest hparams created")
forest_graph = tensor_forest.RandomForestGraphs(hparams)
print("Tensor forest graph created")
train_op = forest_graph.training_graph(xs, yin)
loss_op = forest_graph.training_loss(xs, yin)
print("Loss and train ops created")
predict, _, _ = forest_graph.inference_graph(xs)
print("Tensor forest variables created through predict")
accuracy_op = tf.reduce_mean(
tf.reduce_sum(tf.square(tf.one_hot(yin, output_feats) - predict),
reduction_indices=[1]))
print(
tf.reduce_sum(tf.square(tf.one_hot(yin, output_feats) - predict),
reduction_indices=[1]))
#predict = tf.one_hot(pred);
print("Lambda specific variables created")
# Creating training and testing steps
G2 = np.copy(Gnp)
G2[rowsums > 1, :] = 0
YI = np.matmul(Y, G2)
YIrs = np.sum(YI, axis=1)
trainI = train2[np.in1d(train2, np.where(YIrs == 1))]
print("data type trainI,", trainI.dtype)
testI = test[np.in1d(test, np.where(YIrs == 1))]
print("trainI testI created")
#init_vars=tf.global_variables_initializer()
init_vars = tf.group(
tf.global_variables_initializer(),
resources.initialize_resources(resources.shared_resources()))
sess = tf.Session()
sess.run(init_vars)
print("Session started")
#beep = sess.run(predict,feed_dict={xs:X[1:100,:]});
#beep = sess.run(predict,feed_dict={xs:X[train2[0:bs],:]});
tensor_trainI = {
xs: X[trainI, :],
yin: sess.run(tf.argmax(get_yi(rowsums, G2, Y[trainI, :]), axis=1))
}
print("tensor_trainI made")
tensor_testI = {
xs: X[testI, :],
yin: sess.run(tf.argmax(get_yi(rowsums, G2, Y[testI, :]), axis=1))
}
print("tensor_testI made")
tensor_train = {
xs:
X[train2[0:bs], :],
yin:
sess.run(
tf.argmax(get_yn(
sess.run(predict, feed_dict={xs: X[train2[0:bs], :]}),
Y[train2[0:bs], :], delta, tau, output_feats),
axis=1))
}
print("tensor_train made")
tensor_test = {
xs:
X[test, :],
yin:
sess.run(
tf.argmax(get_yn(sess.run(predict, feed_dict={xs: X[test, :]}),
Y[test, :], delta, tau, output_feats),
axis=1))
}
print("tensor_test made")
#**********************************
#print("Loss and training steps created with sample tensors")
# Setting params and initializing
print("Beginning iterations")
# Starting training iterations
print(X.shape)
for i in range(1, 101):
if i < 50:
sess.run(train_op, feed_dict=tensor_trainI)
#print("ran train op")
if i % 10 == 0:
print(
str(sess.run(accuracy_op, feed_dict=tensor_trainI)) + ' ' +
str(sess.run(accuracy_op, feed_dict=tensor_testI)))
else:
sess.run(train_op, feed_dict=tensor_train)
if i % 10 == 0:
print(
str(sess.run(accuracy_op, feed_dict=tensor_train)) + ' ' +
str(sess.run(accuracy_op, feed_dict=tensor_test)))
elif i % 10 == 0:
np.random_shuffle(train2)
tensor_train = {
xs:
X[train2[0:bs], :],
yin:
sess.run(
get_yn(
sess.run(predict,
feed_dict={xs: X[train2[0:bs], :]}),
Y[train2[0:bs], :], delta, tau, output_feats))
}
if prt:
blah = sess.run(predict, feed_dict=tensor_test)
sio.savemat('preds_cv' + str(cv) + '.mat', {'preds': blah})
sio.savemat('truth_cv' + str(cv) + '.mat', {'labels': Y[test, :]})
acc = sess.run(accuracy_op, feed_dict=tensor_test)
print(
"loss1=%.4f, gamma=%.4f, delta=%.4f, tau=%.4f, prc_cut=%i, bs_prc=%.4f, num_trees=%i, max_nodes=%i"
% (acc, gamma, delta, tau, prc_cut, bs_prc, num_trees, max_nodes))
tf.reset_default_graph()
return (acc)
def testContribsMultipleTree(self):
"""Tests that the contribs work when we have multiple trees."""
with self.cached_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
bucketized_split {
feature_id: 2
threshold: 28
left_id: 1
right_id: 2
}
metadata {
gain: 7.62
original_leaf: {scalar: 2.1}
}
}
nodes {
leaf {
scalar: 1.14
}
}
nodes {
leaf {
scalar: 8.79
}
}
}
trees {
nodes {
bucketized_split {
feature_id: 2
threshold: 26
left_id: 1
right_id: 2
}
}
nodes {
bucketized_split {
feature_id: 0
threshold: 50
left_id: 3
right_id: 4
}
metadata {
original_leaf: {scalar: 5.5}
}
}
nodes {
leaf {
scalar: 7.0
}
}
nodes {
leaf {
scalar: 5.0
}
}
nodes {
leaf {
scalar: 6.0
}
}
}
trees {
nodes {
bucketized_split {
feature_id: 0
threshold: 34
left_id: 1
right_id: 2
}
}
nodes {
leaf {
scalar: -7.0
}
}
nodes {
leaf {
scalar: 5.0
}
}
}
tree_weights: 0.1
tree_weights: 0.2
tree_weights: 1.0
tree_metadata: {
num_layers_grown: 1}
tree_metadata: {
num_layers_grown: 2}
tree_metadata: {
num_layers_grown: 1}
""", tree_ensemble_config)
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
feature_0_values = [36, 32]
feature_1_values = [13, -29
] # Unused. Feature is not in above ensemble.
feature_2_values = [11, 27]
# Expected logits are computed by traversing the logit path and
# subtracting child logits from parent logits.
bias = 2.1 * 0.1 # Root node of tree_0.
expected_feature_ids = ((2, 2, 0, 0), (2, 2, 0))
# example_0 : (bias, 0.1 * 1.14, 0.2 * 5.5 + .114, 0.2 * 5. + .114,
# 1.0 * 5.0 + 0.2 * 5. + .114)
# example_1 : (bias, 0.1 * 1.14, 0.2 * 7 + .114,
# 1.0 * -7. + 0.2 * 7 + .114)
expected_logits_paths = ((bias, 0.114, 1.214, 1.114, 6.114),
(bias, 0.114, 1.514, -5.486))
bucketized_features = [
feature_0_values, feature_1_values, feature_2_values
]
debug_op = boosted_trees_ops.example_debug_outputs(
tree_ensemble_handle,
bucketized_features=bucketized_features,
logits_dimension=1)
serialized_examples_debug_outputs = session.run(debug_op)
feature_ids = []
logits_paths = []
for example in serialized_examples_debug_outputs:
example_debug_outputs = boosted_trees_pb2.DebugOutput()
example_debug_outputs.ParseFromString(example)
feature_ids.append(example_debug_outputs.feature_ids)
logits_paths.append(example_debug_outputs.logits_path)
self.assertAllClose(feature_ids, expected_feature_ids)
self.assertAllClose(logits_paths, expected_logits_paths)
def default_init_op():
return control_flow_ops.group(
variables.global_variables_initializer(),
resources.initialize_resources(
resources.shared_resources()))
def testCachedPredictionFromPreviousTree(self):
"""Tests the predictions work when we have cache from previous trees."""
with self.test_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
bucketized_split {
feature_id: 1
threshold: 28
left_id: 1
right_id: 2
}
metadata {
gain: 7.62
}
}
nodes {
leaf {
scalar: 1.14
}
}
nodes {
leaf {
scalar: 8.79
}
}
}
trees {
nodes {
bucketized_split {
feature_id: 1
threshold: 26
left_id: 1
right_id: 2
}
}
nodes {
bucketized_split {
feature_id: 0
threshold: 50
left_id: 3
right_id: 4
}
}
nodes {
leaf {
scalar: 7
}
}
nodes {
leaf {
scalar: 5
}
}
nodes {
leaf {
scalar: 6
}
}
}
trees {
nodes {
bucketized_split {
feature_id: 0
threshold: 34
left_id: 1
right_id: 2
}
}
nodes {
leaf {
scalar: -7.0
}
}
nodes {
leaf {
scalar: 5.0
}
}
}
tree_metadata {
is_finalized: true
}
tree_metadata {
is_finalized: true
}
tree_metadata {
is_finalized: false
}
tree_weights: 0.1
tree_weights: 0.1
tree_weights: 0.1
""", tree_ensemble_config)
# Create existing ensemble with one root split
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
# Two examples, one was cached in node 1 first, another in node 2.
cached_tree_ids = [0, 0]
cached_node_ids = [1, 0]
# We have two features: 0 and 1.
feature_0_values = [36, 32]
feature_1_values = [11, 27]
# Grow tree ensemble.
predict_op = boosted_trees_ops.training_predict(
tree_ensemble_handle,
cached_tree_ids=cached_tree_ids,
cached_node_ids=cached_node_ids,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits_updates, new_tree_ids, new_node_ids = session.run(
predict_op)
# Example 1 will get to node 3 in tree 1 and node 2 of tree 2
# Example 2 will get to node 2 in tree 1 and node 1 of tree 2
# We are in the last tree.
self.assertAllClose([2, 2], new_tree_ids)
# When using the full tree, the first example will end up in node 4,
# the second in node 5.
self.assertAllClose([2, 1], new_node_ids)
# Example 1: tree 0: 8.79, tree 1: 5.0, tree 2: 5.0 = >
# change = 0.1*(5.0+5.0)
# Example 2: tree 0: 1.14, tree 1: 7.0, tree 2: -7 = >
# change= 0.1(1.14+7.0-7.0)
self.assertAllClose([[1], [0.114]], logits_updates)
def testSerialization(self):
with ops.Graph().as_default() as graph:
with self.test_session(graph):
tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig(
)
# Bias tree only for second class.
tree1 = tree_ensemble_config.trees.add()
_append_to_leaf(tree1.nodes.add().leaf, 1, -0.2)
tree_ensemble_config.tree_weights.append(1.0)
# Depth 2 tree.
tree2 = tree_ensemble_config.trees.add()
tree_ensemble_config.tree_weights.append(1.0)
_set_float_split(
tree2.nodes.add().sparse_float_binary_split_default_right.
split, 1, 4.0, 1, 2)
_set_float_split(tree2.nodes.add().dense_float_binary_split, 0,
9.0, 3, 4)
_append_to_leaf(tree2.nodes.add().leaf, 0, 0.5)
_append_to_leaf(tree2.nodes.add().leaf, 1, 1.2)
_append_to_leaf(tree2.nodes.add().leaf, 0, -0.9)
tree_ensemble_handle = model_ops.tree_ensemble_variable(
stamp_token=7,
tree_ensemble_config=tree_ensemble_config.
SerializeToString(),
name="saver_tree")
stamp_token, serialized_config = model_ops.tree_ensemble_serialize(
tree_ensemble_handle)
resources.initialize_resources(
resources.shared_resources()).run()
self.assertEqual(stamp_token.eval(), 7)
serialized_config = serialized_config.eval()
with ops.Graph().as_default() as graph:
with self.test_session(graph):
tree_ensemble_handle2 = model_ops.tree_ensemble_variable(
stamp_token=9,
tree_ensemble_config=serialized_config,
name="saver_tree2")
resources.initialize_resources(
resources.shared_resources()).run()
# Prepare learner config.
learner_config = learner_pb2.LearnerConfig()
learner_config.num_classes = 3
result, _, _ = prediction_ops.gradient_trees_prediction(
tree_ensemble_handle2,
self._seed, [self._dense_float_tensor],
[self._sparse_float_indices1, self._sparse_float_indices2],
[self._sparse_float_values1, self._sparse_float_values2],
[self._sparse_float_shape1, self._sparse_float_shape2],
[self._sparse_int_indices1], [self._sparse_int_values1],
[self._sparse_int_shape1],
learner_config=learner_config.SerializeToString(),
apply_dropout=False,
apply_averaging=False,
center_bias=False,
reduce_dim=True)
# Re-serialize tree.
stamp_token2, serialized_config2 = model_ops.tree_ensemble_serialize(
tree_ensemble_handle2)
# The first example will get bias class 1 -0.2 from first tree and
# leaf 2 payload (sparse feature missing) of 0.5 hence [0.5, -0.2],
# the second example will get the same bias class 1 -0.2 and leaf 3
# payload of class 1 1.2 hence [0.0, 1.0].
self.assertEqual(stamp_token2.eval(), 9)
# Class 2 does have scores in the leaf => it gets score 0.
self.assertEqual(serialized_config2.eval(), serialized_config)
self.assertAllClose(result.eval(), [[0.5, -0.2], [0, 1.0]])
def testCachedPredictionFromThePreviousTreeWithPostPrunedNodes(self):
"""Tests that prediction based on previous node in the tree works."""
with self.test_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
bucketized_split {
feature_id:0
threshold: 33
left_id: 1
right_id: 2
}
metadata {
gain: -0.2
}
}
nodes {
leaf {
scalar: 0.01
}
}
nodes {
bucketized_split {
feature_id: 1
threshold: 5
left_id: 3
right_id: 4
}
metadata {
gain: 0.5
original_leaf {
scalar: 0.0143
}
}
}
nodes {
leaf {
scalar: 0.0553
}
}
nodes {
leaf {
scalar: 0.0783
}
}
}
trees {
nodes {
leaf {
scalar: 0.55
}
}
}
tree_weights: 1.0
tree_weights: 1.0
tree_metadata {
num_layers_grown: 3
is_finalized: true
post_pruned_nodes_meta {
new_node_id: 0
logit_change: 0.0
}
post_pruned_nodes_meta {
new_node_id: 1
logit_change: 0.0
}
post_pruned_nodes_meta {
new_node_id: 2
logit_change: 0.0
}
post_pruned_nodes_meta {
new_node_id: 1
logit_change: -0.07
}
post_pruned_nodes_meta {
new_node_id: 1
logit_change: -0.083
}
post_pruned_nodes_meta {
new_node_id: 3
logit_change: 0.0
}
post_pruned_nodes_meta {
new_node_id: 4
logit_change: 0.0
}
post_pruned_nodes_meta {
new_node_id: 1
logit_change: -0.22
}
post_pruned_nodes_meta {
new_node_id: 1
logit_change: -0.57
}
}
tree_metadata {
num_layers_grown: 1
is_finalized: false
}
growing_metadata {
num_trees_attempted: 2
num_layers_attempted: 4
}
""", tree_ensemble_config)
# Create existing ensemble.
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
cached_tree_ids = [0, 0, 0, 0, 0, 0]
# Leaves 3,4, 7 and 8 got deleted during post-pruning, leaves 5 and 6
# changed the ids to 3 and 4 respectively.
cached_node_ids = [3, 4, 5, 6, 7, 8]
# We have two features: 0 and 1.
feature_0_values = [12, 17, 35, 36, 23, 11]
feature_1_values = [12, 12, 17, 18, 123, 24]
# Grow tree ensemble.
predict_op = boosted_trees_ops.training_predict(
tree_ensemble_handle,
cached_tree_ids=cached_tree_ids,
cached_node_ids=cached_node_ids,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits_updates, new_tree_ids, new_node_ids = session.run(
predict_op)
# We are in the last tree.
self.assertAllClose([1, 1, 1, 1, 1, 1], new_tree_ids)
# Examples from leaves 3,4,7,8 should be in leaf 1, examples from leaf 5
# and 6 in leaf 3 and 4 in tree 0. For tree 1, all of the examples are in
# the root node.
self.assertAllClose([0, 0, 0, 0, 0, 0], new_node_ids)
cached_values = [[0.08], [0.093], [0.0553], [0.0783],
[0.15 + 0.08], [0.5 + 0.08]]
root = 0.55
self.assertAllClose(
[[root + 0.01], [root + 0.01], [root + 0.0553],
[root + 0.0783], [root + 0.01], [root + 0.01]],
logits_updates + cached_values)
def testGenerateFeatureSplitCandidatesMulticlass(self):
with self.test_session() as sess:
# Batch size is 4, 2 gradients per each instance.
gradients = array_ops.constant(
[[0.2, 0.1], [-0.5, 0.2], [1.2, 3.4], [4.0, -3.5]],
shape=[4, 2])
# 2x2 matrix for each instance
hessian_0 = [[0.12, 0.02], [0.3, 0.11]]
hessian_1 = [[0.07, -0.2], [-0.5, 0.2]]
hessian_2 = [[0.2, -0.23], [-0.8, 0.9]]
hessian_3 = [[0.13, -0.3], [-1.5, 2.2]]
hessians = array_ops.constant(
[hessian_0, hessian_1, hessian_2, hessian_3])
partition_ids = [0, 0, 0, 1]
indices = [[0, 0], [0, 1], [2, 0], [3, 0]]
values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64)
hessians = array_ops.constant(
[hessian_0, hessian_1, hessian_2, hessian_3])
partition_ids = array_ops.constant([0, 0, 0, 1],
dtype=dtypes.int32)
gradient_shape = tensor_shape.TensorShape([2])
hessian_shape = tensor_shape.TensorShape([2, 2])
class_id = -1
split_handler = categorical_split_handler.EqualitySplitHandler(
l1_regularization=0.1,
l2_regularization=1,
tree_complexity_regularization=0,
min_node_weight=0,
sparse_int_column=sparse_tensor.SparseTensor(
indices, values, [4, 1]),
feature_column_group_id=0,
gradient_shape=gradient_shape,
hessian_shape=hessian_shape,
multiclass_strategy=learner_pb2.LearnerConfig.FULL_HESSIAN,
init_stamp_token=0)
resources.initialize_resources(resources.shared_resources()).run()
empty_gradients, empty_hessians = get_empty_tensors(
gradient_shape, hessian_shape)
example_weights = array_ops.ones([4, 1], dtypes.float32)
update_1 = split_handler.update_stats_sync(
0,
partition_ids,
gradients,
hessians,
empty_gradients,
empty_hessians,
example_weights,
is_active=array_ops.constant([True, True]))
with ops.control_dependencies([update_1]):
are_splits_ready, partitions, gains, splits = (
split_handler.make_splits(0, 1, class_id))
are_splits_ready, partitions, gains, splits = (sess.run(
[are_splits_ready, partitions, gains, splits]))
self.assertTrue(are_splits_ready)
self.assertAllEqual([0, 1], partitions)
split_info = split_info_pb2.SplitInfo()
split_info.ParseFromString(splits[0])
left_child = split_info.left_child.vector
right_child = split_info.right_child.vector
split_node = split_info.split_node.categorical_id_binary_split
# Each leaf has 2 element vector.
self.assertEqual(2, len(left_child.value))
self.assertEqual(2, len(right_child.value))
self.assertEqual(1, split_node.feature_id)
split_info.ParseFromString(splits[1])
left_child = split_info.left_child.vector
right_child = split_info.right_child.vector
split_node = split_info.split_node.categorical_id_binary_split
self.assertEqual(2, len(left_child.value))
self.assertEqual(0, len(right_child.value))
self.assertEqual(1, split_node.feature_id)
def testContribsMultipleTreeWhenFirstTreeIsABiasNode(self):
"""Tests case when, after training, first tree contains only a bias node."""
with self.test_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
leaf {
scalar: 1.72
}
}
}
trees {
nodes {
bucketized_split {
feature_id: 2
threshold: 26
left_id: 1
right_id: 2
}
}
nodes {
bucketized_split {
feature_id: 0
threshold: 50
left_id: 3
right_id: 4
}
metadata {
original_leaf: {scalar: 5.5}
}
}
nodes {
leaf {
scalar: 7.0
}
}
nodes {
leaf {
scalar: 5.0
}
}
nodes {
leaf {
scalar: 6.0
}
}
}
tree_weights: 1.
tree_weights: 0.1
tree_metadata: {
num_layers_grown: 0
}
tree_metadata: {
num_layers_grown: 1
}
""", tree_ensemble_config)
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
feature_0_values = [36, 32]
feature_1_values = [13, -29] # Unused feature.
feature_2_values = [11, 27]
# Expected logits are computed by traversing the logit path and
# subtracting child logits from parent logits.
expected_feature_ids = ((2, 0), (2, ))
# bias = 1.72 * 1. # Root node of tree_0.
# example_0 : (bias, 0.1 * 5.5 + bias, 0.1 * 5. + bias)
# example_1 : (bias, 0.1 * 7. + bias )
expected_logits_paths = ((1.72, 2.27, 2.22), (1.72, 2.42))
bucketized_features = [
feature_0_values, feature_1_values, feature_2_values
]
debug_op = boosted_trees_ops.example_debug_outputs(
tree_ensemble_handle,
bucketized_features=bucketized_features,
logits_dimension=1)
serialized_examples_debug_outputs = session.run(debug_op)
feature_ids = []
logits_paths = []
for example in serialized_examples_debug_outputs:
example_debug_outputs = boosted_trees_pb2.DebugOutput()
example_debug_outputs.ParseFromString(example)
feature_ids.append(example_debug_outputs.feature_ids)
logits_paths.append(example_debug_outputs.logits_path)
self.assertAllClose(feature_ids, expected_feature_ids)
self.assertAllClose(logits_paths, expected_logits_paths)
def testCachedPredictionIsCurrent(self):
"""Tests that prediction based on previous node in the tree works."""
with self.test_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
bucketized_split {
feature_id: 1
threshold: 15
left_id: 1
right_id: 2
}
metadata {
gain: 7.62
original_leaf {
scalar: -2
}
}
}
nodes {
leaf {
scalar: 1.14
}
}
nodes {
leaf {
scalar: 8.79
}
}
}
tree_weights: 0.1
tree_metadata {
is_finalized: true
num_layers_grown: 2
}
growing_metadata {
num_trees_attempted: 1
num_layers_attempted: 2
}
""", tree_ensemble_config)
# Create existing ensemble with one root split
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
# Two examples, one was cached in node 1 first, another in node 0.
cached_tree_ids = [0, 0]
cached_node_ids = [1, 2]
# We have two features: 0 and 1. Values don't matter because trees didn't
# change.
feature_0_values = [67, 5]
feature_1_values = [9, 17]
# Grow tree ensemble.
predict_op = boosted_trees_ops.training_predict(
tree_ensemble_handle,
cached_tree_ids=cached_tree_ids,
cached_node_ids=cached_node_ids,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits_updates, new_tree_ids, new_node_ids = session.run(
predict_op)
# Nothing changed.
self.assertAllClose(cached_tree_ids, new_tree_ids)
self.assertAllClose(cached_node_ids, new_node_ids)
self.assertAllClose([[0], [0]], logits_updates)
def testCachedPredictionFromTheSameTree(self):
"""Tests that prediction based on previous node in the tree works."""
with self.test_session() as session:
tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
bucketized_split {
feature_id: 1
threshold: 15
left_id: 1
right_id: 2
}
metadata {
gain: 7.62
original_leaf {
scalar: -2
}
}
}
nodes {
bucketized_split {
feature_id: 1
threshold: 7
left_id: 3
right_id: 4
}
metadata {
gain: 1.4
original_leaf {
scalar: 7.14
}
}
}
nodes {
bucketized_split {
feature_id: 0
threshold: 7
left_id: 5
right_id: 6
}
metadata {
gain: 2.7
original_leaf {
scalar: -4.375
}
}
}
nodes {
leaf {
scalar: 1.14
}
}
nodes {
leaf {
scalar: 8.79
}
}
nodes {
leaf {
scalar: -5.875
}
}
nodes {
leaf {
scalar: -2.075
}
}
}
tree_weights: 0.1
tree_metadata {
is_finalized: true
num_layers_grown: 2
}
growing_metadata {
num_trees_attempted: 1
num_layers_attempted: 2
}
""", tree_ensemble_config)
# Create existing ensemble with one root split
tree_ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
serialized_proto=tree_ensemble_config.SerializeToString())
tree_ensemble_handle = tree_ensemble.resource_handle
resources.initialize_resources(resources.shared_resources()).run()
# Two examples, one was cached in node 1 first, another in node 0.
cached_tree_ids = [0, 0]
cached_node_ids = [1, 0]
# We have two features: 0 and 1.
feature_0_values = [67, 5]
feature_1_values = [9, 17]
# Grow tree ensemble.
predict_op = boosted_trees_ops.training_predict(
tree_ensemble_handle,
cached_tree_ids=cached_tree_ids,
cached_node_ids=cached_node_ids,
bucketized_features=[feature_0_values, feature_1_values],
logits_dimension=1)
logits_updates, new_tree_ids, new_node_ids = session.run(
predict_op)
# We are still in the same tree.
self.assertAllClose([0, 0], new_tree_ids)
# When using the full tree, the first example will end up in node 4,
# the second in node 5.
self.assertAllClose([4, 5], new_node_ids)
# Full predictions for each instance would be 8.79 and -5.875,
# so an update from the previous cached values lr*(7.14 and -2) would be
# 1.65 and -3.875, and then multiply them by 0.1 (lr)
self.assertAllClose([[0.1 * 1.65], [0.1 * -3.875]], logits_updates)
def testSerializeDeserialize(self):
with self.cached_session():
# Initialize.
ensemble = boosted_trees_ops.TreeEnsemble('ensemble',
stamp_token=5)
resources.initialize_resources(resources.shared_resources()).run()
(stamp_token, num_trees, num_finalized_trees, num_attempted_layers,
nodes_range) = ensemble.get_states()
self.assertEqual(5, self.evaluate(stamp_token))
self.assertEqual(0, self.evaluate(num_trees))
self.assertEqual(0, self.evaluate(num_finalized_trees))
self.assertEqual(0, self.evaluate(num_attempted_layers))
self.assertAllEqual([0, 1], self.evaluate(nodes_range))
# Deserialize.
ensemble_proto = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
bucketized_split {
feature_id: 75
threshold: 21
left_id: 1
right_id: 2
}
metadata {
gain: -1.4
}
}
nodes {
leaf {
scalar: -0.6
}
}
nodes {
leaf {
scalar: 0.165
}
}
}
tree_weights: 0.5
tree_metadata {
num_layers_grown: 4 # it's fake intentionally.
is_finalized: false
}
growing_metadata {
num_trees_attempted: 1
num_layers_attempted: 5
last_layer_node_start: 3
last_layer_node_end: 7
}
""", ensemble_proto)
with ops.control_dependencies([
ensemble.deserialize(
stamp_token=3,
serialized_proto=ensemble_proto.SerializeToString())
]):
(stamp_token, num_trees, num_finalized_trees,
num_attempted_layers, nodes_range) = ensemble.get_states()
self.assertEqual(3, self.evaluate(stamp_token))
self.assertEqual(1, self.evaluate(num_trees))
# This reads from metadata, not really counting the layers.
self.assertEqual(5, self.evaluate(num_attempted_layers))
self.assertEqual(0, self.evaluate(num_finalized_trees))
self.assertAllEqual([3, 7], self.evaluate(nodes_range))
# Serialize.
new_ensemble_proto = boosted_trees_pb2.TreeEnsemble()
new_stamp_token, new_serialized = ensemble.serialize()
self.assertEqual(3, self.evaluate(new_stamp_token))
new_ensemble_proto.ParseFromString(new_serialized.eval())
self.assertProtoEquals(ensemble_proto, new_ensemble_proto)
def main(args):
vocab = build_vocab(args.data_path)
data = pd.DataFrame({
'label': vocab.labels,
'lprox': vocab.lprox,
'rprox': vocab.rprox,
'x': vocab.x,
'y': vocab.y,
'z': vocab.z,
})
y = data['label']
lprox = pd.DataFrame(data['lprox'].values.tolist())
rprox = pd.DataFrame(data['rprox'].values.tolist())
xax = pd.DataFrame(data['x'].values.tolist())
yax = pd.DataFrame(data['y'].values.tolist())
zax = pd.DataFrame(data['z'].values.tolist())
X = pd.concat([lprox, rprox, xax, yax, zax], axis=1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
num_steps = 100 # Total steps to train
num_classes = 2
num_features = 585
num_trees = 10
max_nodes = 1000
X = tf.placeholder(tf.float32, shape=[None, num_features])
Y = tf.placeholder(tf.int64, shape=[None])
hparams = tensor_forest.ForestHParams(num_classes=num_classes,
num_features=num_features,
num_trees=num_trees,
max_nodes=max_nodes).fill()
forest_graph = tensor_forest.RandomForestGraphs(hparams)
train_op = forest_graph.training_graph(X, Y)
loss_op = forest_graph.training_loss(X, Y)
infer_op, _, _ = forest_graph.inference_graph(X)
correct_prediction = tf.equal(tf.argmax(infer_op, 1), tf.cast(Y, tf.int64))
accuracy_op = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
init_vars = tf.group(
tf.global_variables_initializer(),
resources.initialize_resources(resources.shared_resources()))
sess = tf.Session()
sess.run(init_vars)
for i in range(1, num_steps + 1):
saver = tf.train.Saver()
_, l = sess.run([train_op, loss_op],
feed_dict={
X: X_train,
Y: y_train
})
if i % 50 == 0 or i == 1:
acc = sess.run(accuracy_op, feed_dict={X: X_train, Y: y_train})
save_path = saver.save(sess, 'models/model%i.ckpt' % (i))
print('Step %i, Loss: %f, Acc: %f' % (i, l, acc))
print("Test Accuracy:",
sess.run(accuracy_op, feed_dict={
X: X_test,
Y: y_test
}))
def testCreateWithProto(self):
with self.cached_session():
ensemble_proto = boosted_trees_pb2.TreeEnsemble()
text_format.Merge(
"""
trees {
nodes {
bucketized_split {
feature_id: 4
left_id: 1
right_id: 2
}
metadata {
gain: 7.62
}
}
nodes {
bucketized_split {
threshold: 21
left_id: 3
right_id: 4
}
metadata {
gain: 1.4
original_leaf {
scalar: 7.14
}
}
}
nodes {
bucketized_split {
feature_id: 1
threshold: 7
left_id: 5
right_id: 6
}
metadata {
gain: 2.7
original_leaf {
scalar: -4.375
}
}
}
nodes {
leaf {
scalar: 6.54
}
}
nodes {
leaf {
scalar: 7.305
}
}
nodes {
leaf {
scalar: -4.525
}
}
nodes {
leaf {
scalar: -4.145
}
}
}
trees {
nodes {
bucketized_split {
feature_id: 75
threshold: 21
left_id: 1
right_id: 2
}
metadata {
gain: -1.4
}
}
nodes {
leaf {
scalar: -0.6
}
}
nodes {
leaf {
scalar: 0.165
}
}
}
tree_weights: 0.15
tree_weights: 1.0
tree_metadata {
num_layers_grown: 2
is_finalized: true
}
tree_metadata {
num_layers_grown: 1
is_finalized: false
}
growing_metadata {
num_trees_attempted: 2
num_layers_attempted: 6
last_layer_node_start: 16
last_layer_node_end: 19
}
""", ensemble_proto)
ensemble = boosted_trees_ops.TreeEnsemble(
'ensemble',
stamp_token=7,
serialized_proto=ensemble_proto.SerializeToString())
resources.initialize_resources(resources.shared_resources()).run()
(stamp_token, num_trees, num_finalized_trees, num_attempted_layers,
nodes_range) = ensemble.get_states()
self.assertEqual(7, self.evaluate(stamp_token))
self.assertEqual(2, self.evaluate(num_trees))
self.assertEqual(1, self.evaluate(num_finalized_trees))
self.assertEqual(6, self.evaluate(num_attempted_layers))
self.assertAllEqual([16, 19], self.evaluate(nodes_range))
def testGenerateFeatureSplitCandidates(self):
with self.test_session() as sess:
# The data looks like the following:
# Example | Gradients | Partition | Feature ID |
# i0 | (0.2, 0.12) | 0 | 1,2 |
# i1 | (-0.5, 0.07) | 0 | |
# i2 | (1.2, 0.2) | 0 | 2 |
# i3 | (4.0, 0.13) | 1 | 1 |
gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0])
hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13])
partition_ids = [0, 0, 0, 1]
indices = [[0, 0], [0, 1], [2, 0], [3, 0]]
values = array_ops.constant([1, 2, 2, 1], dtype=dtypes.int64)
gradient_shape = tensor_shape.scalar()
hessian_shape = tensor_shape.scalar()
class_id = -1
split_handler = categorical_split_handler.EqualitySplitHandler(
l1_regularization=0.1,
l2_regularization=1,
tree_complexity_regularization=0,
min_node_weight=0,
sparse_int_column=sparse_tensor.SparseTensor(
indices, values, [4, 1]),
feature_column_group_id=0,
gradient_shape=gradient_shape,
hessian_shape=hessian_shape,
multiclass_strategy=learner_pb2.LearnerConfig.TREE_PER_CLASS,
init_stamp_token=0)
resources.initialize_resources(resources.shared_resources()).run()
empty_gradients, empty_hessians = get_empty_tensors(
gradient_shape, hessian_shape)
example_weights = array_ops.ones([4, 1], dtypes.float32)
update_1 = split_handler.update_stats_sync(
0,
partition_ids,
gradients,
hessians,
empty_gradients,
empty_hessians,
example_weights,
is_active=array_ops.constant([True, True]))
update_2 = split_handler.update_stats_sync(
0,
partition_ids,
gradients,
hessians,
empty_gradients,
empty_hessians,
example_weights,
is_active=array_ops.constant([True, True]))
with ops.control_dependencies([update_1, update_2]):
are_splits_ready, partitions, gains, splits = (
split_handler.make_splits(0, 1, class_id))
are_splits_ready, partitions, gains, splits = (sess.run(
[are_splits_ready, partitions, gains, splits]))
self.assertTrue(are_splits_ready)
self.assertAllEqual([0, 1], partitions)
# Check the split on partition 0.
# -(0.2 + 1.2 - 0.1) / (0.12 + 0.2 + 1)
expected_left_weight = -0.9848484848484846
# (0.2 + 1.2 - 0.1) ** 2 / (0.12 + 0.2 + 1)
expected_left_gain = 1.2803030303030298
# -(-0.5 + 0.1) / (0.07 + 1)
expected_right_weight = 0.37383177570093457
# (-0.5 + 0.1) ** 2 / (0.07 + 1)
expected_right_gain = 0.14953271028037385
# (0.2 + -0.5 + 1.2 - 0.1) ** 2 / (0.12 + 0.07 + 0.2 + 1)
expected_bias_gain = 0.46043165467625885
split_info = split_info_pb2.SplitInfo()
split_info.ParseFromString(splits[0])
left_child = split_info.left_child.vector
right_child = split_info.right_child.vector
split_node = split_info.split_node.categorical_id_binary_split
self.assertEqual(0, split_node.feature_column)
self.assertEqual(2, split_node.feature_id)
self.assertAllClose(
expected_left_gain + expected_right_gain - expected_bias_gain,
gains[0], 0.00001)
self.assertAllClose([expected_left_weight], left_child.value, 0.00001)
self.assertAllClose([expected_right_weight], right_child.value,
0.00001)
# Check the split on partition 1.
# (-4 + 0.1) / (0.13 + 1)
expected_left_weight = -3.4513274336283186
# (-4 + 0.1) ** 2 / (0.13 + 1)
expected_left_gain = 13.460176991150442
expected_right_weight = 0
expected_right_gain = 0
# (-4 + 0.1) ** 2 / (0.13 + 1)
expected_bias_gain = 13.460176991150442
# Verify candidate for partition 1, there's only one active feature here
# so zero gain is expected.
split_info = split_info_pb2.SplitInfo()
split_info.ParseFromString(splits[1])
left_child = split_info.left_child.vector
right_child = split_info.right_child.vector
split_node = split_info.split_node.categorical_id_binary_split
self.assertAllClose(0.0, gains[1], 0.00001)
self.assertAllClose([expected_left_weight], left_child.value, 0.00001)
self.assertAllClose([expected_right_weight], right_child.value,
0.00001)
self.assertEqual(0, split_node.feature_column)
self.assertEqual(1, split_node.feature_id)
