def testSaveRestoreBeforeFlush(self):
save_dir = os.path.join(self.get_temp_dir(), "save_restore")
save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
with self.test_session(graph=ops.Graph()) as sess:
accumulator = quantile_ops.QuantileAccumulator(init_stamp_token=0,
num_quantiles=3,
epsilon=0.33,
name="q0")
save = saver.Saver()
resources.initialize_resources(resources.shared_resources()).run()
sparse_indices_0 = constant_op.constant(
[[1, 0], [2, 1], [3, 0], [4, 2], [5, 0]], dtype=dtypes.int64)
sparse_values_0 = constant_op.constant([2.0, 3.0, 4.0, 5.0, 6.0],
dtype=dtypes.float32)
sparse_shape_0 = constant_op.constant([6, 3], dtype=dtypes.int64)
example_weights = constant_op.constant([10, 1, 1, 1, 1, 1],
dtype=dtypes.float32,
shape=[6, 1])
update = accumulator.add_summary(stamp_token=0,
column=sparse_tensor.SparseTensor(
sparse_indices_0,
sparse_values_0,
sparse_shape_0),
example_weights=example_weights)
update.run()
save.save(sess, save_path)
reset = accumulator.flush(stamp_token=0, next_stamp_token=1)
with ops.control_dependencies([reset]):
are_ready_flush, buckets = (accumulator.get_buckets(
stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([2, 4, 6.], buckets)
with self.test_session(graph=ops.Graph()) as sess:
accumulator = quantile_ops.QuantileAccumulator(init_stamp_token=0,
num_quantiles=3,
epsilon=0.33,
name="q0")
save = saver.Saver()
# Restore the saved values in the parameter nodes.
save.restore(sess, save_path)
are_ready_noflush = accumulator.get_buckets(stamp_token=0)[0]
with ops.control_dependencies([are_ready_noflush]):
reset = accumulator.flush(stamp_token=0, next_stamp_token=1)
with ops.control_dependencies([reset]):
are_ready_flush, buckets = accumulator.get_buckets(
stamp_token=1)
buckets, are_ready_flush, are_ready_noflush = (sess.run(
[buckets, are_ready_flush, are_ready_noflush]))
self.assertFalse(are_ready_noflush)
self.assertTrue(are_ready_flush)
self.assertAllEqual([2, 4, 6.], buckets)
def testStreamingQuantileBuckets(self):
"""Sets up the quantile summary op test as follows.
100 batches of data is added to the accumulator. The batches are in form:
[0 1 .. 99]
[100 101 .. 200]
...
[9900 9901 .. 9999]
All the batches have 1 for all the example weights.
"""
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=3, epsilon=0.01, name="q1")
resources.initialize_resources(resources.shared_resources()).run()
weight_placeholder = array_ops.placeholder(dtypes.float32)
dense_placeholder = array_ops.placeholder(dtypes.float32)
update = accumulator.add_summary(
stamp_token=0,
column=dense_placeholder,
example_weights=weight_placeholder)
with self.test_session() as sess:
for i in range(100):
dense_float = np.linspace(
i * 100, (i + 1) * 100 - 1, num=100).reshape(-1, 1)
sess.run(update, {
dense_placeholder: dense_float,
weight_placeholder: np.ones(shape=(100, 1), dtype=np.float32)
})
with self.test_session() as sess:
sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1))
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([0, 3335., 6671., 9999.], buckets)
def _testStreamingQuantileBucketsHelper(
self, inputs, num_quantiles=3, expected_buckets=None):
"""Helper to test quantile buckets on different inputs."""
# set generate_quantiles to True since the test will generate fewer
# boundaries otherwise.
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0, num_quantiles=num_quantiles,
epsilon=0.001, name="q1", generate_quantiles=True)
resources.initialize_resources(resources.shared_resources()).run()
input_column = array_ops.placeholder(dtypes.float32)
weights = array_ops.placeholder(dtypes.float32)
update = accumulator.add_summary(
stamp_token=0,
column=input_column,
example_weights=weights)
with self.test_session() as sess:
sess.run(update,
{input_column: inputs,
weights: [1] * len(inputs)})
with self.test_session() as sess:
sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1))
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run(
[buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
# By default, use 3 quantiles, 4 boundaries for simplicity.
self.assertEqual(num_quantiles + 1, len(buckets))
if expected_buckets:
self.assertAllEqual(buckets, expected_buckets)
def __init__(self, num_quantiles, epsilon, serialized_tf_config=None):
self._num_quantiles = num_quantiles
self._epsilon = epsilon
self._serialized_tf_config = serialized_tf_config
# _stamp_token is used to commit the state of the qaccumulator. In
# this case, the qaccumulator state is completely returned and stored
# as part of quantile_state/summary in the combiner fn (i.e the summary is
# extracted and stored outside the qaccumulator). So we don't use
# the timestamp mechanism to signify progress in the qaccumulator state.
self._stamp_token = 0
# Represents an empty summary. This could be changed to a tf.constant
# implemented by the quantile ops library.
self._empty_summary = None
# Create a new session with a new graph for quantile ops.
self._session = tf.Session(
graph=tf.Graph(),
config=_maybe_deserialize_tf_config(serialized_tf_config))
with self._session.graph.as_default():
with self._session.as_default():
self._qaccumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=self._stamp_token,
num_quantiles=self._num_quantiles,
epsilon=self._epsilon,
name='qaccumulator')
resources.initialize_resources(
resources.shared_resources()).run()
def initialize_local_state(self, tf_config=None):
"""Called by the CombineFnWrapper's __init__ method.
This can be used to set non-pickleable local state. It is used in
conjunction with overriding __reduce__ so this state is not pickled. This
method must be called prior to any other method.
Args:
tf_config: (optional) A tf.ConfigProto
"""
# _stamp_token is used to commit the state of the qaccumulator. In
# this case, the qaccumulator state is completely returned and stored
# as part of quantile_state/summary in the combiner fn (i.e the summary is
# extracted and stored outside the qaccumulator). So we don't use
# the timestamp mechanism to signify progress in the qaccumulator state.
self._stamp_token = 0
# Represents an empty summary. This could be changed to a tf.constant
# implemented by the quantile ops library.
self._empty_summary = None
# Create a new session with a new graph for quantile ops.
self._session = tf.Session(graph=tf.Graph(), config=tf_config)
with self._session.graph.as_default():
with self._session.as_default():
self._qaccumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=self._stamp_token,
num_quantiles=self._num_quantiles,
epsilon=self._epsilon,
name='qaccumulator')
resources.initialize_resources(
resources.shared_resources()).run()
def __init__(self,
l1_regularization,
l2_regularization,
tree_complexity_regularization,
min_node_weight,
feature_column_group_id,
epsilon,
num_quantiles,
gradient_shape,
hessian_shape,
multiclass_strategy,
init_stamp_token=0,
loss_uses_sum_reduction=False,
name=None):
"""Initialize the internal state for this split handler.
Args:
l1_regularization: L1 regularization applied for this split handler.
l2_regularization: L2 regularization applied for this split handler.
tree_complexity_regularization: Tree complexity regularization applied
for this split handler.
min_node_weight: Minimum sum of weights of examples in each partition to
be considered for splitting.
feature_column_group_id: Feature column group index.
epsilon: A float, the error bound for quantile computation.
num_quantiles: An int, the number of buckets to create from the histogram.
gradient_shape: A TensorShape, containing shape of gradients.
hessian_shape: A TensorShape, containing shape of hessians.
multiclass_strategy: Strategy describing how to treat multiclass problems.
init_stamp_token: A tensor containing an scalar for initial stamp of the
stamped objects.
loss_uses_sum_reduction: A scalar boolean tensor that specifies whether
SUM or MEAN reduction was used for the loss.
name: An optional handler name.
"""
super(InequalitySplitHandler, self).__init__(
name=name,
l1_regularization=l1_regularization,
l2_regularization=l2_regularization,
tree_complexity_regularization=tree_complexity_regularization,
min_node_weight=min_node_weight,
feature_column_group_id=feature_column_group_id,
gradient_shape=gradient_shape,
hessian_shape=hessian_shape,
multiclass_strategy=multiclass_strategy,
loss_uses_sum_reduction=loss_uses_sum_reduction)
self._stats_accumulator = stats_accumulator_ops.StatsAccumulator(
init_stamp_token,
gradient_shape,
hessian_shape,
name="StatsAccumulator/{}".format(self._name))
# Allocate both stats accumulator and quantile accumulator on the same
# device so that we can build splits with fewer RPCs.
with ops.colocate_with(self._stats_accumulator.resource()):
self._quantile_accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token,
epsilon=epsilon,
num_quantiles=num_quantiles,
name="QuantileAccumulator/{}".format(self._name))
def testSaveRestoreAfterFlush(self):
save_dir = os.path.join(self.get_temp_dir(), "save_restore")
save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
with self.test_session(graph=ops.Graph()) as sess:
accumulator = quantile_ops.QuantileAccumulator(init_stamp_token=0,
num_quantiles=3,
epsilon=0.33,
name="q0")
save = saver.Saver()
resources.initialize_resources(resources.shared_resources()).run()
example_weights = constant_op.constant([10, 1, 1, 1, 1, 1],
dtype=dtypes.float32,
shape=[6, 1])
dense_float_tensor_0 = constant_op.constant([1, 2, 3, 4, 4, 5],
dtype=dtypes.float32,
shape=[6, 1])
update = accumulator.add_summary(stamp_token=0,
column=dense_float_tensor_0,
example_weights=example_weights)
update.run()
reset = accumulator.flush(stamp_token=0, next_stamp_token=1)
with ops.control_dependencies([reset]):
are_ready_flush, buckets = (accumulator.get_buckets(
stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([1, 3, 5], buckets)
save.save(sess, save_path)
with self.test_session(graph=ops.Graph()) as sess:
accumulator = quantile_ops.QuantileAccumulator(init_stamp_token=0,
num_quantiles=3,
epsilon=0.33,
name="q0")
save = saver.Saver()
# Restore the saved values in the parameter nodes.
save.restore(sess, save_path)
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([1, 3, 5], buckets)
def testStreamingQuantileBucketsLowPrecisionInput(self):
"""Tests inputs that simulate low precision float16 values."""
num_quantiles = 3
# set generate_quantiles to True since the test will generate fewer
# boundaries otherwise.
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0,
num_quantiles=num_quantiles,
epsilon=0.001,
name="q1",
generate_quantiles=True)
resources.initialize_resources(resources.shared_resources()).run()
input_column = array_ops.placeholder(dtypes.float32)
weights = array_ops.placeholder(dtypes.float32)
update = accumulator.add_summary(stamp_token=0,
column=input_column,
example_weights=weights)
with self.test_session() as sess:
# This input is generated by integer in the range [2030, 2060]
# but represented by with float16 precision. Integers <= 2048 are
# exactly represented, whereas numbers > 2048 are rounded; and hence
# numbers > 2048 are repeated. For precision loss / rounding, see:
# https://en.wikipedia.org/wiki/Half-precision_floating-point_format.
#
# The intent of the test is not handling of float16 values, but to
# validate the number of buckets is returned, in cases where the input
# may contain repeated values.
inputs = [
2030.0, 2031.0, 2032.0, 2033.0, 2034.0, 2035.0, 2036.0, 2037.0,
2038.0, 2039.0, 2040.0, 2041.0, 2042.0, 2043.0, 2044.0, 2045.0,
2046.0, 2047.0, 2048.0, 2048.0, 2050.0, 2052.0, 2052.0, 2052.0,
2054.0, 2056.0, 2056.0, 2056.0, 2058.0, 2060.0
]
sess.run(update, {
input_column: inputs,
weights: [1] * len(inputs)
})
with self.test_session() as sess:
sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1))
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertEqual(num_quantiles + 1, len(buckets))
self.assertAllEqual([2030, 2040, 2050, 2060], buckets)
def testStreamingQuantileBucketsWithVaryingBatch(self):
"""Sets up the quantile summary op test as follows.
Creates batches examples with different number of inputs in each batch.
The input values are dense in the range [1 ... N]
The data looks like this:
| Batch | Start | InputList
| 1 | 1 | [1]
| 2 | 2 | [2, 3]
| 3 | 4 | [4, 5, 6]
| 4 | 7 | [7, 8, 9, 10]
| 5 | 11 | [11, 12, 13, 14, 15]
| 6 | 16 | [16, 17, 18, 19, 20, 21]
"""
num_quantiles = 3
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=0,
num_quantiles=num_quantiles,
epsilon=0.001,
name="q1")
resources.initialize_resources(resources.shared_resources()).run()
input_column = array_ops.placeholder(dtypes.float32)
weights = array_ops.placeholder(dtypes.float32)
update = accumulator.add_summary(stamp_token=0,
column=input_column,
example_weights=weights)
with self.test_session() as sess:
for i in range(1, 23):
# start = 1, 2, 4, 7, 11, 16 ... (see comment above)
start = int((i * (i - 1) / 2) + 1)
sess.run(update, {
input_column: range(start, start + i),
weights: [1] * i
})
with self.test_session() as sess:
sess.run(accumulator.flush(stamp_token=0, next_stamp_token=1))
are_ready_flush, buckets = (accumulator.get_buckets(stamp_token=1))
buckets, are_ready_flush = (sess.run([buckets, are_ready_flush]))
self.assertEqual(True, are_ready_flush)
self.assertEqual(num_quantiles + 1, len(buckets))
self.assertAllEqual([1, 86., 170., 253.], buckets)
def testStreamingQuantileBuckets(self):
"""Sets up the quantile summary op test as follows.
Create a batch of 6 examples having a dense and sparse features.
The data looks like this
| Instance | instance weights | Dense 0
| 0 | 10 | 1
| 1 | 1 | 2
| 2 | 1 | 3
| 3 | 1 | 4
| 4 | 1 | 4
| 5 | 1 | 5
"""
dense_float_tensor_0 = np.array([1, 2, 3, 4, 4, 5])
example_weights = np.array([10, 1, 1, 1, 1, 1])
with self.test_session() as sess:
accumulator = quantile_ops.QuantileAccumulator(init_stamp_token=0,
num_quantiles=3,
epsilon=0.33,
name="q1")
resources.initialize_resources(resources.shared_resources()).run()
are_ready_noflush, _, = (accumulator.get_buckets(stamp_token=0))
update = accumulator.add_summary(stamp_token=0,
column=dense_float_tensor_0,
example_weights=example_weights)
with ops.control_dependencies([are_ready_noflush, update]):
reset = accumulator.flush(stamp_token=0, next_stamp_token=1)
with ops.control_dependencies([reset]):
are_ready_flush, buckets = (accumulator.get_buckets(
stamp_token=1))
buckets, are_ready_noflush, are_ready_flush = (sess.run(
[buckets, are_ready_noflush, are_ready_flush]))
self.assertEqual(False, are_ready_noflush)
self.assertEqual(True, are_ready_flush)
self.assertAllEqual([1, 3, 5], buckets)
def initialize_local_state(self, tf_config=None):
"""Called by the CombineFnWrapper's __init__ method.
This can be used to set non-pickleable local state. It is used in
conjunction with overriding __reduce__ so this state is not pickled. This
method must be called prior to any other method.
Args:
tf_config: (optional) A tf.ConfigProto
"""
# stamp_token is used to commit the state of the qaccumulator. In
# this case, the qaccumulator state is completely returned and stored
# as part of quantile_state/summary in the combiner fn (i.e the summary is
# extracted and stored outside the qaccumulator). So we don't use
# the timestamp mechanism to signify progress in the qaccumulator state.
stamp_token = 0
# Create a new session with a new graph for quantile ops.
self._session = tf.Session(graph=tf.Graph(), config=tf_config)
with self._session.graph.as_default():
with self._session.as_default():
self._qaccumulator = quantile_ops.QuantileAccumulator(
init_stamp_token=stamp_token,
num_quantiles=self._num_quantiles,
epsilon=self._epsilon,
name='qaccumulator')
resources.initialize_resources(resources.shared_resources()).run()
# Create placeholder that will be used to provide input the
# QuantileAccumulator. Has shape (1, None) as this is what the
# QuantileAccumulator accepts.
self._add_summary_input = tf.placeholder(
dtype=self._bucket_numpy_dtype, shape=[1, None])
# Create op to update the accumulator with new input fed from
# self._qaccumulator_input.
self._add_summary_op = self._qaccumulator.add_summary(
stamp_token=stamp_token,
column=self._add_summary_input,
# All weights are equal, and the weight vector is the
# same length as the input.
example_weights=tf.ones_like(self._add_summary_input))
# Create op to add a prebuilt summary to the accumulator, and a
# placeholder tensor to provide the input for this op.
self._prebuilt_summary_input = tf.placeholder(
dtype=tf.string, shape=[])
self._add_prebuilt_summary_op = self._qaccumulator.add_prebuilt_summary(
stamp_token=stamp_token,
summary=self._prebuilt_summary_input)
# Create op to flush summaries and return a summary representing the
# summaries that were added the accumulator so far.
self._flush_summary_op = self._qaccumulator.flush_summary(
stamp_token=stamp_token,
next_stamp_token=stamp_token)
# Create ops to flush the accumulator and return approximate boundaries.
self._flush_op = self._qaccumulator.flush(
stamp_token=stamp_token,
next_stamp_token=stamp_token)
_, self._buckets_op = self._qaccumulator.get_buckets(
stamp_token=stamp_token)
# We generate an empty summary by calling self._flush_summary_op.
# We cache this as some implementations may call create_accumulator for
# every input, and it can be cached since it will always be the same and
# immutable.
self._empty_summary = self._session.run(self._flush_summary_op)