def test_complex_nested_query(self):
with self.cached_session() as sess:
query_ab = gaussian_query.GaussianSumQuery(l2_norm_clip=1.0,
stddev=0.0)
query_c = normalized_query.NormalizedQuery(
gaussian_query.GaussianSumQuery(l2_norm_clip=10.0, stddev=0.0),
denominator=2.0)
query_d = gaussian_query.GaussianSumQuery(l2_norm_clip=10.0,
stddev=0.0)
query = nested_query.NestedSumQuery(
[query_ab, {
'c': query_c,
'd': [query_d]
}])
record1 = [{
'a': 0.0,
'b': 2.71828
}, {
'c': (-4.0, 6.0),
'd': [-4.0]
}]
record2 = [{
'a': 3.14159,
'b': 0.0
}, {
'c': (6.0, -4.0),
'd': [5.0]
}]
query_result, _ = test_utils.run_query(query, [record1, record2])
result = sess.run(query_result)
expected = [{'a': 1.0, 'b': 1.0}, {'c': (1.0, 1.0), 'd': [1.0]}]
self.assertAllClose(result, expected)
def test_nested_query_with_noise(self):
with self.cached_session() as sess:
stddev = 2.71828
denominator = 3.14159
query1 = gaussian_query.GaussianSumQuery(l2_norm_clip=1.5,
stddev=stddev)
query2 = normalized_query.NormalizedQuery(
gaussian_query.GaussianSumQuery(l2_norm_clip=0.5,
stddev=stddev),
denominator=denominator)
query = nested_query.NestedSumQuery((query1, query2))
record1 = (3.0, [2.0, 1.5])
record2 = (0.0, [-1.0, -3.5])
query_result, _ = test_utils.run_query(query, [record1, record2])
noised_averages = []
for _ in range(1000):
noised_averages.append(tf.nest.flatten(sess.run(query_result)))
result_stddev = np.std(noised_averages, 0)
avg_stddev = stddev / denominator
expected_stddev = [stddev, avg_stddev, avg_stddev]
self.assertArrayNear(result_stddev, expected_stddev, 0.1)
def test_nested_gaussian_average_with_clip_no_noise(self):
with self.cached_session() as sess:
query1 = normalized_query.NormalizedQuery(
gaussian_query.GaussianSumQuery(l2_norm_clip=4.0, stddev=0.0),
denominator=5.0)
query2 = normalized_query.NormalizedQuery(
gaussian_query.GaussianSumQuery(l2_norm_clip=5.0, stddev=0.0),
denominator=5.0)
query = nested_query.NestedSumQuery([query1, query2])
record1 = [1.0, [12.0, 9.0]] # Clipped to [1.0, [4.0, 3.0]]
record2 = [5.0, [1.0, 2.0]] # Clipped to [4.0, [1.0, 2.0]]
query_result, _ = test_utils.run_query(query, [record1, record2])
result = sess.run(query_result)
expected = [1.0, [1.0, 1.0]]
self.assertAllClose(result, expected)
def _construct_below_estimate_query(self, below_estimate_stddev,
expected_num_records):
# See comments in `QuantileEstimatorQuery._construct_below_estimate_query`
# for why clip norm 0.5 is used for the query.
sum_query = tree_aggregation_query.TreeResidualSumQuery.build_l2_gaussian_query(
clip_norm=0.5,
noise_multiplier=2 * below_estimate_stddev,
record_specs=tf.TensorSpec([]))
return normalized_query.NormalizedQuery(
sum_query, denominator=expected_num_records)
def test_normalization(self):
with self.cached_session() as sess:
record1 = tf.constant([-6.0, 8.0]) # Clipped to [-3.0, 4.0].
record2 = tf.constant([4.0, -3.0]) # Not clipped.
sum_query = gaussian_query.GaussianSumQuery(l2_norm_clip=5.0,
stddev=0.0)
query = normalized_query.NormalizedQuery(numerator_query=sum_query,
denominator=2.0)
query_result, _ = test_utils.run_query(query, [record1, record2])
result = sess.run(query_result)
expected = [0.5, 0.5]
self.assertAllClose(result, expected)
def _construct_below_estimate_query(self, below_estimate_stddev,
expected_num_records):
# A DPQuery used to estimate the fraction of records that are less than the
# current quantile estimate. It accumulates an indicator 0/1 of whether each
# record is below the estimate, and normalizes by the expected number of
# records. In practice, we accumulate counts shifted by -0.5 so they are
# centered at zero. This makes the sensitivity of the below_estimate count
# query 0.5 instead of 1.0, since the maximum that a single record could
# affect the count is 0.5. Note that although the l2_norm_clip of the
# below_estimate query is 0.5, no clipping will ever actually occur
# because the value of each record is always +/-0.5.
return normalized_query.NormalizedQuery(
gaussian_query.GaussianSumQuery(l2_norm_clip=0.5,
stddev=below_estimate_stddev),
denominator=expected_num_records)