def testMeanHigherDimension(self, dtype): testee_lkj = tfd.LKJ(dimension=6, concentration=dtype([1., 3., 5.]), validate_args=True) num_samples = 20000 results = testee_lkj.sample(sample_shape=[num_samples], seed=test_util.test_seed()) mean = testee_lkj.mean() self.assertEqual(mean.shape, [3, 6, 6]) # tfd.LKJ has some small numerical issues, so we allow for some amount of # numerical tolerance when testing means. numerical_tolerance = 1e-5 check1 = st.assert_true_mean_in_interval_by_dkwm( samples=results, low=-1., high=1., expected_low=mean - numerical_tolerance, expected_high=mean + numerical_tolerance, false_fail_rate=1e-6) check2 = assert_util.assert_less( st.min_discrepancy_of_true_means_detectable_by_dkwm( num_samples, low=-1., high=1., # Smaller false fail rate because of different batch sizes between # these two checks. false_fail_rate=1e-7, false_pass_rate=1e-6), # 4% relative error 0.08) self.evaluate([check1, check2])
def test_dkwm_mean_in_interval_one_sample_assertion(self): rng = np.random.RandomState(seed=0) num_samples = 5000 # Test that the test assertion agrees that the mean of the standard # uniform distribution is between 0.4 and 0.6. samples = rng.uniform(size=num_samples).astype(np.float32) self.evaluate(st.assert_true_mean_in_interval_by_dkwm( samples, 0., 1., expected_low=0.4, expected_high=0.6, false_fail_rate=1e-6)) # Test that the test assertion confirms that the mean of the # standard uniform distribution is not between 0.2 and 0.4. with self.assertRaisesOpError("true mean greater than expected"): self.evaluate(st.assert_true_mean_in_interval_by_dkwm( samples, 0., 1., expected_low=0.2, expected_high=0.4, false_fail_rate=1e-6)) # Test that the test assertion confirms that the mean of the # standard uniform distribution is not between 0.6 and 0.8. with self.assertRaisesOpError("true mean smaller than expected"): self.evaluate(st.assert_true_mean_in_interval_by_dkwm( samples, 0., 1., expected_low=0.6, expected_high=0.8, false_fail_rate=1e-6))
def test_dkwm_mean_in_interval_one_sample_assertion(self): rng = np.random.RandomState(seed=0) num_samples = 5000 # Test that the test assertion agrees that the mean of the standard # uniform distribution is between 0.4 and 0.6. samples = rng.uniform(size=num_samples).astype(np.float32) self.evaluate( st.assert_true_mean_in_interval_by_dkwm(samples, 0., 1., expected_low=0.4, expected_high=0.6, false_fail_rate=1e-6)) # Test that the test assertion confirms that the mean of the # standard uniform distribution is not between 0.2 and 0.4. with self.assertRaisesOpError("true mean greater than expected"): self.evaluate( st.assert_true_mean_in_interval_by_dkwm(samples, 0., 1., expected_low=0.2, expected_high=0.4, false_fail_rate=1e-6)) # Test that the test assertion confirms that the mean of the # standard uniform distribution is not between 0.6 and 0.8. with self.assertRaisesOpError("true mean smaller than expected"): self.evaluate( st.assert_true_mean_in_interval_by_dkwm(samples, 0., 1., expected_low=0.6, expected_high=0.8, false_fail_rate=1e-6))
def _testSampleConsistentLogProbInterval(self, concentrations, det_bounds, dim, num_samples=int(1e5), dtype=np.float32, false_fail_rate=1e-6, target_discrepancy=0.1, seed=42): # Consider the set M of dim x dim correlation matrices whose # determinant exceeds some bound (rationale for bound forthwith). # - This is a (convex!) shape in dim * (dim - 1) / 2 dimensions # (because a correlation matrix is determined by its lower # triangle, and the main diagonal is all 1s). # - Further, M is contained entirely in the [-1,1] cube, # because no correlation can fall outside that interval. # # We have two different ways to estimate the volume of M: # - Importance sampling from the LKJ distribution # - Importance sampling from the uniform distribution on the cube # # This test checks that these two methods agree. However, because # the uniform proposal leads to many rejections (thus slowness), # those volumes are computed offline and the confidence intervals # are presented to this test procedure in the "volume_bounds" # table. # # Why place a lower bound on the determinant? Because for eta > 1, # the density of LKJ approaches 0 as the determinant approaches 0. # However, the test methodology requires an upper bound on the # improtance weights produced. Rejecting matrices with too-small # determinant (from both methods) allows me to supply that bound. # # I considered several alternative regions whose volume I might # know analytically (without having to do rejection). # - Option a: Some hypersphere guaranteed to be contained inside M. # - Con: I don't know a priori how to find a radius for it. # - Con: I still need a lower bound on the determinants that appear # in this sphere, and I don't know how to compute it. # - Option b: Some trapezoid given as the convex hull of the # nearly-extreme correlation matrices (i.e., those that partition # the variables into two strongly anti-correclated groups). # - Con: Would have to dig up n-d convex hull code to implement this. # - Con: Need to compute the volume of that convex hull. # - Con: Need a bound on the determinants of the matrices in that hull. # - Option c: Same thing, but with the matrices that make a single pair # of variables strongly correlated (or anti-correlated), and leaves # the others uncorrelated. # - Same cons, except that there is a determinant bound (which # felt pretty loose). lows = [dtype(volume_bounds[dim][db][0]) for db in det_bounds] highs = [dtype(volume_bounds[dim][db][1]) for db in det_bounds] concentration = np.array(concentrations, dtype=dtype) det_bounds = np.array(det_bounds, dtype=dtype) # Due to possible numerical inaccuracies while lower bounding the # determinant, the maximum of the importance weights may exceed the # theoretical maximum (importance_maxima). We add a tolerance to guard # against this. An alternative would have been to add a threshold while # filtering in _det_ok_mask, but that would affect the mean as well. high_tolerance = 1e-6 testee_lkj = tfd.LKJ(dimension=dim, concentration=concentration, validate_args=True) x = testee_lkj.sample(num_samples, seed=seed) importance_weights = (tf.exp(-testee_lkj.log_prob(x)) * _det_ok_mask(x, det_bounds)) importance_maxima = (1. / det_bounds)**(concentration - 1) * tf.exp( testee_lkj._log_normalization()) check1 = st.assert_true_mean_in_interval_by_dkwm( samples=importance_weights, low=0., high=importance_maxima + high_tolerance, expected_low=lows, expected_high=highs, false_fail_rate=false_fail_rate) check2 = tf.assert_less( st.min_discrepancy_of_true_means_detectable_by_dkwm( num_samples, low=0., high=importance_maxima + high_tolerance, false_fail_rate=false_fail_rate, false_pass_rate=false_fail_rate), dtype(target_discrepancy)) self.evaluate([check1, check2])
def _testSampleConsistentLogProbInterval( self, concentrations, det_bounds, dim, num_samples=int(1e5), dtype=np.float32, false_fail_rate=1e-6, target_discrepancy=0.1, seed=42): # Consider the set M of dim x dim correlation matrices whose # determinant exceeds some bound (rationale for bound forthwith). # - This is a (convex!) shape in dim * (dim - 1) / 2 dimensions # (because a correlation matrix is determined by its lower # triangle, and the main diagonal is all 1s). # - Further, M is contained entirely in the [-1,1] cube, # because no correlation can fall outside that interval. # # We have two different ways to estimate the volume of M: # - Importance sampling from the LKJ distribution # - Importance sampling from the uniform distribution on the cube # # This test checks that these two methods agree. However, because # the uniform proposal leads to many rejections (thus slowness), # those volumes are computed offline and the confidence intervals # are presented to this test procedure in the "volume_bounds" # table. # # Why place a lower bound on the determinant? Because for eta > 1, # the density of LKJ approaches 0 as the determinant approaches 0. # However, the test methodology requires an upper bound on the # improtance weights produced. Rejecting matrices with too-small # determinant (from both methods) allows me to supply that bound. # # I considered several alternative regions whose volume I might # know analytically (without having to do rejection). # - Option a: Some hypersphere guaranteed to be contained inside M. # - Con: I don't know a priori how to find a radius for it. # - Con: I still need a lower bound on the determinants that appear # in this sphere, and I don't know how to compute it. # - Option b: Some trapezoid given as the convex hull of the # nearly-extreme correlation matrices (i.e., those that partition # the variables into two strongly anti-correclated groups). # - Con: Would have to dig up n-d convex hull code to implement this. # - Con: Need to compute the volume of that convex hull. # - Con: Need a bound on the determinants of the matrices in that hull. # - Option c: Same thing, but with the matrices that make a single pair # of variables strongly correlated (or anti-correlated), and leaves # the others uncorrelated. # - Same cons, except that there is a determinant bound (which # felt pretty loose). lows = [dtype(volume_bounds[dim][db][0]) for db in det_bounds] highs = [dtype(volume_bounds[dim][db][1]) for db in det_bounds] concentration = np.array(concentrations, dtype=dtype) det_bounds = np.array(det_bounds, dtype=dtype) # Due to possible numerical inaccuracies while lower bounding the # determinant, the maximum of the importance weights may exceed the # theoretical maximum (importance_maxima). We add a tolerance to guard # against this. An alternative would have been to add a threshold while # filtering in _det_ok_mask, but that would affect the mean as well. high_tolerance = 1e-6 testee_lkj = tfd.LKJ( dimension=dim, concentration=concentration, validate_args=True) x = testee_lkj.sample(num_samples, seed=seed) importance_weights = ( tf.exp(-testee_lkj.log_prob(x)) * _det_ok_mask(x, det_bounds)) importance_maxima = (1. / det_bounds) ** (concentration - 1) * tf.exp( testee_lkj._log_normalization()) check1 = st.assert_true_mean_in_interval_by_dkwm( samples=importance_weights, low=0., high=importance_maxima + high_tolerance, expected_low=lows, expected_high=highs, false_fail_rate=false_fail_rate) check2 = tf.assert_less( st.min_discrepancy_of_true_means_detectable_by_dkwm( num_samples, low=0., high=importance_maxima + high_tolerance, false_fail_rate=false_fail_rate, false_pass_rate=false_fail_rate), dtype(target_discrepancy)) self.evaluate([check1, check2])