def test_chi2_sample_grad(self): set_rng_seed(0) # see Note [Randomized statistical tests] num_samples = 100 for df in [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3, 1e4]: dfs = Variable(torch.Tensor([df] * num_samples), requires_grad=True) x = Chi2(dfs).rsample() x.sum().backward() x, ind = x.data.sort() x = x.numpy() actual_grad = dfs.grad.data[ind].numpy() # Compare with expected gradient dx/ddf along constant cdf(x,df). cdf = scipy.stats.chi2.cdf pdf = scipy.stats.chi2.pdf eps = 0.02 * df if df < 100 else 0.02 * df ** 0.5 cdf_df = (cdf(x, df + eps) - cdf(x, df - eps)) / (2 * eps) cdf_x = pdf(x, df) expected_grad = -cdf_df / cdf_x rel_error = np.abs(actual_grad - expected_grad) / (expected_grad + 1e-100) self.assertLess(np.max(rel_error), 0.005, '\n'.join(['Bad gradients for Chi2({})'.format(df), 'x {}'.format(x), 'expected {}'.format(expected_grad), 'actual {}'.format(actual_grad), 'rel error {}'.format(rel_error), 'max error {}'.format(rel_error.max())]))
def test_beta_sample_grad(self): set_rng_seed(0) # see Note [Randomized statistical tests] num_samples = 20 grid = [1e-2, 1e-1, 1e0, 1e1, 1e2] for alpha, beta in product(grid, grid): alphas = Variable(torch.FloatTensor([alpha] * num_samples), requires_grad=True) betas = Variable(torch.FloatTensor([beta] * num_samples).type_as(alphas)) x = Beta(alphas, betas).rsample() x.sum().backward() x, ind = x.data.sort() x = x.numpy() actual_grad = alphas.grad.data[ind].numpy() # Compare with expected gradient dx/dalpha along constant cdf(x,alpha,beta). cdf = scipy.stats.beta.cdf pdf = scipy.stats.beta.pdf eps = 0.01 * alpha / (1.0 + np.sqrt(alpha)) cdf_alpha = (cdf(x, alpha + eps, beta) - cdf(x, alpha - eps, beta)) / (2 * eps) cdf_x = pdf(x, alpha, beta) expected_grad = -cdf_alpha / cdf_x rel_error = np.abs(actual_grad - expected_grad) / (expected_grad + 1e-100) self.assertLess(np.max(rel_error), 0.001, '\n'.join(['Bad gradients for Beta({}, {})'.format(alpha, beta), 'x {}'.format(x), 'expected {}'.format(expected_grad), 'actual {}'.format(actual_grad), 'rel error {}'.format(rel_error), 'max error {}'.format(rel_error.max())]))
def test_dirichlet_sample(self): set_rng_seed(0) # see Note [Randomized statistical tests] alpha = torch.exp(torch.randn(3)) self._check_sampler_sampler(Dirichlet(alpha), scipy.stats.dirichlet(alpha.numpy()), 'Dirichlet(alpha={})'.format(list(alpha)), multivariate=True)
def test_gamma_sample_grad(self): set_rng_seed(1) # see Note [Randomized statistical tests] num_samples = 100 for alpha in [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3, 1e4]: alphas = Variable(torch.FloatTensor([alpha] * num_samples), requires_grad=True) betas = Variable(torch.ones(num_samples).type_as(alphas)) x = Gamma(alphas, betas).rsample() x.sum().backward() x, ind = x.data.sort() x = x.numpy() actual_grad = alphas.grad.data[ind].numpy() # Compare with expected gradient dx/dalpha along constant cdf(x,alpha). cdf = scipy.stats.gamma.cdf pdf = scipy.stats.gamma.pdf eps = 0.01 * alpha / (1.0 + alpha ** 0.5) cdf_alpha = (cdf(x, alpha + eps) - cdf(x, alpha - eps)) / (2 * eps) cdf_x = pdf(x, alpha) expected_grad = -cdf_alpha / cdf_x rel_error = np.abs(actual_grad - expected_grad) / (expected_grad + 1e-30) self.assertLess(np.max(rel_error), 0.0005, '\n'.join(['Bad gradients for Gamma({}, 1)'.format(alpha), 'x {}'.format(x), 'expected {}'.format(expected_grad), 'actual {}'.format(actual_grad), 'rel error {}'.format(rel_error), 'max error {}'.format(rel_error.max()), 'at alpha={}, x={}'.format(alpha, x[rel_error.argmax()])]))
def test_cauchy(self): loc = Variable(torch.zeros(5, 5), requires_grad=True) scale = Variable(torch.ones(5, 5), requires_grad=True) loc_1d = Variable(torch.zeros(1), requires_grad=True) scale_1d = Variable(torch.ones(1), requires_grad=True) self.assertEqual(Cauchy(loc, scale).sample().size(), (5, 5)) self.assertEqual(Cauchy(loc, scale).sample_n(7).size(), (7, 5, 5)) self.assertEqual(Cauchy(loc_1d, scale_1d).sample().size(), (1,)) self.assertEqual(Cauchy(loc_1d, scale_1d).sample_n(1).size(), (1, 1)) self.assertEqual(Cauchy(0.0, 1.0).sample_n(1).size(), (1,)) set_rng_seed(1) self._gradcheck_log_prob(Uniform, (loc, scale)) self._gradcheck_log_prob(Uniform, (loc, 1.0)) self._gradcheck_log_prob(Uniform, (0.0, scale)) state = torch.get_rng_state() eps = loc.new(loc.size()).cauchy_() torch.set_rng_state(state) c = Cauchy(loc, scale).rsample() c.backward(torch.ones_like(c)) self.assertEqual(loc.grad, torch.ones_like(scale)) self.assertEqual(scale.grad, eps) loc.grad.zero_() scale.grad.zero_()
def test_uniform(self): low = Variable(torch.zeros(5, 5), requires_grad=True) high = Variable(torch.ones(5, 5) * 3, requires_grad=True) low_1d = Variable(torch.zeros(1), requires_grad=True) high_1d = Variable(torch.ones(1) * 3, requires_grad=True) self.assertEqual(Uniform(low, high).sample().size(), (5, 5)) self.assertEqual(Uniform(low, high).sample_n(7).size(), (7, 5, 5)) self.assertEqual(Uniform(low_1d, high_1d).sample().size(), (1,)) self.assertEqual(Uniform(low_1d, high_1d).sample_n(1).size(), (1, 1)) self.assertEqual(Uniform(0.0, 1.0).sample_n(1).size(), (1,)) # Check log_prob computation when value outside range uniform = Uniform(low_1d, high_1d) above_high = Variable(torch.Tensor([4.0])) below_low = Variable(torch.Tensor([-1.0])) self.assertEqual(uniform.log_prob(above_high).data[0], -float('inf'), allow_inf=True) self.assertEqual(uniform.log_prob(below_low).data[0], -float('inf'), allow_inf=True) set_rng_seed(1) self._gradcheck_log_prob(Uniform, (low, high)) self._gradcheck_log_prob(Uniform, (low, 1.0)) self._gradcheck_log_prob(Uniform, (0.0, high)) state = torch.get_rng_state() rand = low.new(low.size()).uniform_() torch.set_rng_state(state) u = Uniform(low, high).rsample() u.backward(torch.ones_like(u)) self.assertEqual(low.grad, 1 - rand) self.assertEqual(high.grad, rand) low.grad.zero_() high.grad.zero_()
def test_beta_sample(self): set_rng_seed(1) # see Note [Randomized statistical tests] for alpha, beta in product([0.1, 1.0, 10.0], [0.1, 1.0, 10.0]): self._check_sampler_sampler(Beta(alpha, beta), scipy.stats.beta(alpha, beta), 'Beta(alpha={}, beta={})'.format(alpha, beta)) # Check that small alphas do not cause NANs. for Tensor in [torch.FloatTensor, torch.DoubleTensor]: x = Beta(Tensor([1e-6]), Tensor([1e-6])).sample()[0] self.assertTrue(np.isfinite(x) and x > 0, 'Invalid Beta.sample(): {}'.format(x))
def test_normal(self): mean = Variable(torch.randn(5, 5), requires_grad=True) std = Variable(torch.randn(5, 5).abs(), requires_grad=True) mean_1d = Variable(torch.randn(1), requires_grad=True) std_1d = Variable(torch.randn(1), requires_grad=True) mean_delta = torch.Tensor([1.0, 0.0]) std_delta = torch.Tensor([1e-5, 1e-5]) self.assertEqual(Normal(mean, std).sample().size(), (5, 5)) self.assertEqual(Normal(mean, std).sample_n(7).size(), (7, 5, 5)) self.assertEqual(Normal(mean_1d, std_1d).sample_n(1).size(), (1, 1)) self.assertEqual(Normal(mean_1d, std_1d).sample().size(), (1,)) self.assertEqual(Normal(0.2, .6).sample_n(1).size(), (1,)) self.assertEqual(Normal(-0.7, 50.0).sample_n(1).size(), (1,)) # sample check for extreme value of mean, std set_rng_seed(1) self.assertEqual(Normal(mean_delta, std_delta).sample(sample_shape=(1, 2)), torch.Tensor([[[1.0, 0.0], [1.0, 0.0]]]), prec=1e-4) self._gradcheck_log_prob(Normal, (mean, std)) self._gradcheck_log_prob(Normal, (mean, 1.0)) self._gradcheck_log_prob(Normal, (0.0, std)) state = torch.get_rng_state() eps = torch.normal(torch.zeros_like(mean), torch.ones_like(std)) torch.set_rng_state(state) z = Normal(mean, std).rsample() z.backward(torch.ones_like(z)) self.assertEqual(mean.grad, torch.ones_like(mean)) self.assertEqual(std.grad, eps) mean.grad.zero_() std.grad.zero_() self.assertEqual(z.size(), (5, 5)) def ref_log_prob(idx, x, log_prob): m = mean.data.view(-1)[idx] s = std.data.view(-1)[idx] expected = (math.exp(-(x - m) ** 2 / (2 * s ** 2)) / math.sqrt(2 * math.pi * s ** 2)) self.assertAlmostEqual(log_prob, math.log(expected), places=3) self._check_log_prob(Normal(mean, std), ref_log_prob)
def test_laplace(self): loc = Variable(torch.randn(5, 5), requires_grad=True) scale = Variable(torch.randn(5, 5).abs(), requires_grad=True) loc_1d = Variable(torch.randn(1), requires_grad=True) scale_1d = Variable(torch.randn(1), requires_grad=True) loc_delta = torch.Tensor([1.0, 0.0]) scale_delta = torch.Tensor([1e-5, 1e-5]) self.assertEqual(Laplace(loc, scale).sample().size(), (5, 5)) self.assertEqual(Laplace(loc, scale).sample_n(7).size(), (7, 5, 5)) self.assertEqual(Laplace(loc_1d, scale_1d).sample_n(1).size(), (1, 1)) self.assertEqual(Laplace(loc_1d, scale_1d).sample().size(), (1,)) self.assertEqual(Laplace(0.2, .6).sample_n(1).size(), (1,)) self.assertEqual(Laplace(-0.7, 50.0).sample_n(1).size(), (1,)) # sample check for extreme value of mean, std set_rng_seed(0) self.assertEqual(Laplace(loc_delta, scale_delta).sample(sample_shape=(1, 2)), torch.Tensor([[[1.0, 0.0], [1.0, 0.0]]]), prec=1e-4) self._gradcheck_log_prob(Laplace, (loc, scale)) self._gradcheck_log_prob(Laplace, (loc, 1.0)) self._gradcheck_log_prob(Laplace, (0.0, scale)) state = torch.get_rng_state() eps = torch.ones_like(loc).uniform_(-.5, .5) torch.set_rng_state(state) z = Laplace(loc, scale).rsample() z.backward(torch.ones_like(z)) self.assertEqual(loc.grad, torch.ones_like(loc)) self.assertEqual(scale.grad, -eps.sign() * torch.log1p(-2 * eps.abs())) loc.grad.zero_() scale.grad.zero_() self.assertEqual(z.size(), (5, 5)) def ref_log_prob(idx, x, log_prob): m = loc.data.view(-1)[idx] s = scale.data.view(-1)[idx] expected = (-math.log(2 * s) - abs(x - m) / s) self.assertAlmostEqual(log_prob, expected, places=3) self._check_log_prob(Laplace(loc, scale), ref_log_prob)
def test_categorical_2d(self): probabilities = [[0.1, 0.2, 0.3], [0.5, 0.3, 0.2]] probabilities_1 = [[1.0, 0.0], [0.0, 1.0]] p = Variable(torch.Tensor(probabilities), requires_grad=True) s = Variable(torch.Tensor(probabilities_1), requires_grad=True) self.assertEqual(Categorical(p).sample().size(), (2,)) self.assertEqual(Categorical(p).sample(sample_shape=(3, 4)).size(), (3, 4, 2)) self.assertEqual(Categorical(p).sample_n(6).size(), (6, 2)) self._gradcheck_log_prob(Categorical, (p,)) # sample check for extreme value of probs set_rng_seed(0) self.assertEqual(Categorical(s).sample(sample_shape=(2,)).data, torch.Tensor([[0, 1], [0, 1]])) def ref_log_prob(idx, val, log_prob): sample_prob = p.data[idx][val] / p.data[idx].sum() self.assertEqual(log_prob, math.log(sample_prob)) self._check_log_prob(Categorical(p), ref_log_prob)
def test_chi2_sample(self): set_rng_seed(0) # see Note [Randomized statistical tests] for df in [0.1, 1.0, 5.0]: self._check_sampler_sampler(Chi2(df), scipy.stats.chi2(df), 'Chi2(df={})'.format(df))
def test_gamma_sample(self): set_rng_seed(0) # see Note [Randomized statistical tests] for alpha, beta in product([0.1, 1.0, 5.0], [0.1, 1.0, 10.0]): self._check_sampler_sampler(Gamma(alpha, beta), scipy.stats.gamma(alpha, scale=1.0 / beta), 'Gamma(alpha={}, beta={})'.format(alpha, beta))
def test_laplace_sample(self): set_rng_seed(1) # see Note [Randomized statistical tests] for loc, scale in product([-1.0, 0.0, 1.0], [0.1, 1.0, 10.0]): self._check_sampler_sampler(Laplace(loc, scale), scipy.stats.laplace(loc=loc, scale=scale), 'Laplace(loc={}, scale={})'.format(loc, scale))
def test_exponential_sample(self): set_rng_seed(1) # see Note [Randomized statistical tests] for rate in [1e-5, 1.0, 10.]: self._check_sampler_sampler(Exponential(rate), scipy.stats.expon(scale=1. / rate), 'Exponential(rate={})'.format(rate))
def test_normal_sample(self): set_rng_seed(0) # see Note [Randomized statistical tests] for mean, std in product([-1.0, 0.0, 1.0], [0.1, 1.0, 10.0]): self._check_sampler_sampler(Normal(mean, std), scipy.stats.norm(loc=mean, scale=std), 'Normal(mean={}, std={})'.format(mean, std))