def test_inference_mini_captcha_posterior_importance_sampling(self):
samples = int(importance_sampling_samples / len(self._model._alphabet))
test_letters = self._model._alphabet
start = time.time()
posteriors = []
map_estimates = []
for i in range(len(self._model._alphabet)):
posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING, observe={'query_image': self._test_images[i]})
posteriors.append(posterior)
map_estimates.append(self._model._alphabet[int(posterior.mode)])
add_importance_sampling_duration(time.time() - start)
accuracy = sum([1 if map_estimates[i] == test_letters[i] else 0 for i in range(len(test_letters))])/len(test_letters)
kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(util.empirical_to_categorical(p, max_val=len(self._model._alphabet)-1), tp) for (p, tp) in zip(posteriors, self._true_posteriors)]))
util.eval_print('samples', 'test_letters', 'map_estimates', 'accuracy', 'kl_divergence')
add_importance_sampling_kl_divergence(kl_divergence)
self.assertGreater(accuracy, 0.9)
self.assertLess(kl_divergence, 0.25)
def test_inference_gum_posterior_random_walk_metropolis_hastings(self):
samples = random_walk_metropolis_hastings_samples
burn_in = random_walk_metropolis_hastings_burn_in
true_posterior = Normal(7.25, math.sqrt(1/1.2))
posterior_mean_correct = float(true_posterior.mean)
posterior_stddev_correct = float(true_posterior.stddev)
start = time.time()
posterior = self._model.posterior_results(samples, inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS, observe={'obs0': 8, 'obs1': 9})[burn_in:]
add_random_walk_metropolis_hastings_duration(time.time() - start)
posterior_mean = float(posterior.mean)
posterior_stddev = float(posterior.stddev)
kl_divergence = float(pyprob.distributions.Distribution.kl_divergence(true_posterior, Normal(posterior.mean, posterior.stddev)))
util.eval_print('samples', 'burn_in', 'posterior_mean', 'posterior_mean_correct', 'posterior_stddev', 'posterior_stddev_correct', 'kl_divergence')
add_random_walk_metropolis_hastings_kl_divergence(kl_divergence)
self.assertAlmostEqual(posterior_mean, posterior_mean_correct, places=0)
self.assertAlmostEqual(posterior_stddev, posterior_stddev_correct, places=0)
self.assertLess(kl_divergence, 0.25)
def test_model_remote_set_defaults_and_addresses_addresses(self):
addresses_controlled_correct = [
'[forward()+0x3a9]__Normal__1', '[forward()+0x3a9]__Normal__2',
'[forward()+0xc88]__Normal__2'
]
addresses_all_correct = [
'[forward()+0x3a9]__Normal__1', '[forward()+0x3a9]__Normal__2',
'[forward()+0xc88]__Normal__1', '[forward()+0xc88]__Normal__2',
'[forward()+0x1573]__Normal__1', '[forward()+0x1573]__Normal__2',
'[forward()+0x1ee5]__Normal__1'
]
trace = next(self._model._trace_generator())
addresses_controlled = [s.address for s in trace.variables_controlled]
addresses_all = [s.address for s in trace.variables]
util.eval_print('addresses_controlled', 'addresses_controlled_correct',
'addresses_all', 'addresses_all_correct')
self.assertEqual(addresses_controlled, addresses_controlled_correct)
self.assertEqual(addresses_all, addresses_all_correct)
def test_model_prior_on_disk(self):
file_name = os.path.join(tempfile.mkdtemp(), str(uuid.uuid4()))
num_traces = 1000
prior_mean_correct = 1
prior_stddev_correct = math.sqrt(5)
prior_length_correct = 2 * num_traces
prior = self._model.prior_distribution(num_traces, file_name=file_name)
prior.close()
prior = self._model.prior_distribution(num_traces, file_name=file_name)
# prior.close()
prior_length = prior.length
prior_mean = float(prior.mean)
prior_stddev = float(prior.stddev)
util.eval_print('num_traces', 'prior_mean', 'prior_mean_correct',
'prior_stddev', 'prior_stddev_correct', 'prior_length',
'prior_length_correct')
self.assertAlmostEqual(prior_mean, prior_mean_correct, places=0)
self.assertAlmostEqual(prior_stddev, prior_stddev_correct, places=0)
self.assertEqual(prior_length, prior_length_correct)
def test_posterior_importance_sampling_with_inference_network(self):
trace_addresses_controlled_correct = [
'34__forward__x__Uniform__2', '48__forward__y__Uniform__2'
]
trace_addresses_correct = [
'34__forward__x__Uniform__1', '48__forward__y__Uniform__1',
'34__forward__x__Uniform__2', '48__forward__y__Uniform__2',
'92__forward__?__Normal__1', '106__forward__?__Normal__1'
]
self._model.learn_inference_network(num_traces=10,
observe_embeddings={
'obs0': {
'dim': 128,
'depth': 6
},
'obs1': {
'dim': 128,
'depth': 6
}
})
posterior = self._model.posterior(
1,
inference_engine=InferenceEngine.
IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK,
observe={
'obs0': 8,
'obs1': 9
})
trace = posterior[0]
trace_addresses_controlled = [
v.address for v in trace.variables_controlled
]
trace_addresses = [v.address for v in trace.variables]
util.eval_print('trace', 'trace_addresses_controlled',
'trace_addresses')
self.assertEqual(trace_addresses_controlled,
trace_addresses_controlled_correct)
self.assertEqual(trace_addresses, trace_addresses_correct)
def test_inference_hmm_posterior_importance_sampling(self):
samples = importance_sampling_samples
observation = {
'obs{}'.format(i): self._observation[i]
for i in range(len(self._observation))
}
posterior_mean_correct = self._posterior_mean_correct
posterior_effective_sample_size_min = samples * 0.0015
start = time.time()
posterior = self._model.posterior_distribution(samples,
observe=observation)
add_importance_sampling_duration(time.time() - start)
posterior_mean_unweighted = posterior.unweighted().mean
posterior_mean = posterior.mean
posterior_effective_sample_size = float(
posterior.effective_sample_size)
print(posterior[0])
l2_distance = float(
F.pairwise_distance(posterior_mean, posterior_mean_correct).sum())
kl_divergence = float(
sum([
pyprob.distributions.Distribution.kl_divergence(
Categorical(i + util._epsilon),
Categorical(j + util._epsilon))
for (i, j) in zip(posterior_mean, posterior_mean_correct)
]))
util.eval_print('samples', 'posterior_mean_unweighted',
'posterior_mean', 'posterior_mean_correct',
'posterior_effective_sample_size',
'posterior_effective_sample_size_min', 'l2_distance',
'kl_divergence')
add_importance_sampling_kl_divergence(kl_divergence)
self.assertGreater(posterior_effective_sample_size,
posterior_effective_sample_size_min)
self.assertLess(l2_distance, 3)
self.assertLess(kl_divergence, 1)
def test_inference_branching_importance_sampling(self):
samples = importance_sampling_samples
posterior_correct = util.empirical_to_categorical(
self.true_posterior(), max_val=40)
start = time.time()
posterior = util.empirical_to_categorical(
self._model.posterior_distribution(samples, observe={'obs': 6}),
max_val=40)
add_importance_sampling_duration(time.time() - start)
posterior_probs = util.to_numpy(posterior._probs)
posterior_probs_correct = util.to_numpy(posterior_correct._probs)
kl_divergence = float(
pyprob.distributions.Distribution.kl_divergence(
posterior, posterior_correct))
util.eval_print('samples', 'posterior_probs',
'posterior_probs_correct', 'kl_divergence')
add_importance_sampling_kl_divergence(kl_divergence)
self.assertLess(kl_divergence, 0.75)
def test_prior_for_inference_network(self):
trace_addresses_accepted_correct = [
'38__forward__x__Uniform__replaced',
'48__forward__y__Uniform__replaced'
]
trace_addresses_correct = [
'38__forward__x__Uniform__replaced',
'48__forward__y__Uniform__replaced', '100__forward__?__Normal__1',
'114__forward__?__Normal__1'
]
trace = OnlineDataset(model=self._model).__getitem__(0)[0]
trace_addresses_accepted = [
v['address'] for v in trace if v['control'] and v['accepted']
]
trace_addresses = [v['address'] for v in trace]
util.eval_print('trace', 'trace_addresses_accepted', 'trace_addresses')
self.assertEqual(trace_addresses_accepted,
trace_addresses_accepted_correct)
self.assertEqual(trace_addresses, trace_addresses_correct)
def test_inference_mini_captcha_posterior_random_walk_metropolis_hastings(self):
samples = int(random_walk_metropolis_hastings_samples / len(self._model._alphabet))
burn_in = int(random_walk_metropolis_hastings_burn_in / len(self._model._alphabet))
test_letters = self._model._alphabet
start = time.time()
posteriors = []
map_estimates = []
for i in range(len(self._model._alphabet)):
posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS, observe={'query_image': self._test_images[i]})[burn_in:]
posteriors.append(posterior)
map_estimates.append(self._model._alphabet[int(posterior.combine_duplicates().mode)])
add_random_walk_metropolis_hastings_duration(time.time() - start)
accuracy = sum([1 if map_estimates[i] == test_letters[i] else 0 for i in range(len(test_letters))])/len(test_letters)
kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(util.empirical_to_categorical(p, max_val=len(self._model._alphabet)-1), tp) for (p, tp) in zip(posteriors, self._true_posteriors)]))
util.eval_print('samples', 'test_letters', 'map_estimates', 'accuracy', 'kl_divergence')
add_random_walk_metropolis_hastings_kl_divergence(kl_divergence)
self.assertGreater(accuracy, 0.9)
self.assertLess(kl_divergence, 0.25)
def test_observation_style2_gum_posterior_importance_sampling(self):
samples = importance_sampling_samples
true_posterior = Normal(7.25, math.sqrt(1/1.2))
posterior_mean_correct = float(true_posterior.mean)
posterior_stddev_correct = float(true_posterior.stddev)
prior_mean_correct = 1.
prior_stddev_correct = math.sqrt(5)
posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING, observe={'obs0': 8, 'obs1': 9})
posterior_mean = float(posterior.mean)
posterior_mean_unweighted = float(posterior.unweighted().mean)
posterior_stddev = float(posterior.stddev)
posterior_stddev_unweighted = float(posterior.unweighted().stddev)
kl_divergence = float(pyprob.distributions.Distribution.kl_divergence(true_posterior, Normal(posterior.mean, posterior.stddev)))
util.eval_print('samples', 'prior_mean_correct', 'posterior_mean_unweighted', 'posterior_mean', 'posterior_mean_correct', 'prior_stddev_correct', 'posterior_stddev_unweighted', 'posterior_stddev', 'posterior_stddev_correct', 'kl_divergence')
self.assertAlmostEqual(posterior_mean_unweighted, prior_mean_correct, places=0)
self.assertAlmostEqual(posterior_stddev_unweighted, prior_stddev_correct, places=0)
self.assertAlmostEqual(posterior_mean, posterior_mean_correct, places=0)
self.assertAlmostEqual(posterior_stddev, posterior_stddev_correct, places=0)
self.assertLess(kl_divergence, 0.25)
def test_save_offline_dataset(self):
dataset_dir = tempfile.mkdtemp()
num_traces_correct = 20
num_traces_per_file_correct = 5
num_files_correct = 4
self._model.save_dataset(dataset_dir=dataset_dir, num_traces=num_traces_correct, num_traces_per_file=num_traces_per_file_correct)
files = sorted(glob(os.path.join(dataset_dir, 'pyprob_traces_*')))
num_files = len(files)
dataset = OfflineDataset(dataset_dir)
hashes = dataset._hashes
indices = dataset._sorted_indices
sorted_on_disk = util.is_sorted(indices)
num_traces = len(dataset)
num_traces_per_file = num_traces / num_files
shutil.rmtree(dataset_dir)
util.eval_print('dataset_dir', 'num_traces', 'num_traces_correct', 'num_traces_per_file', 'num_traces_per_file_correct', 'files', 'num_files', 'num_files_correct', 'hashes', 'indices', 'sorted_on_disk')
self.assertEqual(num_files, num_files_correct)
self.assertEqual(num_traces, num_traces_correct)
self.assertEqual(num_traces_per_file, num_traces_per_file_correct)
self.assertFalse(sorted_on_disk)
def test_inference_mini_captcha_posterior_importance_sampling_with_inference_network(self):
samples = int(importance_sampling_with_inference_network_samples / len(self._model._alphabet))
test_letters = self._model._alphabet
self._model.learn_inference_network(num_traces=importance_sampling_with_inference_network_training_traces, observe_embeddings={'query_image': {'dim': 32, 'reshape': [1, 28, 28], 'embedding': ObserveEmbedding.CNN2D5C}})
start = time.time()
posteriors = []
map_estimates = []
for i in range(len(self._model._alphabet)):
posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK, observe={'query_image': self._test_images[i]})
posteriors.append(posterior)
map_estimates.append(self._model._alphabet[int(posterior.mode)])
add_importance_sampling_with_inference_network_duration(time.time() - start)
accuracy = sum([1 if map_estimates[i] == test_letters[i] else 0 for i in range(len(test_letters))])/len(test_letters)
kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(util.empirical_to_categorical(p, max_val=len(self._model._alphabet)-1), tp) for (p, tp) in zip(posteriors, self._true_posteriors)]))
util.eval_print('samples', 'test_letters', 'map_estimates', 'accuracy', 'kl_divergence')
add_importance_sampling_with_inference_network_kl_divergence(kl_divergence)
self.assertGreater(accuracy, 0.9)
self.assertLess(kl_divergence, 0.25)
def test_prior_inflation(self):
samples = 5000
categorical_prior_mean_correct = 1.7
categorical_prior_stddev_correct = 0.640312
categorical_prior_inflated_mean_correct = 1
categorical_prior_inflated_stddev_correct = 0.816497
normal_prior_mean_correct = 5
normal_prior_stddev_correct = 2
normal_prior_inflated_mean_correct = 5
normal_prior_inflated_stddev_correct = normal_prior_stddev_correct * 3
prior = self._model.prior_distribution(samples, prior_inflation=pyprob.PriorInflation.DISABLED)
categorical_prior = prior.map(lambda x: x[0])
categorical_prior_mean = float(categorical_prior.mean)
categorical_prior_stddev = float(categorical_prior.stddev)
normal_prior = prior.map(lambda x: x[1])
normal_prior_mean = float(normal_prior.mean)
normal_prior_stddev = float(normal_prior.stddev)
prior_inflated = self._model.prior_distribution(samples, prior_inflation=pyprob.PriorInflation.ENABLED)
categorical_prior_inflated = prior_inflated.map(lambda x: x[0])
categorical_prior_inflated_mean = float(categorical_prior_inflated.mean)
categorical_prior_inflated_stddev = float(categorical_prior_inflated.stddev)
normal_prior_inflated = prior_inflated.map(lambda x: x[1])
normal_prior_inflated_mean = float(normal_prior_inflated.mean)
normal_prior_inflated_stddev = float(normal_prior_inflated.stddev)
util.eval_print('samples', 'categorical_prior_mean', 'categorical_prior_mean_correct', 'categorical_prior_stddev', 'categorical_prior_stddev_correct', 'categorical_prior_inflated_mean', 'categorical_prior_inflated_mean_correct', 'categorical_prior_inflated_stddev', 'categorical_prior_inflated_stddev_correct', 'normal_prior_mean', 'normal_prior_mean_correct', 'normal_prior_stddev', 'normal_prior_stddev_correct', 'normal_prior_inflated_mean', 'normal_prior_inflated_mean_correct', 'normal_prior_inflated_stddev', 'normal_prior_inflated_stddev_correct')
self.assertAlmostEqual(categorical_prior_mean, categorical_prior_mean_correct, places=0)
self.assertAlmostEqual(categorical_prior_stddev, categorical_prior_stddev_correct, places=0)
self.assertAlmostEqual(categorical_prior_inflated_mean, categorical_prior_inflated_mean_correct, places=0)
self.assertAlmostEqual(categorical_prior_inflated_stddev, categorical_prior_inflated_stddev_correct, places=0)
self.assertAlmostEqual(normal_prior_mean, normal_prior_mean_correct, places=0)
self.assertAlmostEqual(normal_prior_stddev, normal_prior_stddev_correct, places=0)
self.assertAlmostEqual(normal_prior_inflated_mean, normal_prior_inflated_mean_correct, places=0)
self.assertAlmostEqual(normal_prior_inflated_stddev, normal_prior_inflated_stddev_correct, places=0)
def test_inference_hmm_posterior_importance_sampling_with_inference_network(self):
samples = importance_sampling_with_inference_network_samples
observation = {'obs{}'.format(i): self._observation[i] for i in range(len(self._observation))}
posterior_mean_correct = self._posterior_mean_correct
posterior_effective_sample_size_min = samples * 0.03
self._model.learn_inference_network(num_traces=importance_sampling_with_inference_network_training_traces, observe_embeddings={'obs{}'.format(i): {'depth': 2, 'dim': 16} for i in range(len(observation))})
start = time.time()
posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK, observe=observation)
add_importance_sampling_with_inference_network_duration(time.time() - start)
posterior_mean_unweighted = posterior.unweighted().mean
posterior_mean = posterior.mean
posterior_effective_sample_size = float(posterior.effective_sample_size)
l2_distance = float(F.pairwise_distance(posterior_mean, posterior_mean_correct).sum())
kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(Categorical(i + util._epsilon), Categorical(j + util._epsilon)) for (i, j) in zip(posterior_mean, posterior_mean_correct)]))
util.eval_print('samples', 'posterior_mean_unweighted', 'posterior_mean', 'posterior_mean_correct', 'posterior_effective_sample_size', 'posterior_effective_sample_size_min', 'l2_distance', 'kl_divergence')
add_importance_sampling_with_inference_network_kl_divergence(kl_divergence)
self.assertGreater(posterior_effective_sample_size, posterior_effective_sample_size_min)
self.assertLess(l2_distance, 3)
self.assertLess(kl_divergence, 1)
def test_trace_prior_for_inference_network(self):
trace_addresses_controlled_correct = [
'30__forward__x__Uniform__1', '40__forward__y__Uniform__1',
'30__forward__x__Uniform__2', '40__forward__y__Uniform__2'
]
trace_addresses_correct = [
'30__forward__x__Uniform__1', '40__forward__y__Uniform__1',
'30__forward__x__Uniform__2', '40__forward__y__Uniform__2',
'84__forward__?__Normal__1', '98__forward__?__Normal__1'
]
trace = OnlineDataset(model=self._model)[0]
trace_addresses_controlled = [
v.address for v in trace.variables_controlled
]
trace_addresses = [v.address for v in trace.variables]
util.eval_print('trace', 'trace_addresses_controlled',
'trace_addresses_controlled_correct',
'trace_addresses', 'trace_addresses_correct')
self.assertEqual(trace_addresses_controlled_correct,
trace_addresses_controlled)
self.assertEqual(trace_addresses_correct, trace_addresses)
def test_inference_branching_random_walk_metropolis_hastings(self):
samples = importance_sampling_samples
posterior_correct = util.empirical_to_categorical(
self.true_posterior(), max_val=40)
start = time.time()
posterior = util.empirical_to_categorical(self._model.posterior_return(
samples,
inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS,
observe={'obs': 6}),
max_val=40)
add_random_walk_metropolis_hastings_duration(time.time() - start)
posterior_probs = util.to_numpy(posterior._probs)
posterior_probs_correct = util.to_numpy(posterior_correct._probs)
kl_divergence = float(
pyprob.distributions.Distribution.kl_divergence(
posterior, posterior_correct))
util.eval_print('samples', 'posterior_probs',
'posterior_probs_correct', 'kl_divergence')
add_random_walk_metropolis_hastings_kl_divergence(kl_divergence)
self.assertLess(kl_divergence, 0.75)
def test_random_seed(self):
samples = 10
stochastic_samples = []
for i in range(samples):
pyprob.set_random_seed(None)
dist = pyprob.distributions.Normal(0, 1)
sample = dist.sample()
stochastic_samples.append(float(sample))
deterministic_samples = []
for i in range(samples):
pyprob.set_random_seed(123)
dist = pyprob.distributions.Normal(0, 1)
sample = dist.sample()
deterministic_samples.append(float(sample))
util.eval_print('samples', 'stochastic_samples',
'deterministic_samples')
self.assertTrue(not all(sample == stochastic_samples[0]
for sample in stochastic_samples))
self.assertTrue(
all(sample == deterministic_samples[0]
for sample in deterministic_samples))
def test_distributions_remote(self):
num_samples = 4000
prior_normal_mean_correct = Normal(1.75, 0.5).mean
prior_uniform_mean_correct = Uniform(1.2, 2.5).mean
prior_categorical_mean_correct = 1. # Categorical([0.1, 0.5, 0.4])
prior_poisson_mean_correct = Poisson(4.0).mean
prior_bernoulli_mean_correct = Bernoulli(0.2).mean
prior_beta_mean_correct = Beta(1.2, 2.5).mean
prior_exponential_mean_correct = Exponential(2.2).mean
prior_gamma_mean_correct = Gamma(0.5, 1.2).mean
prior_log_normal_mean_correct = LogNormal(0.5, 0.2).mean
prior_binomial_mean_correct = Binomial(10, 0.72).mean
prior_weibull_mean_correct = Weibull(1.1, 0.6).mean
prior = self._model.prior(num_samples)
prior_normal = prior.map(
lambda trace: trace.named_variables['normal'].value)
prior_uniform = prior.map(
lambda trace: trace.named_variables['uniform'].value)
prior_categorical = prior.map(
lambda trace: trace.named_variables['categorical'].value)
prior_poisson = prior.map(
lambda trace: trace.named_variables['poisson'].value)
prior_bernoulli = prior.map(
lambda trace: trace.named_variables['bernoulli'].value)
prior_beta = prior.map(
lambda trace: trace.named_variables['beta'].value)
prior_exponential = prior.map(
lambda trace: trace.named_variables['exponential'].value)
prior_gamma = prior.map(
lambda trace: trace.named_variables['gamma'].value)
prior_log_normal = prior.map(
lambda trace: trace.named_variables['log_normal'].value)
prior_binomial = prior.map(
lambda trace: trace.named_variables['binomial'].value)
prior_weibull = prior.map(
lambda trace: trace.named_variables['weibull'].value)
prior_normal_mean = util.to_numpy(prior_normal.mean)
prior_uniform_mean = util.to_numpy(prior_uniform.mean)
prior_categorical_mean = util.to_numpy(int(prior_categorical.mean))
prior_poisson_mean = util.to_numpy(prior_poisson.mean)
prior_bernoulli_mean = util.to_numpy(prior_bernoulli.mean)
prior_beta_mean = util.to_numpy(prior_beta.mean)
prior_exponential_mean = util.to_numpy(prior_exponential.mean)
prior_gamma_mean = util.to_numpy(prior_gamma.mean)
prior_log_normal_mean = util.to_numpy(prior_log_normal.mean)
prior_binomial_mean = util.to_numpy(prior_binomial.mean)
prior_weibull_mean = util.to_numpy(prior_weibull.mean)
util.eval_print('num_samples', 'prior_normal_mean',
'prior_normal_mean_correct', 'prior_uniform_mean',
'prior_uniform_mean_correct', 'prior_categorical_mean',
'prior_categorical_mean_correct', 'prior_poisson_mean',
'prior_poisson_mean_correct', 'prior_bernoulli_mean',
'prior_bernoulli_mean_correct', 'prior_beta_mean',
'prior_beta_mean_correct', 'prior_exponential_mean',
'prior_exponential_mean_correct', 'prior_gamma_mean',
'prior_gamma_mean_correct', 'prior_log_normal_mean',
'prior_log_normal_mean_correct', 'prior_binomial_mean',
'prior_binomial_mean_correct', 'prior_weibull_mean',
'prior_weibull_mean_correct')
self.assertTrue(
np.allclose(prior_normal_mean, prior_normal_mean_correct,
atol=0.1))
self.assertTrue(
np.allclose(prior_uniform_mean,
prior_uniform_mean_correct,
atol=0.1))
self.assertTrue(
np.allclose(prior_categorical_mean,
prior_categorical_mean_correct,
atol=0.1))
self.assertTrue(
np.allclose(prior_poisson_mean,
prior_poisson_mean_correct,
atol=0.1))
self.assertTrue(
np.allclose(prior_bernoulli_mean,
prior_bernoulli_mean_correct,
atol=0.1))
self.assertTrue(
np.allclose(prior_beta_mean, prior_beta_mean_correct, atol=0.1))
self.assertTrue(
np.allclose(prior_exponential_mean,
prior_exponential_mean_correct,
atol=0.1))
self.assertTrue(
np.allclose(prior_gamma_mean, prior_gamma_mean_correct, atol=0.1))
self.assertTrue(
np.allclose(prior_log_normal_mean,
prior_log_normal_mean_correct,
atol=0.1))
self.assertTrue(
np.allclose(prior_binomial_mean,
prior_binomial_mean_correct,
atol=0.1))
self.assertTrue(
np.allclose(prior_weibull_mean,
prior_weibull_mean_correct,
atol=0.1))
def test_state_address(self):
address = self._sample_address()
address_correct = '4__test_state_address__address'
util.eval_print('address', 'address_correct')
self.assertEqual(address, address_correct)
def test_train_offline_adam_larc_lr_poly2(self):
dataset_dir = tempfile.mkdtemp()
dataset_num_traces = 128
dataset_num_traces_per_file = 32
file_name = os.path.join(tempfile.mkdtemp(), str(uuid.uuid4()))
file_name_2 = os.path.join(tempfile.mkdtemp(), str(uuid.uuid4()))
file_name_3 = os.path.join(tempfile.mkdtemp(), str(uuid.uuid4()))
num_traces_end = 256
batch_size = 16
learning_rate_init_correct = 0.1
learning_rate_end_correct = 0.0025
print('Saving dataset\n')
self._model.save_dataset(
dataset_dir=dataset_dir,
num_traces=dataset_num_traces,
num_traces_per_file=dataset_num_traces_per_file)
self._model.reset_inference_network()
print('Training\n')
self._model.learn_inference_network(
dataset_dir=dataset_dir,
num_traces=num_traces_end / 2,
num_traces_end=num_traces_end,
batch_size=batch_size,
observe_embeddings={
'obs0': {
'dim': 16
},
'obs1': {
'dim': 16
}
},
optimizer_type=Optimizer.ADAM_LARC,
learning_rate_scheduler_type=LearningRateScheduler.POLY2,
learning_rate_init=learning_rate_init_correct,
learning_rate_end=learning_rate_end_correct,
log_file_name=file_name_2)
print('Saving\n')
# print(self._model._inference_network._optimizer)
optimizer_state_step_before_save = list(
self._model._inference_network._optimizer.state_dict()
['state'].values())[0]['step']
self._model.save_inference_network(file_name)
print('\nlog_file_name contents')
with open(file_name_2) as file:
for line in file:
print(line, end='')
self._model.reset_inference_network()
print('Loading\n')
self._model.load_inference_network(file_name)
# print(self._model._inference_network._optimizer)
optimizer_state_step_after_load = list(
self._model._inference_network._optimizer.state_dict()
['state'].values())[0]['step']
print('Training\n')
self._model.learn_inference_network(
dataset_dir=dataset_dir,
num_traces=num_traces_end / 2,
num_traces_end=num_traces_end,
batch_size=batch_size,
observe_embeddings={
'obs0': {
'dim': 16
},
'obs1': {
'dim': 16
}
},
optimizer_type=Optimizer.ADAM_LARC,
learning_rate_scheduler_type=LearningRateScheduler.POLY2,
learning_rate_init=learning_rate_init_correct,
learning_rate_end=learning_rate_end_correct,
log_file_name=file_name_3)
learning_rate_end = self._model._inference_network._optimizer.param_groups[
0]['lr']
print('\nlog_file_name contents')
with open(file_name_3) as file:
for line in file:
print(line, end='')
os.remove(file_name)
os.remove(file_name_2)
os.remove(file_name_3)
shutil.rmtree(dataset_dir)
util.eval_print('dataset_dir', 'dataset_num_traces',
'dataset_num_traces_per_file', 'file_name',
'file_name_2', 'file_name_3', 'num_traces_end',
'batch_size', 'learning_rate_init_correct',
'optimizer_state_step_before_save',
'optimizer_state_step_after_load', 'learning_rate_end',
'learning_rate_end_correct')
# util.eval_print('file_name', 'file_name_2', 'file_name_3', 'num_traces_end', 'batch_size', 'learning_rate_init_correct', 'learning_rate_end', 'learning_rate_end_correct')
self.assertEqual(optimizer_state_step_before_save,
optimizer_state_step_after_load)
self.assertAlmostEqual(learning_rate_end,
learning_rate_end_correct,
delta=learning_rate_end_correct)
def test_model_lmh_posterior_with_online_thinning(self):
thinning_steps = 10
posterior_num_traces = 4000
posterior_with_thinning_num_traces_correct = posterior_num_traces / thinning_steps
true_posterior = Normal(7.25, math.sqrt(1 / 1.2))
posterior_mean_correct = float(true_posterior.mean)
posterior_stddev_correct = float(true_posterior.stddev)
posterior = self._model.posterior_results(
num_traces=posterior_num_traces,
inference_engine=InferenceEngine.LIGHTWEIGHT_METROPOLIS_HASTINGS,
observe={
'obs0': 8,
'obs1': 9
})
posterior_num_traces = posterior.length
posterior_mean = float(posterior.mean)
posterior_stddev = float(posterior.stddev)
kl_divergence = float(
pyprob.distributions.Distribution.kl_divergence(
true_posterior, Normal(posterior.mean, posterior.stddev)))
posterior_with_thinning = self._model.posterior_results(
num_traces=posterior_num_traces,
inference_engine=InferenceEngine.LIGHTWEIGHT_METROPOLIS_HASTINGS,
observe={
'obs0': 8,
'obs1': 9
},
thinning_steps=thinning_steps)
posterior_with_thinning_num_traces = posterior_with_thinning.length
posterior_with_thinning_mean = float(posterior_with_thinning.mean)
posterior_with_thinning_stddev = float(posterior_with_thinning.stddev)
kl_divergence_with_thinning = float(
pyprob.distributions.Distribution.kl_divergence(
true_posterior,
Normal(posterior_with_thinning.mean,
posterior_with_thinning.stddev)))
util.eval_print('posterior_num_traces', 'posterior_mean',
'posterior_mean_correct', 'posterior_stddev',
'posterior_stddev_correct', 'kl_divergence',
'thinning_steps', 'posterior_with_thinning_num_traces',
'posterior_with_thinning_num_traces_correct',
'posterior_with_thinning_mean',
'posterior_with_thinning_stddev',
'kl_divergence_with_thinning')
self.assertEqual(posterior_with_thinning_num_traces,
posterior_with_thinning_num_traces_correct)
self.assertAlmostEqual(posterior_mean,
posterior_mean_correct,
places=0)
self.assertAlmostEqual(posterior_stddev,
posterior_stddev_correct,
places=0)
self.assertLess(kl_divergence, 0.25)
self.assertAlmostEqual(posterior_with_thinning_mean,
posterior_mean_correct,
places=0)
self.assertAlmostEqual(posterior_with_thinning_stddev,
posterior_stddev_correct,
places=0)
self.assertLess(kl_divergence_with_thinning, 0.25)
def test_inference_gum_marsaglia_replacement_posterior_importance_sampling_with_inference_network(
self):
samples = importance_sampling_samples
true_posterior = Normal(7.25, math.sqrt(1 / 1.2))
posterior_mean_correct = float(true_posterior.mean)
posterior_stddev_correct = float(true_posterior.stddev)
posterior_effective_sample_size_min = samples * 0.03
self._model.learn_inference_network(
num_traces=
importance_sampling_with_inference_network_training_traces,
observe_embeddings={
'obs0': {
'dim': 256,
'depth': 1
},
'obs1': {
'dim': 256,
'depth': 1
}
})
start = time.time()
posterior = self._model.posterior_distribution(
samples,
inference_engine=InferenceEngine.
IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK,
observe={
'obs0': 8,
'obs1': 9
})
add_importance_sampling_with_inference_network_duration(time.time() -
start)
posterior_mean = float(posterior.mean)
posterior_mean_unweighted = float(posterior.unweighted().mean)
posterior_stddev = float(posterior.stddev)
posterior_stddev_unweighted = float(posterior.unweighted().stddev)
posterior_effective_sample_size = float(
posterior.effective_sample_size)
kl_divergence = float(
pyprob.distributions.Distribution.kl_divergence(
true_posterior, Normal(posterior.mean, posterior.stddev)))
util.eval_print('samples', 'posterior_mean_unweighted',
'posterior_mean', 'posterior_mean_correct',
'posterior_stddev_unweighted', 'posterior_stddev',
'posterior_stddev_correct',
'posterior_effective_sample_size',
'posterior_effective_sample_size_min', 'kl_divergence')
add_importance_sampling_with_inference_network_kl_divergence(
kl_divergence)
self.assertAlmostEqual(posterior_mean,
posterior_mean_correct,
places=0)
self.assertAlmostEqual(posterior_stddev,
posterior_stddev_correct,
places=0)
self.assertGreater(posterior_effective_sample_size,
posterior_effective_sample_size_min)
self.assertLess(kl_divergence, 0.25)
