def test_inference_branching_random_walk_metropolis_hastings(self):
samples = importance_sampling_samples
posterior_correct = util.empirical_to_categorical(
self.true_posterior(), max_val=40)
start = time.time()
posterior = util.empirical_to_categorical(
self._model.posterior_distribution(
samples,
inference_engine=InferenceEngine.
RANDOM_WALK_METROPOLIS_HASTINGS,
observe={'obs': 6}),
max_val=40)
add_random_walk_metropolis_hastings_duration(time.time() - start)
posterior_probs = util.to_numpy(posterior._probs)
posterior_probs_correct = util.to_numpy(posterior_correct._probs)
kl_divergence = float(
pyprob.distributions.Distribution.kl_divergence(
posterior, posterior_correct))
util.eval_print('samples', 'posterior_probs',
'posterior_probs_correct', 'kl_divergence')
add_random_walk_metropolis_hastings_kl_divergence(kl_divergence)
self.assertLess(kl_divergence, 0.75)
def test_inference_remote_branching_importance_sampling(self):
samples = importance_sampling_samples
posterior_correct = util.empirical_to_categorical(self.true_posterior(), max_val=40)
start = time.time()
posterior = util.empirical_to_categorical(self._model.posterior_results(samples, observe={'obs': 6}), max_val=40)
add_importance_sampling_duration(time.time() - start)
posterior_probs = util.to_numpy(posterior._probs)
posterior_probs_correct = util.to_numpy(posterior_correct._probs)
kl_divergence = float(pyprob.distributions.Distribution.kl_divergence(posterior, posterior_correct))
util.eval_print('samples', 'posterior_probs', 'posterior_probs_correct', 'kl_divergence')
add_importance_sampling_kl_divergence(kl_divergence)
self.assertLess(kl_divergence, 0.75)
def test_model_remote_branching_importance_sampling(self):
observation = 6
posterior_correct = util.empirical_to_categorical(
self.true_posterior(observation), max_val=40)
posterior = util.empirical_to_categorical(
self._model.posterior_distribution(samples,
observation=observation),
max_val=40)
posterior_probs = util.to_numpy(posterior._probs[0])
posterior_probs_correct = util.to_numpy(posterior_correct._probs[0])
kl_divergence = float(
util.kl_divergence_categorical(posterior_correct, posterior))
util.debug('samples', 'posterior_probs', 'posterior_probs_correct',
'kl_divergence')
self.assertLess(kl_divergence, 0.25)
def test_model_remote_branching_random_walk_metropolis_hastings(self):
observation = 6
posterior_correct = util.empirical_to_categorical(
self.true_posterior(observation), max_val=40)
posterior = util.empirical_to_categorical(
self._model.posterior_distribution(
samples,
observation=observation,
inference_engine=pyprob.InferenceEngine.
RANDOM_WALK_METROPOLIS_HASTINGS),
max_val=40)
posterior_probs = util.to_numpy(posterior._probs[0])
posterior_probs_correct = util.to_numpy(posterior_correct._probs[0])
kl_divergence = float(
util.kl_divergence_categorical(posterior_correct, posterior))
util.debug('samples', 'posterior_probs', 'posterior_probs_correct',
'kl_divergence')
self.assertLess(kl_divergence, 0.25)
def test_inference_mini_captcha_posterior_importance_sampling(self):
samples = int(importance_sampling_samples / len(self._model._alphabet))
test_letters = self._model._alphabet
start = time.time()
posteriors = []
map_estimates = []
for i in range(len(self._model._alphabet)):
posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING, observe={'query_image': self._test_images[i]})
posteriors.append(posterior)
map_estimates.append(self._model._alphabet[int(posterior.mode)])
add_importance_sampling_duration(time.time() - start)
accuracy = sum([1 if map_estimates[i] == test_letters[i] else 0 for i in range(len(test_letters))])/len(test_letters)
kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(util.empirical_to_categorical(p, max_val=len(self._model._alphabet)-1), tp) for (p, tp) in zip(posteriors, self._true_posteriors)]))
util.eval_print('samples', 'test_letters', 'map_estimates', 'accuracy', 'kl_divergence')
add_importance_sampling_kl_divergence(kl_divergence)
self.assertGreater(accuracy, 0.9)
self.assertLess(kl_divergence, 0.25)
def test_inference_mini_captcha_posterior_random_walk_metropolis_hastings(self):
samples = int(random_walk_metropolis_hastings_samples / len(self._model._alphabet))
burn_in = int(random_walk_metropolis_hastings_burn_in / len(self._model._alphabet))
test_letters = self._model._alphabet
start = time.time()
posteriors = []
map_estimates = []
for i in range(len(self._model._alphabet)):
posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS, observe={'query_image': self._test_images[i]})[burn_in:]
posteriors.append(posterior)
map_estimates.append(self._model._alphabet[int(posterior.combine_duplicates().mode)])
add_random_walk_metropolis_hastings_duration(time.time() - start)
accuracy = sum([1 if map_estimates[i] == test_letters[i] else 0 for i in range(len(test_letters))])/len(test_letters)
kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(util.empirical_to_categorical(p, max_val=len(self._model._alphabet)-1), tp) for (p, tp) in zip(posteriors, self._true_posteriors)]))
util.eval_print('samples', 'test_letters', 'map_estimates', 'accuracy', 'kl_divergence')
add_random_walk_metropolis_hastings_kl_divergence(kl_divergence)
self.assertGreater(accuracy, 0.9)
self.assertLess(kl_divergence, 0.25)
def test_inference_mini_captcha_posterior_importance_sampling_with_inference_network(self):
samples = int(importance_sampling_with_inference_network_samples / len(self._model._alphabet))
test_letters = self._model._alphabet
self._model.learn_inference_network(num_traces=importance_sampling_with_inference_network_training_traces, observe_embeddings={'query_image': {'dim': 32, 'reshape': [1, 28, 28], 'embedding': ObserveEmbedding.CNN2D5C}})
start = time.time()
posteriors = []
map_estimates = []
for i in range(len(self._model._alphabet)):
posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK, observe={'query_image': self._test_images[i]})
posteriors.append(posterior)
map_estimates.append(self._model._alphabet[int(posterior.mode)])
add_importance_sampling_with_inference_network_duration(time.time() - start)
accuracy = sum([1 if map_estimates[i] == test_letters[i] else 0 for i in range(len(test_letters))])/len(test_letters)
kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(util.empirical_to_categorical(p, max_val=len(self._model._alphabet)-1), tp) for (p, tp) in zip(posteriors, self._true_posteriors)]))
util.eval_print('samples', 'test_letters', 'map_estimates', 'accuracy', 'kl_divergence')
add_importance_sampling_with_inference_network_kl_divergence(kl_divergence)
self.assertGreater(accuracy, 0.9)
self.assertLess(kl_divergence, 0.25)
