def test_inference_mini_captcha_posterior_importance_sampling(self): samples = int(importance_sampling_samples / len(self._model._alphabet)) test_letters = self._model._alphabet start = time.time() posteriors = [] map_estimates = [] for i in range(len(self._model._alphabet)): posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING, observe={'query_image': self._test_images[i]}) posteriors.append(posterior) map_estimates.append(self._model._alphabet[int(posterior.mode)]) add_importance_sampling_duration(time.time() - start) accuracy = sum([1 if map_estimates[i] == test_letters[i] else 0 for i in range(len(test_letters))])/len(test_letters) kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(util.empirical_to_categorical(p, max_val=len(self._model._alphabet)-1), tp) for (p, tp) in zip(posteriors, self._true_posteriors)])) util.eval_print('samples', 'test_letters', 'map_estimates', 'accuracy', 'kl_divergence') add_importance_sampling_kl_divergence(kl_divergence) self.assertGreater(accuracy, 0.9) self.assertLess(kl_divergence, 0.25)
def test_inference_gum_posterior_random_walk_metropolis_hastings(self): samples = random_walk_metropolis_hastings_samples burn_in = random_walk_metropolis_hastings_burn_in true_posterior = Normal(7.25, math.sqrt(1/1.2)) posterior_mean_correct = float(true_posterior.mean) posterior_stddev_correct = float(true_posterior.stddev) start = time.time() posterior = self._model.posterior_results(samples, inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS, observe={'obs0': 8, 'obs1': 9})[burn_in:] add_random_walk_metropolis_hastings_duration(time.time() - start) posterior_mean = float(posterior.mean) posterior_stddev = float(posterior.stddev) kl_divergence = float(pyprob.distributions.Distribution.kl_divergence(true_posterior, Normal(posterior.mean, posterior.stddev))) util.eval_print('samples', 'burn_in', 'posterior_mean', 'posterior_mean_correct', 'posterior_stddev', 'posterior_stddev_correct', 'kl_divergence') add_random_walk_metropolis_hastings_kl_divergence(kl_divergence) self.assertAlmostEqual(posterior_mean, posterior_mean_correct, places=0) self.assertAlmostEqual(posterior_stddev, posterior_stddev_correct, places=0) self.assertLess(kl_divergence, 0.25)
def test_model_remote_set_defaults_and_addresses_addresses(self): addresses_controlled_correct = [ '[forward()+0x3a9]__Normal__1', '[forward()+0x3a9]__Normal__2', '[forward()+0xc88]__Normal__2' ] addresses_all_correct = [ '[forward()+0x3a9]__Normal__1', '[forward()+0x3a9]__Normal__2', '[forward()+0xc88]__Normal__1', '[forward()+0xc88]__Normal__2', '[forward()+0x1573]__Normal__1', '[forward()+0x1573]__Normal__2', '[forward()+0x1ee5]__Normal__1' ] trace = next(self._model._trace_generator()) addresses_controlled = [s.address for s in trace.variables_controlled] addresses_all = [s.address for s in trace.variables] util.eval_print('addresses_controlled', 'addresses_controlled_correct', 'addresses_all', 'addresses_all_correct') self.assertEqual(addresses_controlled, addresses_controlled_correct) self.assertEqual(addresses_all, addresses_all_correct)
def test_model_prior_on_disk(self): file_name = os.path.join(tempfile.mkdtemp(), str(uuid.uuid4())) num_traces = 1000 prior_mean_correct = 1 prior_stddev_correct = math.sqrt(5) prior_length_correct = 2 * num_traces prior = self._model.prior_distribution(num_traces, file_name=file_name) prior.close() prior = self._model.prior_distribution(num_traces, file_name=file_name) # prior.close() prior_length = prior.length prior_mean = float(prior.mean) prior_stddev = float(prior.stddev) util.eval_print('num_traces', 'prior_mean', 'prior_mean_correct', 'prior_stddev', 'prior_stddev_correct', 'prior_length', 'prior_length_correct') self.assertAlmostEqual(prior_mean, prior_mean_correct, places=0) self.assertAlmostEqual(prior_stddev, prior_stddev_correct, places=0) self.assertEqual(prior_length, prior_length_correct)
def test_posterior_importance_sampling_with_inference_network(self): trace_addresses_controlled_correct = [ '34__forward__x__Uniform__2', '48__forward__y__Uniform__2' ] trace_addresses_correct = [ '34__forward__x__Uniform__1', '48__forward__y__Uniform__1', '34__forward__x__Uniform__2', '48__forward__y__Uniform__2', '92__forward__?__Normal__1', '106__forward__?__Normal__1' ] self._model.learn_inference_network(num_traces=10, observe_embeddings={ 'obs0': { 'dim': 128, 'depth': 6 }, 'obs1': { 'dim': 128, 'depth': 6 } }) posterior = self._model.posterior( 1, inference_engine=InferenceEngine. IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK, observe={ 'obs0': 8, 'obs1': 9 }) trace = posterior[0] trace_addresses_controlled = [ v.address for v in trace.variables_controlled ] trace_addresses = [v.address for v in trace.variables] util.eval_print('trace', 'trace_addresses_controlled', 'trace_addresses') self.assertEqual(trace_addresses_controlled, trace_addresses_controlled_correct) self.assertEqual(trace_addresses, trace_addresses_correct)
def test_inference_hmm_posterior_importance_sampling(self): samples = importance_sampling_samples observation = { 'obs{}'.format(i): self._observation[i] for i in range(len(self._observation)) } posterior_mean_correct = self._posterior_mean_correct posterior_effective_sample_size_min = samples * 0.0015 start = time.time() posterior = self._model.posterior_distribution(samples, observe=observation) add_importance_sampling_duration(time.time() - start) posterior_mean_unweighted = posterior.unweighted().mean posterior_mean = posterior.mean posterior_effective_sample_size = float( posterior.effective_sample_size) print(posterior[0]) l2_distance = float( F.pairwise_distance(posterior_mean, posterior_mean_correct).sum()) kl_divergence = float( sum([ pyprob.distributions.Distribution.kl_divergence( Categorical(i + util._epsilon), Categorical(j + util._epsilon)) for (i, j) in zip(posterior_mean, posterior_mean_correct) ])) util.eval_print('samples', 'posterior_mean_unweighted', 'posterior_mean', 'posterior_mean_correct', 'posterior_effective_sample_size', 'posterior_effective_sample_size_min', 'l2_distance', 'kl_divergence') add_importance_sampling_kl_divergence(kl_divergence) self.assertGreater(posterior_effective_sample_size, posterior_effective_sample_size_min) self.assertLess(l2_distance, 3) self.assertLess(kl_divergence, 1)
def test_inference_branching_importance_sampling(self): samples = importance_sampling_samples posterior_correct = util.empirical_to_categorical( self.true_posterior(), max_val=40) start = time.time() posterior = util.empirical_to_categorical( self._model.posterior_distribution(samples, observe={'obs': 6}), max_val=40) add_importance_sampling_duration(time.time() - start) posterior_probs = util.to_numpy(posterior._probs) posterior_probs_correct = util.to_numpy(posterior_correct._probs) kl_divergence = float( pyprob.distributions.Distribution.kl_divergence( posterior, posterior_correct)) util.eval_print('samples', 'posterior_probs', 'posterior_probs_correct', 'kl_divergence') add_importance_sampling_kl_divergence(kl_divergence) self.assertLess(kl_divergence, 0.75)
def test_prior_for_inference_network(self): trace_addresses_accepted_correct = [ '38__forward__x__Uniform__replaced', '48__forward__y__Uniform__replaced' ] trace_addresses_correct = [ '38__forward__x__Uniform__replaced', '48__forward__y__Uniform__replaced', '100__forward__?__Normal__1', '114__forward__?__Normal__1' ] trace = OnlineDataset(model=self._model).__getitem__(0)[0] trace_addresses_accepted = [ v['address'] for v in trace if v['control'] and v['accepted'] ] trace_addresses = [v['address'] for v in trace] util.eval_print('trace', 'trace_addresses_accepted', 'trace_addresses') self.assertEqual(trace_addresses_accepted, trace_addresses_accepted_correct) self.assertEqual(trace_addresses, trace_addresses_correct)
def test_inference_mini_captcha_posterior_random_walk_metropolis_hastings(self): samples = int(random_walk_metropolis_hastings_samples / len(self._model._alphabet)) burn_in = int(random_walk_metropolis_hastings_burn_in / len(self._model._alphabet)) test_letters = self._model._alphabet start = time.time() posteriors = [] map_estimates = [] for i in range(len(self._model._alphabet)): posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS, observe={'query_image': self._test_images[i]})[burn_in:] posteriors.append(posterior) map_estimates.append(self._model._alphabet[int(posterior.combine_duplicates().mode)]) add_random_walk_metropolis_hastings_duration(time.time() - start) accuracy = sum([1 if map_estimates[i] == test_letters[i] else 0 for i in range(len(test_letters))])/len(test_letters) kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(util.empirical_to_categorical(p, max_val=len(self._model._alphabet)-1), tp) for (p, tp) in zip(posteriors, self._true_posteriors)])) util.eval_print('samples', 'test_letters', 'map_estimates', 'accuracy', 'kl_divergence') add_random_walk_metropolis_hastings_kl_divergence(kl_divergence) self.assertGreater(accuracy, 0.9) self.assertLess(kl_divergence, 0.25)
def test_observation_style2_gum_posterior_importance_sampling(self): samples = importance_sampling_samples true_posterior = Normal(7.25, math.sqrt(1/1.2)) posterior_mean_correct = float(true_posterior.mean) posterior_stddev_correct = float(true_posterior.stddev) prior_mean_correct = 1. prior_stddev_correct = math.sqrt(5) posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING, observe={'obs0': 8, 'obs1': 9}) posterior_mean = float(posterior.mean) posterior_mean_unweighted = float(posterior.unweighted().mean) posterior_stddev = float(posterior.stddev) posterior_stddev_unweighted = float(posterior.unweighted().stddev) kl_divergence = float(pyprob.distributions.Distribution.kl_divergence(true_posterior, Normal(posterior.mean, posterior.stddev))) util.eval_print('samples', 'prior_mean_correct', 'posterior_mean_unweighted', 'posterior_mean', 'posterior_mean_correct', 'prior_stddev_correct', 'posterior_stddev_unweighted', 'posterior_stddev', 'posterior_stddev_correct', 'kl_divergence') self.assertAlmostEqual(posterior_mean_unweighted, prior_mean_correct, places=0) self.assertAlmostEqual(posterior_stddev_unweighted, prior_stddev_correct, places=0) self.assertAlmostEqual(posterior_mean, posterior_mean_correct, places=0) self.assertAlmostEqual(posterior_stddev, posterior_stddev_correct, places=0) self.assertLess(kl_divergence, 0.25)
def test_save_offline_dataset(self): dataset_dir = tempfile.mkdtemp() num_traces_correct = 20 num_traces_per_file_correct = 5 num_files_correct = 4 self._model.save_dataset(dataset_dir=dataset_dir, num_traces=num_traces_correct, num_traces_per_file=num_traces_per_file_correct) files = sorted(glob(os.path.join(dataset_dir, 'pyprob_traces_*'))) num_files = len(files) dataset = OfflineDataset(dataset_dir) hashes = dataset._hashes indices = dataset._sorted_indices sorted_on_disk = util.is_sorted(indices) num_traces = len(dataset) num_traces_per_file = num_traces / num_files shutil.rmtree(dataset_dir) util.eval_print('dataset_dir', 'num_traces', 'num_traces_correct', 'num_traces_per_file', 'num_traces_per_file_correct', 'files', 'num_files', 'num_files_correct', 'hashes', 'indices', 'sorted_on_disk') self.assertEqual(num_files, num_files_correct) self.assertEqual(num_traces, num_traces_correct) self.assertEqual(num_traces_per_file, num_traces_per_file_correct) self.assertFalse(sorted_on_disk)
def test_inference_mini_captcha_posterior_importance_sampling_with_inference_network(self): samples = int(importance_sampling_with_inference_network_samples / len(self._model._alphabet)) test_letters = self._model._alphabet self._model.learn_inference_network(num_traces=importance_sampling_with_inference_network_training_traces, observe_embeddings={'query_image': {'dim': 32, 'reshape': [1, 28, 28], 'embedding': ObserveEmbedding.CNN2D5C}}) start = time.time() posteriors = [] map_estimates = [] for i in range(len(self._model._alphabet)): posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK, observe={'query_image': self._test_images[i]}) posteriors.append(posterior) map_estimates.append(self._model._alphabet[int(posterior.mode)]) add_importance_sampling_with_inference_network_duration(time.time() - start) accuracy = sum([1 if map_estimates[i] == test_letters[i] else 0 for i in range(len(test_letters))])/len(test_letters) kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(util.empirical_to_categorical(p, max_val=len(self._model._alphabet)-1), tp) for (p, tp) in zip(posteriors, self._true_posteriors)])) util.eval_print('samples', 'test_letters', 'map_estimates', 'accuracy', 'kl_divergence') add_importance_sampling_with_inference_network_kl_divergence(kl_divergence) self.assertGreater(accuracy, 0.9) self.assertLess(kl_divergence, 0.25)
def test_prior_inflation(self): samples = 5000 categorical_prior_mean_correct = 1.7 categorical_prior_stddev_correct = 0.640312 categorical_prior_inflated_mean_correct = 1 categorical_prior_inflated_stddev_correct = 0.816497 normal_prior_mean_correct = 5 normal_prior_stddev_correct = 2 normal_prior_inflated_mean_correct = 5 normal_prior_inflated_stddev_correct = normal_prior_stddev_correct * 3 prior = self._model.prior_distribution(samples, prior_inflation=pyprob.PriorInflation.DISABLED) categorical_prior = prior.map(lambda x: x[0]) categorical_prior_mean = float(categorical_prior.mean) categorical_prior_stddev = float(categorical_prior.stddev) normal_prior = prior.map(lambda x: x[1]) normal_prior_mean = float(normal_prior.mean) normal_prior_stddev = float(normal_prior.stddev) prior_inflated = self._model.prior_distribution(samples, prior_inflation=pyprob.PriorInflation.ENABLED) categorical_prior_inflated = prior_inflated.map(lambda x: x[0]) categorical_prior_inflated_mean = float(categorical_prior_inflated.mean) categorical_prior_inflated_stddev = float(categorical_prior_inflated.stddev) normal_prior_inflated = prior_inflated.map(lambda x: x[1]) normal_prior_inflated_mean = float(normal_prior_inflated.mean) normal_prior_inflated_stddev = float(normal_prior_inflated.stddev) util.eval_print('samples', 'categorical_prior_mean', 'categorical_prior_mean_correct', 'categorical_prior_stddev', 'categorical_prior_stddev_correct', 'categorical_prior_inflated_mean', 'categorical_prior_inflated_mean_correct', 'categorical_prior_inflated_stddev', 'categorical_prior_inflated_stddev_correct', 'normal_prior_mean', 'normal_prior_mean_correct', 'normal_prior_stddev', 'normal_prior_stddev_correct', 'normal_prior_inflated_mean', 'normal_prior_inflated_mean_correct', 'normal_prior_inflated_stddev', 'normal_prior_inflated_stddev_correct') self.assertAlmostEqual(categorical_prior_mean, categorical_prior_mean_correct, places=0) self.assertAlmostEqual(categorical_prior_stddev, categorical_prior_stddev_correct, places=0) self.assertAlmostEqual(categorical_prior_inflated_mean, categorical_prior_inflated_mean_correct, places=0) self.assertAlmostEqual(categorical_prior_inflated_stddev, categorical_prior_inflated_stddev_correct, places=0) self.assertAlmostEqual(normal_prior_mean, normal_prior_mean_correct, places=0) self.assertAlmostEqual(normal_prior_stddev, normal_prior_stddev_correct, places=0) self.assertAlmostEqual(normal_prior_inflated_mean, normal_prior_inflated_mean_correct, places=0) self.assertAlmostEqual(normal_prior_inflated_stddev, normal_prior_inflated_stddev_correct, places=0)
def test_inference_hmm_posterior_importance_sampling_with_inference_network(self): samples = importance_sampling_with_inference_network_samples observation = {'obs{}'.format(i): self._observation[i] for i in range(len(self._observation))} posterior_mean_correct = self._posterior_mean_correct posterior_effective_sample_size_min = samples * 0.03 self._model.learn_inference_network(num_traces=importance_sampling_with_inference_network_training_traces, observe_embeddings={'obs{}'.format(i): {'depth': 2, 'dim': 16} for i in range(len(observation))}) start = time.time() posterior = self._model.posterior_distribution(samples, inference_engine=InferenceEngine.IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK, observe=observation) add_importance_sampling_with_inference_network_duration(time.time() - start) posterior_mean_unweighted = posterior.unweighted().mean posterior_mean = posterior.mean posterior_effective_sample_size = float(posterior.effective_sample_size) l2_distance = float(F.pairwise_distance(posterior_mean, posterior_mean_correct).sum()) kl_divergence = float(sum([pyprob.distributions.Distribution.kl_divergence(Categorical(i + util._epsilon), Categorical(j + util._epsilon)) for (i, j) in zip(posterior_mean, posterior_mean_correct)])) util.eval_print('samples', 'posterior_mean_unweighted', 'posterior_mean', 'posterior_mean_correct', 'posterior_effective_sample_size', 'posterior_effective_sample_size_min', 'l2_distance', 'kl_divergence') add_importance_sampling_with_inference_network_kl_divergence(kl_divergence) self.assertGreater(posterior_effective_sample_size, posterior_effective_sample_size_min) self.assertLess(l2_distance, 3) self.assertLess(kl_divergence, 1)
def test_trace_prior_for_inference_network(self): trace_addresses_controlled_correct = [ '30__forward__x__Uniform__1', '40__forward__y__Uniform__1', '30__forward__x__Uniform__2', '40__forward__y__Uniform__2' ] trace_addresses_correct = [ '30__forward__x__Uniform__1', '40__forward__y__Uniform__1', '30__forward__x__Uniform__2', '40__forward__y__Uniform__2', '84__forward__?__Normal__1', '98__forward__?__Normal__1' ] trace = OnlineDataset(model=self._model)[0] trace_addresses_controlled = [ v.address for v in trace.variables_controlled ] trace_addresses = [v.address for v in trace.variables] util.eval_print('trace', 'trace_addresses_controlled', 'trace_addresses_controlled_correct', 'trace_addresses', 'trace_addresses_correct') self.assertEqual(trace_addresses_controlled_correct, trace_addresses_controlled) self.assertEqual(trace_addresses_correct, trace_addresses)
def test_inference_branching_random_walk_metropolis_hastings(self): samples = importance_sampling_samples posterior_correct = util.empirical_to_categorical( self.true_posterior(), max_val=40) start = time.time() posterior = util.empirical_to_categorical(self._model.posterior_return( samples, inference_engine=InferenceEngine.RANDOM_WALK_METROPOLIS_HASTINGS, observe={'obs': 6}), max_val=40) add_random_walk_metropolis_hastings_duration(time.time() - start) posterior_probs = util.to_numpy(posterior._probs) posterior_probs_correct = util.to_numpy(posterior_correct._probs) kl_divergence = float( pyprob.distributions.Distribution.kl_divergence( posterior, posterior_correct)) util.eval_print('samples', 'posterior_probs', 'posterior_probs_correct', 'kl_divergence') add_random_walk_metropolis_hastings_kl_divergence(kl_divergence) self.assertLess(kl_divergence, 0.75)
def test_random_seed(self): samples = 10 stochastic_samples = [] for i in range(samples): pyprob.set_random_seed(None) dist = pyprob.distributions.Normal(0, 1) sample = dist.sample() stochastic_samples.append(float(sample)) deterministic_samples = [] for i in range(samples): pyprob.set_random_seed(123) dist = pyprob.distributions.Normal(0, 1) sample = dist.sample() deterministic_samples.append(float(sample)) util.eval_print('samples', 'stochastic_samples', 'deterministic_samples') self.assertTrue(not all(sample == stochastic_samples[0] for sample in stochastic_samples)) self.assertTrue( all(sample == deterministic_samples[0] for sample in deterministic_samples))
def test_distributions_remote(self): num_samples = 4000 prior_normal_mean_correct = Normal(1.75, 0.5).mean prior_uniform_mean_correct = Uniform(1.2, 2.5).mean prior_categorical_mean_correct = 1. # Categorical([0.1, 0.5, 0.4]) prior_poisson_mean_correct = Poisson(4.0).mean prior_bernoulli_mean_correct = Bernoulli(0.2).mean prior_beta_mean_correct = Beta(1.2, 2.5).mean prior_exponential_mean_correct = Exponential(2.2).mean prior_gamma_mean_correct = Gamma(0.5, 1.2).mean prior_log_normal_mean_correct = LogNormal(0.5, 0.2).mean prior_binomial_mean_correct = Binomial(10, 0.72).mean prior_weibull_mean_correct = Weibull(1.1, 0.6).mean prior = self._model.prior(num_samples) prior_normal = prior.map( lambda trace: trace.named_variables['normal'].value) prior_uniform = prior.map( lambda trace: trace.named_variables['uniform'].value) prior_categorical = prior.map( lambda trace: trace.named_variables['categorical'].value) prior_poisson = prior.map( lambda trace: trace.named_variables['poisson'].value) prior_bernoulli = prior.map( lambda trace: trace.named_variables['bernoulli'].value) prior_beta = prior.map( lambda trace: trace.named_variables['beta'].value) prior_exponential = prior.map( lambda trace: trace.named_variables['exponential'].value) prior_gamma = prior.map( lambda trace: trace.named_variables['gamma'].value) prior_log_normal = prior.map( lambda trace: trace.named_variables['log_normal'].value) prior_binomial = prior.map( lambda trace: trace.named_variables['binomial'].value) prior_weibull = prior.map( lambda trace: trace.named_variables['weibull'].value) prior_normal_mean = util.to_numpy(prior_normal.mean) prior_uniform_mean = util.to_numpy(prior_uniform.mean) prior_categorical_mean = util.to_numpy(int(prior_categorical.mean)) prior_poisson_mean = util.to_numpy(prior_poisson.mean) prior_bernoulli_mean = util.to_numpy(prior_bernoulli.mean) prior_beta_mean = util.to_numpy(prior_beta.mean) prior_exponential_mean = util.to_numpy(prior_exponential.mean) prior_gamma_mean = util.to_numpy(prior_gamma.mean) prior_log_normal_mean = util.to_numpy(prior_log_normal.mean) prior_binomial_mean = util.to_numpy(prior_binomial.mean) prior_weibull_mean = util.to_numpy(prior_weibull.mean) util.eval_print('num_samples', 'prior_normal_mean', 'prior_normal_mean_correct', 'prior_uniform_mean', 'prior_uniform_mean_correct', 'prior_categorical_mean', 'prior_categorical_mean_correct', 'prior_poisson_mean', 'prior_poisson_mean_correct', 'prior_bernoulli_mean', 'prior_bernoulli_mean_correct', 'prior_beta_mean', 'prior_beta_mean_correct', 'prior_exponential_mean', 'prior_exponential_mean_correct', 'prior_gamma_mean', 'prior_gamma_mean_correct', 'prior_log_normal_mean', 'prior_log_normal_mean_correct', 'prior_binomial_mean', 'prior_binomial_mean_correct', 'prior_weibull_mean', 'prior_weibull_mean_correct') self.assertTrue( np.allclose(prior_normal_mean, prior_normal_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_uniform_mean, prior_uniform_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_categorical_mean, prior_categorical_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_poisson_mean, prior_poisson_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_bernoulli_mean, prior_bernoulli_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_beta_mean, prior_beta_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_exponential_mean, prior_exponential_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_gamma_mean, prior_gamma_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_log_normal_mean, prior_log_normal_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_binomial_mean, prior_binomial_mean_correct, atol=0.1)) self.assertTrue( np.allclose(prior_weibull_mean, prior_weibull_mean_correct, atol=0.1))
def test_state_address(self): address = self._sample_address() address_correct = '4__test_state_address__address' util.eval_print('address', 'address_correct') self.assertEqual(address, address_correct)
def test_train_offline_adam_larc_lr_poly2(self): dataset_dir = tempfile.mkdtemp() dataset_num_traces = 128 dataset_num_traces_per_file = 32 file_name = os.path.join(tempfile.mkdtemp(), str(uuid.uuid4())) file_name_2 = os.path.join(tempfile.mkdtemp(), str(uuid.uuid4())) file_name_3 = os.path.join(tempfile.mkdtemp(), str(uuid.uuid4())) num_traces_end = 256 batch_size = 16 learning_rate_init_correct = 0.1 learning_rate_end_correct = 0.0025 print('Saving dataset\n') self._model.save_dataset( dataset_dir=dataset_dir, num_traces=dataset_num_traces, num_traces_per_file=dataset_num_traces_per_file) self._model.reset_inference_network() print('Training\n') self._model.learn_inference_network( dataset_dir=dataset_dir, num_traces=num_traces_end / 2, num_traces_end=num_traces_end, batch_size=batch_size, observe_embeddings={ 'obs0': { 'dim': 16 }, 'obs1': { 'dim': 16 } }, optimizer_type=Optimizer.ADAM_LARC, learning_rate_scheduler_type=LearningRateScheduler.POLY2, learning_rate_init=learning_rate_init_correct, learning_rate_end=learning_rate_end_correct, log_file_name=file_name_2) print('Saving\n') # print(self._model._inference_network._optimizer) optimizer_state_step_before_save = list( self._model._inference_network._optimizer.state_dict() ['state'].values())[0]['step'] self._model.save_inference_network(file_name) print('\nlog_file_name contents') with open(file_name_2) as file: for line in file: print(line, end='') self._model.reset_inference_network() print('Loading\n') self._model.load_inference_network(file_name) # print(self._model._inference_network._optimizer) optimizer_state_step_after_load = list( self._model._inference_network._optimizer.state_dict() ['state'].values())[0]['step'] print('Training\n') self._model.learn_inference_network( dataset_dir=dataset_dir, num_traces=num_traces_end / 2, num_traces_end=num_traces_end, batch_size=batch_size, observe_embeddings={ 'obs0': { 'dim': 16 }, 'obs1': { 'dim': 16 } }, optimizer_type=Optimizer.ADAM_LARC, learning_rate_scheduler_type=LearningRateScheduler.POLY2, learning_rate_init=learning_rate_init_correct, learning_rate_end=learning_rate_end_correct, log_file_name=file_name_3) learning_rate_end = self._model._inference_network._optimizer.param_groups[ 0]['lr'] print('\nlog_file_name contents') with open(file_name_3) as file: for line in file: print(line, end='') os.remove(file_name) os.remove(file_name_2) os.remove(file_name_3) shutil.rmtree(dataset_dir) util.eval_print('dataset_dir', 'dataset_num_traces', 'dataset_num_traces_per_file', 'file_name', 'file_name_2', 'file_name_3', 'num_traces_end', 'batch_size', 'learning_rate_init_correct', 'optimizer_state_step_before_save', 'optimizer_state_step_after_load', 'learning_rate_end', 'learning_rate_end_correct') # util.eval_print('file_name', 'file_name_2', 'file_name_3', 'num_traces_end', 'batch_size', 'learning_rate_init_correct', 'learning_rate_end', 'learning_rate_end_correct') self.assertEqual(optimizer_state_step_before_save, optimizer_state_step_after_load) self.assertAlmostEqual(learning_rate_end, learning_rate_end_correct, delta=learning_rate_end_correct)
def test_model_lmh_posterior_with_online_thinning(self): thinning_steps = 10 posterior_num_traces = 4000 posterior_with_thinning_num_traces_correct = posterior_num_traces / thinning_steps true_posterior = Normal(7.25, math.sqrt(1 / 1.2)) posterior_mean_correct = float(true_posterior.mean) posterior_stddev_correct = float(true_posterior.stddev) posterior = self._model.posterior_results( num_traces=posterior_num_traces, inference_engine=InferenceEngine.LIGHTWEIGHT_METROPOLIS_HASTINGS, observe={ 'obs0': 8, 'obs1': 9 }) posterior_num_traces = posterior.length posterior_mean = float(posterior.mean) posterior_stddev = float(posterior.stddev) kl_divergence = float( pyprob.distributions.Distribution.kl_divergence( true_posterior, Normal(posterior.mean, posterior.stddev))) posterior_with_thinning = self._model.posterior_results( num_traces=posterior_num_traces, inference_engine=InferenceEngine.LIGHTWEIGHT_METROPOLIS_HASTINGS, observe={ 'obs0': 8, 'obs1': 9 }, thinning_steps=thinning_steps) posterior_with_thinning_num_traces = posterior_with_thinning.length posterior_with_thinning_mean = float(posterior_with_thinning.mean) posterior_with_thinning_stddev = float(posterior_with_thinning.stddev) kl_divergence_with_thinning = float( pyprob.distributions.Distribution.kl_divergence( true_posterior, Normal(posterior_with_thinning.mean, posterior_with_thinning.stddev))) util.eval_print('posterior_num_traces', 'posterior_mean', 'posterior_mean_correct', 'posterior_stddev', 'posterior_stddev_correct', 'kl_divergence', 'thinning_steps', 'posterior_with_thinning_num_traces', 'posterior_with_thinning_num_traces_correct', 'posterior_with_thinning_mean', 'posterior_with_thinning_stddev', 'kl_divergence_with_thinning') self.assertEqual(posterior_with_thinning_num_traces, posterior_with_thinning_num_traces_correct) self.assertAlmostEqual(posterior_mean, posterior_mean_correct, places=0) self.assertAlmostEqual(posterior_stddev, posterior_stddev_correct, places=0) self.assertLess(kl_divergence, 0.25) self.assertAlmostEqual(posterior_with_thinning_mean, posterior_mean_correct, places=0) self.assertAlmostEqual(posterior_with_thinning_stddev, posterior_stddev_correct, places=0) self.assertLess(kl_divergence_with_thinning, 0.25)
def test_inference_gum_marsaglia_replacement_posterior_importance_sampling_with_inference_network( self): samples = importance_sampling_samples true_posterior = Normal(7.25, math.sqrt(1 / 1.2)) posterior_mean_correct = float(true_posterior.mean) posterior_stddev_correct = float(true_posterior.stddev) posterior_effective_sample_size_min = samples * 0.03 self._model.learn_inference_network( num_traces= importance_sampling_with_inference_network_training_traces, observe_embeddings={ 'obs0': { 'dim': 256, 'depth': 1 }, 'obs1': { 'dim': 256, 'depth': 1 } }) start = time.time() posterior = self._model.posterior_distribution( samples, inference_engine=InferenceEngine. IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK, observe={ 'obs0': 8, 'obs1': 9 }) add_importance_sampling_with_inference_network_duration(time.time() - start) posterior_mean = float(posterior.mean) posterior_mean_unweighted = float(posterior.unweighted().mean) posterior_stddev = float(posterior.stddev) posterior_stddev_unweighted = float(posterior.unweighted().stddev) posterior_effective_sample_size = float( posterior.effective_sample_size) kl_divergence = float( pyprob.distributions.Distribution.kl_divergence( true_posterior, Normal(posterior.mean, posterior.stddev))) util.eval_print('samples', 'posterior_mean_unweighted', 'posterior_mean', 'posterior_mean_correct', 'posterior_stddev_unweighted', 'posterior_stddev', 'posterior_stddev_correct', 'posterior_effective_sample_size', 'posterior_effective_sample_size_min', 'kl_divergence') add_importance_sampling_with_inference_network_kl_divergence( kl_divergence) self.assertAlmostEqual(posterior_mean, posterior_mean_correct, places=0) self.assertAlmostEqual(posterior_stddev, posterior_stddev_correct, places=0) self.assertGreater(posterior_effective_sample_size, posterior_effective_sample_size_min) self.assertLess(kl_divergence, 0.25)