Qx.append(NormalVariable(BF.sigmoid(Qlambda[t])*new_mu + (1 - BF.sigmoid(Qlambda[t]))*Qx_mean[t], np.sqrt(dt) * driving_noise, x_names[t], learnable=True)) variational_posterior = ProbabilisticModel(Qx) AR_model.set_posterior_model(variational_posterior) # Inference # inference.perform_inference(AR_model, number_iterations=N_itr, number_samples=N_smpl, optimizer=optimizer, lr=lr) loss_list1 = AR_model.diagnostics["loss curve"] # ELBO ELBO1.append(float(AR_model.estimate_log_model_evidence(N_ELBO_smpl).detach().numpy())) print("PE {}".format(ELBO1[-1])) # Mean-field variational distribution # Qx = [NormalVariable(0., 1., 'x0', learnable=True)] for t in range(1, T): Qx.append(NormalVariable(0, 2., x_names[t], learnable=True)) variational_posterior = ProbabilisticModel(Qx) AR_model.set_posterior_model(variational_posterior) # Inference # inference.perform_inference(AR_model, number_iterations=N_itr, number_samples=N_smpl, optimizer=optimizer,
variational_posterior = ProbabilisticModel(Qx + Qh + Qz) AR_model.set_posterior_model(variational_posterior) # Inference # inference.perform_inference(AR_model, number_iterations=N_itr, number_samples=N_smpl, optimizer=optimizer, lr=lr) loss_list1 = AR_model.diagnostics["loss curve"] # ELBO ELBO1.append( float( AR_model.estimate_log_model_evidence( N_ELBO_smpl).detach().numpy())) print("PE {}".format(ELBO1[-1])) # Mean field Qx = [NormalVariable(0., 1., 'x0', learnable=True)] Qh = [NormalVariable(0., 1., 'h0', learnable=True)] Qz = [NormalVariable(0., 1., 'z0', learnable=True)] for t in range(1, T): Qx.append( NormalVariable(0., driving_noise, x_names[t], learnable=True)) Qh.append( NormalVariable(0., driving_noise, h_names[t], learnable=True)) Qz.append( NormalVariable(0., driving_noise, z_names[t], learnable=True))
model.set_posterior_model(ProbabilisticModel(Qpeople_means + Qgroup_means)) # Inference # N_itr = 300 N_smpl = 50 optimizer = "SGD" lr = 0.00001 inference.perform_inference(model, number_iterations=N_itr, number_samples=N_smpl, optimizer=optimizer, lr=lr) loss_list2 = model.diagnostics["loss curve"] N_ELBO = 1000 ELBO2 = model.estimate_log_model_evidence(1000) # Structured NN distribution # hidden_size = 5 latent_size = 5 out_size = N_groups + N_people Qepsilon = Normal(np.zeros((latent_size, 1)), np.ones((latent_size, )), 'epsilon', learnable=True) W1 = RootVariable(np.random.normal(0, 0.1, (hidden_size, latent_size)), "W1", learnable=True) W2 = RootVariable(np.random.normal(0, 0.1, (out_size, hidden_size)), "W2", learnable=True)
encoder_output3 = DeterministicVariable(encoder3(encoder_output2["mean"]), name="encoder_output3") Qz1 = NormalVariable(encoder_output3["mean"], encoder_output3["sd"], name="z1") model.set_posterior_model(ProbabilisticModel([Qx, Qz1, Qz2, Qz3])) # Joint-contrastive inference inference.perform_inference(model, inference_method=ReverseKL(gradient_estimator=PathwiseDerivativeEstimator), number_iterations=num_itr, number_samples=1, optimizer="Adam", lr=0.0005) loss_list1.append(np.array(model.diagnostics["loss curve"])) ELBO1 = [] for n in range(N_ELBO_ITR): ELBO1.append(model.estimate_log_model_evidence(N_ELBO).detach().numpy()) print("MF ELBO: {} +- {}".format(np.mean(ELBO1), np.std(ELBO1)/np.sqrt(float(N_ELBO_ITR)))) ## Structered hierarchical model # Initialize encoder and decoders noise_inpt_size = 50 encoder1 = BF.BrancherFunction(EncoderArchitecture1(image_size=image_size, latent_size3=latent_size3, noise_inpt_size=noise_inpt_size)) encoder2 = BF.BrancherFunction(EncoderArchitecture2(latent_size2=latent_size2, latent_size3=latent_size3, noise_inpt_size=noise_inpt_size)) encoder3 = BF.BrancherFunction(EncoderArchitecture3(latent_size1=latent_size1, latent_size2=latent_size2, noise_inpt_size=noise_inpt_size)) decoder1 = BF.BrancherFunction(DecoderArchitecture1(latent_size1=latent_size1, latent_size2=latent_size2)) decoder2 = BF.BrancherFunction(DecoderArchitecture2(latent_size2=latent_size2, latent_size3=latent_size3)) decoder3 = BF.BrancherFunction(DecoderArchitecture3(latent_size3=latent_size3, image_size=image_size)) # Generative model z1 = NormalVariable(np.zeros((latent_size1,)), z1sd*np.ones((latent_size1,)), name="z1")
# Joint-contrastive inference inference.perform_inference( model, inference_method=ReverseKL( gradient_estimator=PathwiseDerivativeEstimator), number_iterations=num_itr, number_samples=1, optimizer="Adam", lr=0.0005) loss_list1.append(np.array(model.diagnostics["loss curve"])) ELBO1 = [] for n in range(N_ELBO_ITR): ELBO1.append( model.estimate_log_model_evidence(N_ELBO).detach().numpy()) print("MF ELBO: {} +- {}".format( np.mean(ELBO1), np.std(ELBO1) / np.sqrt(float(N_ELBO_ITR)))) # ## Structered hierarchical model # # Initialize encoder and decoders # noise_inpt_size = 50 # encoder1 = BF.BrancherFunction( # EncoderArchitecture1(image_size=image_size, latent_size3=latent_size3, noise_inpt_size=noise_inpt_size)) # encoder2 = BF.BrancherFunction( # EncoderArchitecture2(latent_size2=latent_size2, latent_size3=latent_size3, noise_inpt_size=noise_inpt_size)) # encoder3 = BF.BrancherFunction( # EncoderArchitecture3(latent_size1=latent_size1, latent_size2=latent_size2, noise_inpt_size=noise_inpt_size)) # # decoder1 = BF.BrancherFunction(DecoderArchitecture1(latent_size1=latent_size1, latent_size2=latent_size2))
model_output = sum([ output * w for output, w in zip(model_output_list, inference_method.weights) ]) output_label = int(np.argmax(model_output)) scores_ne.append(1 if output_label == int(test_label.detach().numpy()) else 0) s += 1 if output_label == int(test_label.detach().numpy()) else 0 #print("Accuracy Ensemble: {} %".format(100*s/float(num_images))) PELBO = sum([ w * float( model.estimate_log_model_evidence( number_samples=50000, posterior_model=sampler, for_gradient=False).detach().numpy()) for sampler, w in zip(inference_method.sampler_model, inference_method.weights) ]) entropy = -sum([ w * np.log(w) if w > 0. else 0. for w in inference_method.weights ]) print("ELBO: " + str(PELBO + entropy)) #current_results.append(100*s/float(num_images)) current_results.append(PELBO + entropy) print("Exp {}: {} +- {}".format( N, np.mean(current_results), np.sqrt(np.var(current_results) / num_repetitions))) results.append(current_results) errors.append((np.mean(current_results),
ProbabilisticModel(Qpeople_means + Qgroup_means)) # Inference # N_itr = 300 N_smpl = 50 optimizer = "SGD" lr = 0.00001 inference.perform_inference(model, number_iterations=N_itr, number_samples=N_smpl, optimizer=optimizer, lr=lr) loss_list2 = model.diagnostics["loss curve"] N_ELBO = 1000 ELBO2 += [model.estimate_log_model_evidence(1000).detach().numpy()] # Structured NN distribution # hidden_size = 5 latent_size = 5 out_size = N_groups + N_people Qepsilon = Normal(np.zeros((latent_size, 1)), np.ones((latent_size, )), 'epsilon', learnable=True) W1 = RootVariable(np.random.normal(0, 0.1, (hidden_size, latent_size)), "W1", learnable=True) W2 = RootVariable(np.random.normal(0, 0.1, (out_size, hidden_size)), "W2", learnable=True)
# Joint-contrastive inference num_itr = 2000 inference.perform_inference( model, inference_method=ReverseKL(gradient_estimator=PathwiseDerivativeEstimator), number_iterations=num_itr, number_samples=1, optimizer="Adam", lr=0.0005) loss_list1 = model.diagnostics["loss curve"] N_ELBO = 20 N_ELBO_ITR = 1 ELBO = 0 for n in range(N_ELBO_ITR): ELBO += (model.estimate_log_model_evidence(N_ELBO) / float(N_ELBO_ITR)).detach().numpy() print(ELBO) # # sigmoid = lambda x: 1/(np.exp(-x) + 1) # image_grid = [] # z_range = np.linspace(-3, 3, 30) # for z1a in z_range: # image_row = [] # for z1b in z_range: # sample = model.get_sample(1, input_values={z1: np.array([z1a, z1b])}) # image = sigmoid(np.reshape(sample["decoder_output2"].values[0]["mean"], newshape=(28, 28))) # image_row += [image] # image_grid += [np.concatenate(image_row, axis=0)] # image_grid = np.concatenate(image_grid, axis=1)
x_names[t], learnable=True)) variational_posterior = ProbabilisticModel(Qx) AR_model.set_posterior_model(variational_posterior) # Inference # inference.perform_inference(AR_model, number_iterations=N_itr, number_samples=N_smpl, optimizer=optimizer, lr=lr) loss_list1 = AR_model.diagnostics["loss curve"] N_ELBO_smpl = 1000 ELBO1 = AR_model.estimate_log_model_evidence(N_ELBO_smpl) print("PE {}".format(ELBO1)) # Statistics posterior_samples1 = AR_model._get_posterior_sample(2000) x_mean1 = [] lower_bound1 = [] upper_bound1 = [] sigmoid = lambda x: 1 / (1 + np.exp(-x)) lambda_list = [sigmoid(float(l._get_sample(1)[l].detach().numpy())) \ for l in Qlambda] for xt in x: x_posterior_samples1 = transform(
# Inference # N_iter = 400 n_samples = 10 optimizer = "Adam" lr = 0.01 inference.perform_inference(AR_model, number_iterations=N_iter, number_samples=n_samples, optimizer=optimizer, lr=lr) loss_list = AR_model.diagnostics["loss curve"] # ELBO ELBO = AR_model.estimate_log_model_evidence(15000) print("The ELBO is {}".format(ELBO)) # Statistics posterior_samples = AR_model._get_posterior_sample(2000) b_posterior_samples = posterior_samples[b].detach().numpy().flatten() b_mean = np.mean(b_posterior_samples) b_sd = np.sqrt(np.var(b_posterior_samples)) x_mean = [] lower_bound = [] upper_bound = [] for xt in x: x_posterior_samples = posterior_samples[xt].detach().numpy().flatten() mean = np.mean(x_posterior_samples) sd = np.sqrt(np.var(x_posterior_samples))