# %% log_ratio_mean = gpdre.logit(X_grid, convert_to_tensor_fn=tfd.Distribution.mean) log_ratio_stddev = gpdre.logit(X_grid, convert_to_tensor_fn=tfd.Distribution.stddev) # %% fig, ax = plt.subplots() ax.plot(X_grid, r.logit(X_grid), c='k', label=r"$f(x) = \log p(x) - \log q(x)$") ax.plot(X_grid, log_ratio_mean.numpy().T, label="posterior mean") fill_between_stddev(X_grid.squeeze(), log_ratio_mean.numpy().squeeze(), log_ratio_stddev.numpy().squeeze(), alpha=0.1, label="posterior std dev", ax=ax) ax.set_xlabel('$x$') ax.set_ylabel('$f(x)$') ax.legend() plt.show() # %% ratio_estimator = gpdre.ratio_distribution(X_grid) # %%
def main(name, width, aspect, extension, output_dir): # preamble random_state = np.random.RandomState(SEED) figsize = size(width, aspect) suffix = f"{width:.0f}x{width/aspect:.0f}" rc = { "figure.figsize": figsize, "font.serif": ["Times New Roman"], "text.usetex": True, } sns.set(context="paper", style="ticks", palette="colorblind", font="serif", rc=rc) output_path = Path(output_dir).joinpath(name) output_path.mkdir(parents=True, exist_ok=True) # /preamble # dataset = load_boston() # X_train, X_test, y_train, y_test = train_test_split(dataset.data, # dataset.target, # test_size=TEST_SIZE, # random_state=random_state) # test_metric = make_test_metric(X_train, y_train, X_test, y_test) # %% # log_gamma_min, log_gamma_max = -8.0, 0.0 # # equivalent to: # # gamma = np.logspace(log_gamma_min, log_gamma_max, NUM_INDEX_POINTS) # log_gamma = np.linspace(log_gamma_min, log_gamma_max, NUM_INDEX_POINTS) # gamma = 10.0**log_gamma gamma = np.linspace(-1.0, 2.0, NUM_INDEX_POINTS) y = latent(gamma) # %% fig, ax = plt.subplots() ax.plot(gamma, y) # ax.set_xscale("log") ax.set_xlabel(r'$x$') ax.set_ylabel(r"$y$ (test mse)") for ext in extension: fig.savefig(output_path.joinpath(f"objective_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% # log_gamma_samples = random_state.uniform(low=log_gamma_min, # high=log_gamma_max, # size=NUM_INIT_RANDOM) # gamma_samples = 10.0**log_gamma_samples load_observations = make_regression_dataset(latent) gamma_samples, y_samples = load_observations(num_samples=NUM_INIT_RANDOM, num_features=1, noise_variance=0.2, x_min=-1.0, x_max=2.0, random_state=random_state) # %% y_threshold = np.quantile(y_samples, q=PI) mask_lesser = (y_samples <= y_threshold) mask_greater = ~mask_lesser gamma_samples_lesser = gamma_samples[mask_lesser] gamma_samples_greater = gamma_samples[mask_greater] # log_gamma_samples_lesser = log_gamma_samples[mask_lesser] # log_gamma_samples_greater = log_gamma_samples[mask_greater] y_samples_lesser = y_samples[mask_lesser] y_samples_greater = y_samples[mask_greater] # %% fig, ax = plt.subplots() ax.plot(gamma, y) # ax.scatter(log_gamma_samples, y_samples, c=mask_lesser, # alpha=0.7, cmap="coolwarm") ax.scatter(gamma_samples_lesser, y_samples_lesser, alpha=0.8) ax.scatter(gamma_samples_greater, y_samples_greater, alpha=0.8) ax.axhline(y_threshold, xmin=0, xmax=1.0, color='k', linewidth=1.0, linestyle='dashed') # ax.set_xscale("log") ax.set_xlabel(r'$x$') ax.set_ylabel(r"$y$ (test mse)") for ext in extension: fig.savefig(output_path.joinpath(f"candidates_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% y_samples_sorted = np.sort(y_samples) y_samples_quantile = np.arange(NUM_INIT_RANDOM) / NUM_INIT_RANDOM # %% fig, ax = plt.subplots() ax.plot(y_samples_sorted, y_samples_quantile) ax.vlines(y_threshold, ymin=0, ymax=PI, colors='k', linestyles='dashed', linewidth=1.0) ax.hlines(PI, xmin=y_samples_sorted[0], xmax=y_threshold, colors='k', linestyles='dashed', linewidth=1.0) ax.set_xlabel(r'$y$') ax.set_ylabel(r'$\Phi(y)$ (ecdf)') for ext in extension: fig.savefig(output_path.joinpath(f"ecdf_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% fig, ax = plt.subplots() sns.distplot(gamma_samples_lesser, hist=False, rug=True, label=r'$\ell(x)$', kde_kws=dict(shade=True, bw=BANDWIDTH), ax=ax) sns.distplot(gamma_samples_greater, hist=False, rug=True, label=r'$g(x)$', kde_kws=dict(shade=True, bw=BANDWIDTH), ax=ax) ax.set_xlabel(r'$\log_{10}{x}$') ax.set_ylabel("density") ax.legend() for ext in extension: fig.savefig(output_path.joinpath(f"kde_seaborn_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% kde_lesser = sm.nonparametric.KDEUnivariate(gamma_samples_lesser) # kde_lesser.fit(bw=BANDWIDTH) kde_lesser.fit(bw="normal_reference") kde_greater = sm.nonparametric.KDEUnivariate(gamma_samples_greater) # kde_greater.fit(bw=BANDWIDTH) kde_greater.fit(bw="normal_reference") # %% fig, ax = plt.subplots() ax.plot(gamma, kde_lesser.evaluate(gamma), label=fr"$\ell(x)$ -- bw {kde_lesser.bw:.2f}") ax.plot(gamma, kde_greater.evaluate(gamma), label=fr"$g(x)$ -- bw {kde_greater.bw:.2f}") sns.rugplot(gamma_samples_lesser, c='tab:blue', ax=ax) sns.rugplot(gamma_samples_greater, c='tab:orange', ax=ax) # ax.set_xscale("log") ax.set_xlabel(r'$x$') ax.set_ylabel("density") ax.legend() for ext in extension: fig.savefig(output_path.joinpath( f"kde_statsmodel_normal_reference_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% fig, ax = plt.subplots() ax.plot(gamma, kde_lesser.evaluate(gamma), linestyle="dashed", alpha=0.4, label=fr"$\ell(x)$ -- bw {kde_lesser.bw:.2f}") ax.plot(gamma, kde_greater.evaluate(gamma), linestyle="dashed", alpha=0.4, label=fr"$g(x)$ -- bw {kde_greater.bw:.2f}") ax.plot(gamma, kde_lesser.evaluate(gamma) / kde_greater.evaluate(gamma), label=r'$\ell(x) / g(x)$') ax.plot( gamma, kde_lesser.evaluate(gamma) / mixture( kde_lesser.evaluate(gamma), kde_greater.evaluate(gamma), pi=PI), label=r'$r_{\gamma}(x)$') sns.rugplot(gamma_samples_lesser, c='tab:blue', ax=ax) sns.rugplot(gamma_samples_greater, c='tab:orange', ax=ax) # ax.set_xscale("log") ax.set_xlabel(r'$x$') ax.set_ylabel(r"$\alpha(x)$") ax.legend() for ext in extension: fig.savefig(output_path.joinpath( f"ratio_kde_statsmodel_normal_reference_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% fig, ax_main = plt.subplots() divider = make_axes_locatable(ax_main) ax_main.plot(gamma, y, color="tab:gray", label="latent function") ax_main.scatter(gamma_samples_lesser, y_samples[mask_lesser], alpha=0.8, label=r'$y < \tau$') ax_main.scatter(gamma_samples_greater, y_samples[mask_greater], alpha=0.8, label=r'$y \geq \tau$') ax_main.axhline(y_threshold, xmin=0, xmax=1.0, color='gray', linewidth=1.0, linestyle='dashed') ax_main.annotate(rf"$y^{{\star}}={{{y_threshold:.2f}}}$", xy=(gamma[0], y_threshold), xycoords='data', xytext=(-5.0, -8.0), textcoords='offset points', fontsize="x-small", arrowprops=dict(facecolor='black', arrowstyle='-')) # ax_main.set_xscale("log") ax_main.set_xlabel(r'$x$') ax_main.set_ylabel(r"$y$ (val mse)") ax_main.legend() ax_x = divider.append_axes("top", size=0.9, pad=0.1, sharex=ax_main) ax_x.plot(gamma, kde_lesser.evaluate(gamma), label=r"$\ell(x)$") ax_x.plot(gamma, kde_greater.evaluate(gamma), label=r"$g(x)$") sns.rugplot(gamma_samples_lesser, c='tab:blue', ax=ax_x) sns.rugplot(gamma_samples_greater, c='tab:orange', ax=ax_x) ax_x.set_ylabel("density") ax_x.xaxis.set_tick_params(labelbottom=False) ax_x.legend() ax_y = divider.append_axes("right", size=0.9, pad=0.1, sharey=ax_main) ax_y.plot(y_samples_quantile, y_samples_sorted) ax_y.hlines(y_threshold, xmin=0, xmax=PI, colors='gray', linestyles='dashed', linewidth=1.0) ax_y.vlines(PI, ymin=y_samples_sorted[0], ymax=y_threshold, colors='gray', linestyles='dashed', linewidth=1.0) ax_y.annotate(rf"$\gamma={{{PI:.2f}}}$", xy=(PI, y_samples_sorted[0]), xycoords='data', xytext=(5.0, 0.0), textcoords='offset points', fontsize="x-small", arrowprops=dict(facecolor='black', arrowstyle='-')) ax_y.set_xlabel(r'$\Phi(y)$ (ecdf)') ax_y.yaxis.set_tick_params(labelleft=False) for ext in extension: fig.savefig(output_path.joinpath(f"summary_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% kde_lesser = KernelDensity(kernel='gaussian', bandwidth=BANDWIDTH) \ .fit(gamma_samples_lesser.reshape(-1, 1)) log_density_lesser = kde_lesser.score_samples(gamma.reshape(-1, 1)) kde_greater = KernelDensity(kernel='gaussian', bandwidth=BANDWIDTH) \ .fit(gamma_samples_greater.reshape(-1, 1)) log_density_greater = kde_greater.score_samples(gamma.reshape(-1, 1)) # %% fig, ax = plt.subplots() ax.plot(gamma, np.exp(log_density_lesser), label=r'$\ell(x)$') ax.plot(gamma, np.exp(log_density_greater), label=r'$g(x)$') sns.rugplot(gamma_samples_lesser, c='tab:blue', ax=ax) sns.rugplot(gamma_samples_greater, c='tab:orange', ax=ax) # ax.set_xscale("log") ax.set_xlabel(r'$x$') ax.set_ylabel("density") ax.legend() for ext in extension: fig.savefig(output_path.joinpath(f"kde_sklearn_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% fig, ax = plt.subplots() ax.plot(gamma, np.exp(log_density_lesser), linestyle="dashed", alpha=0.4, label=fr"$\ell(x)$ -- bw {BANDWIDTH:.2f}") ax.plot(gamma, np.exp(log_density_greater), linestyle="dashed", alpha=0.4, label=fr"$g(x)$ -- bw {BANDWIDTH:.2f}") ax.plot(gamma, np.exp(log_density_lesser - log_density_greater), label=r'$\ell(x) / g(x)$') sns.rugplot(gamma_samples_lesser, c='tab:blue', ax=ax) sns.rugplot(gamma_samples_greater, c='tab:orange', ax=ax) # ax.set_xscale("log") ax.set_xlabel(r'$x$') ax.set_ylabel(r"$\alpha(x)$") ax.legend() for ext in extension: fig.savefig(output_path.joinpath(f"ratio_kde_sklearn_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% from gpdre.external.rulsif import RuLSIFDensityRatioEstimator # from gpdre.external.kliep import KLIEPDensityRatioEstimator # %% # kliep = KLIEPDensityRatioEstimator(seed=SEED) # kliep.fit(log_gamma_samples_lesser.reshape(-1, 1), # log_gamma_samples_greater.reshape(-1, 1)) rulsif = RuLSIFDensityRatioEstimator(alpha=PI) rulsif.fit(gamma_samples_lesser, gamma_samples_greater) # %% fig, ax = plt.subplots() ax.plot(gamma, rulsif.ratio(gamma), c="tab:orange", label='RuLSIF') sns.rugplot(gamma_samples_lesser, c='tab:blue', ax=ax) sns.rugplot(gamma_samples_greater, c='tab:orange', ax=ax) # ax.set_xscale("log") ax.set_xlabel(r'$x$') ax.set_ylabel(r"$\alpha(x)$") ax.legend() for ext in extension: fig.savefig(output_path.joinpath(f"ratio_rulsif_{suffix}.{ext}"), bbox_inches="tight") plt.show() # %% amplitude = tfp.util.TransformedVariable(1.0, bijector=tfp.bijectors.Softplus(), dtype="float64", name='amplitude') length_scale = tfp.util.TransformedVariable( 0.5, bijector=tfp.bijectors.Softplus(), dtype="float64", name='length_scale') observation_noise_variance = tfp.util.TransformedVariable( 1e-1, bijector=tfp.bijectors.Softplus(), dtype="float64", name='observation_noise_variance') kernel = kernel_cls(amplitude=amplitude, length_scale=length_scale) gp = tfd.GaussianProcess( kernel=kernel, index_points=gamma_samples.reshape(-1, 1), observation_noise_variance=observation_noise_variance) optimizer = tf.keras.optimizers.Adam(learning_rate=0.05, beta_1=0.5, beta_2=0.99) num_epochs = 200 for epoch in range(num_epochs): with tf.GradientTape() as tape: nll = -gp.log_prob(y_samples) gradients = tape.gradient(nll, gp.trainable_variables) optimizer.apply_gradients(zip(gradients, gp.trainable_variables)) gprm = tfd.GaussianProcessRegressionModel( kernel=kernel, index_points=gamma.reshape(-1, 1), observation_index_points=gamma_samples.reshape(-1, 1), observations=y_samples, observation_noise_variance=observation_noise_variance, jitter=1e-6) fig, ax = plt.subplots() ax.plot(gamma, gprm.mean(), label="posterior predictive mean") fill_between_stddev(gamma, gprm.mean().numpy().squeeze(), gprm.stddev().numpy().squeeze(), alpha=0.1, label="posterior predictive std dev", ax=ax) ax.plot(gamma, y, label="true", color="tab:gray") ax.scatter(gamma_samples, y_samples, alpha=0.8) ax.legend() ax.set_xlabel(r'$x$') ax.set_ylabel(r'$y$') for ext in extension: fig.savefig( output_path.joinpath(f"gp_posterior_predictive_{suffix}.{ext}"), bbox_inches="tight") plt.show() gprm_marginals = tfd.Normal(loc=gprm.mean(), scale=gprm.stddev()) ei = np.maximum(y_threshold - gprm.mean(), 0.) * gprm_marginals.cdf(y_threshold) ei += gprm.stddev() * gprm_marginals.prob(y_threshold) fig, ax = plt.subplots() ax.plot(gamma, ei) for ext in extension: fig.savefig(output_path.joinpath(f"ei_{suffix}.{ext}"), bbox_inches="tight") plt.show() pis = np.arange(0., 0.5, 0.15) y_quantiles = np.quantile(y_samples, q=pis) eis = (y_quantiles.reshape(-1, 1) - gprm.mean()) * gprm_marginals.cdf( y_quantiles.reshape(-1, 1)) eis += gprm.stddev() * gprm_marginals.prob(y_quantiles.reshape(-1, 1)) import pandas as pd data = pd.DataFrame(data=eis.numpy(), index=pis, columns=gamma) data.index.name = r"$\gamma$" data.columns.name = r"$x$" s = data.stack() s.name = "ei" df = s.reset_index() fig, ax = plt.subplots() sns.lineplot(x=r"$x$", y="ei", hue=r"$\gamma$", data=df, ax=ax) # ax.plot(gamma, eis.numpy().T) for ext in extension: fig.savefig(output_path.joinpath(f"eis_{suffix}.{ext}"), bbox_inches="tight") plt.show() return 0
# %% # m = tf.matmul(vgp.kernel.K(X_train), vgp.q_alpha) # %% fig, ax = plt.subplots() ax.plot(X_grid, r.logit(X_grid), c='k', label=r"$f(x) = \log p(x) - \log q(x)$") ax.plot(X_grid, qf_loc, label="posterior mean") fill_between_stddev(X_grid.squeeze(), qf_loc, qf_scale, alpha=0.1, label="posterior std dev", ax=ax) # ax.scatter(X_train, m.numpy(), marker='+', c="tab:blue", # label="inducing variable mean") ax.set_xlabel('$x$') ax.set_ylabel('$f(x)$') ax.legend() plt.show() # %%