def main(filename, dataset_filename, optimizer, num_layers, num_units, activation, epochs, batch_size, l1_factor, l2_factor, seed): random_state = np.random.RandomState(seed) # Don't get confused -- train and test refer to those of the downstream # prediction task. Both train and test are used for training of the DRE # here. (X_train, y_train), (X_test, y_test) = load_hdf5(dataset_filename) X, y = make_classification_dataset(X_test, X_train) # Model specification model = build_model(output_dim=1, num_layers=num_layers, num_units=num_units, activation=activation, kernel_regularizer=l1_l2(l1_factor, l2_factor), seed=seed) model.compile(loss=binary_crossentropy_from_logits, optimizer=optimizer, metrics=["accuracy"]) hist = model.fit(X, y, batch_size=batch_size, epochs=epochs, shuffle=True) importance_weights = np.exp(model.predict(X_train)).squeeze() with h5py.File(filename, 'w') as f: f.create_dataset("importance_weights", data=importance_weights) return 0
def main(name, summary_dir, seed): summary_path = Path(summary_dir) # .joinpath(name) summary_path.mkdir(parents=True, exist_ok=True) r = SugiyamaKrauledatMuellerDensityRatioMarginals() rows = [] for seed in range(num_seeds): # (X_train, y_train), (X_test, y_test) = r.train_test_split(X, y, seed=seed) (X_train, y_train), (X_test, y_test) = r.make_covariate_shift_dataset( num_test, num_train, class_posterior_fn=class_posterior, threshold=0.5, seed=seed) X, s = make_classification_dataset(X_test, X_train) # # Uniform # acc = metric(X_train, y_train, X_test, y_test, random_state=seed) # rows.append(dict(weight="uniform", acc=acc, seed=seed)) # # Exact # acc = metric(X_train, y_train, X_test, y_test, # sample_weight=r.ratio(X_train).numpy(), random_state=seed) # rows.append(dict(weight="exact", acc=acc, seed=seed)) for epochs in [500, 1000, 1500, 2000]: # for sparsity_factor in sparsity_factors: num_inducing_points = int(len(X) * sparsity_factor) # Gaussian Processes gpdre = GaussianProcessDensityRatioEstimator( input_dim=num_features, kernel_cls=Matern52, use_ard=use_ard, num_inducing_points=num_inducing_points, inducing_index_points_initializer=KMeans(X, seed=seed), vgp_cls=SVGP, whiten=True, jitter=jitter, seed=seed) gpdre.compile(optimizer=optimizer) gpdre.fit(X_test, X_train, epochs=epochs, batch_size=batch_size, buffer_size=buffer_size) for prop_name, prop in props.items(): r_prop = gpdre.ratio(X_train, convert_to_tensor_fn=prop) acc = metric(X_train, y_train, X_test, y_test, sample_weight=r_prop.numpy(), random_state=seed) rows.append(dict(weight=prop_name, acc=acc, seed=seed, sparsity_factor=sparsity_factor, use_ard=use_ard, epochs=epochs)) data = pd.DataFrame(rows) data.to_csv(str(summary_path.joinpath(f"{name}.csv"))) return 0
fig, ax = plt.subplots() ax.plot(X_grid, r.ratio(X_grid), c='k', label=r"$r(x) = \exp{f(x)}$") ax.set_xlabel('$x$') ax.set_ylabel('$r(x)$') ax.legend() plt.show() # %% # Create classification dataset. X_p, X_q = r.make_dataset(num_train, seed=dataset_seed) X_train, y_train = make_classification_dataset(X_p, X_q) # %% # Dataset visualized against the Bayes optimal classifier. fig, ax = plt.subplots() ax.plot(X_grid, r.prob(X_grid), c='k', label=r"$\pi(x) = \sigma(f(x))$") ax.scatter(X_train, y_train, c=y_train, s=12.**2, marker='s', alpha=0.1, cmap="coolwarm_r") ax.set_yticks([0, 1]) ax.set_yticklabels([r"$x_q \sim q(x)$", r"$x_p \sim p(x)$"]) ax.set_xlabel('$x$') ax.legend()
def main(name, summary_dir, seed): summary_path = Path(summary_dir).joinpath("sugiyama") summary_path.mkdir(parents=True, exist_ok=True) r = SugiyamaKrauledatMuellerDensityRatioMarginals() rows = [] for seed in range(num_seeds): # (X_train, y_train), (X_test, y_test) = r.train_test_split(X, y, seed=seed) (X_train, y_train), (X_test, y_test) = r.make_covariate_shift_dataset( num_test, num_train, class_posterior_fn=class_posterior, threshold=0.5, seed=seed) X, s = make_classification_dataset(X_test, X_train) # Uniform acc = metric(X_train, y_train, X_test, y_test, random_state=seed) rows.append( dict(weight="uniform", acc=acc, seed=seed, dataset_seed=seed)) # Exact acc = metric(X_train, y_train, X_test, y_test, sample_weight=r.ratio(X_train).numpy(), random_state=seed) rows.append(dict(weight="exact", acc=acc, seed=seed, dataset_seed=seed)) # RuLSIF r_rulsif = RuLSIFDensityRatioEstimator(alpha=1e-6) r_rulsif.fit(X_test, X_train) sample_weight = np.maximum(1e-6, r_rulsif.ratio(X_train)) acc = metric(X_train, y_train, X_test, y_test, sample_weight=sample_weight, random_state=seed) rows.append( dict(weight="rulsif", acc=acc, seed=seed, dataset_seed=seed)) # KLIEP # sigmas = [0.1, 0.25, 0.5, 0.75, 1.0] sigmas = list(np.maximum(0.25 * np.arange(5), 0.1)) r_kliep = KLIEPDensityRatioEstimator(sigmas=sigmas, seed=seed) r_kliep.fit(X_test, X_train) sample_weight = np.maximum(1e-6, r_kliep.ratio(X_train)) acc = metric(X_train, y_train, X_test, y_test, sample_weight=sample_weight, random_state=seed) rows.append(dict(weight="kliep", acc=acc, seed=seed, dataset_seed=seed)) # KMM r_kmm = KMMDensityRatioEstimator(B=1000.0) r_kmm.fit(X_test, X_train) sample_weight = np.maximum(1e-6, r_kmm.ratio(X_train)) acc = metric(X_train, y_train, X_test, y_test, sample_weight=sample_weight, random_state=seed) rows.append(dict(weight="kmm", acc=acc, seed=seed, dataset_seed=seed)) # Logistic Regression (Linear) r_logreg = LogisticRegressionDensityRatioEstimator(seed=seed) r_logreg.fit(X_test, X_train) sample_weight = np.maximum(1e-6, r_logreg.ratio(X_train).numpy()) acc = metric(X_train, y_train, X_test, y_test, sample_weight=sample_weight, random_state=seed) rows.append( dict(weight="logreg", acc=acc, seed=seed, dataset_seed=seed)) # Logistic Regression (MLP) r_mlp = MLPDensityRatioEstimator(num_layers=1, num_units=8, activation="tanh", seed=seed) r_mlp.compile(optimizer=optimizer, metrics=["accuracy"]) r_mlp.fit(X_test, X_train, epochs=epochs, batch_size=batch_size) sample_weight = np.maximum(1e-6, r_mlp.ratio(X_train).numpy()) acc = metric(X_train, y_train, X_test, y_test, sample_weight=sample_weight, random_state=seed) rows.append(dict(weight="mlp", acc=acc, seed=seed, dataset_seed=seed)) # Gaussian Processes gpdre = GaussianProcessDensityRatioEstimator( input_dim=num_features, kernel_cls=Matern52, num_inducing_points=num_inducing_points, inducing_index_points_initializer=KMeans(X, seed=seed), vgp_cls=SVGP, whiten=True, jitter=jitter, seed=seed) gpdre.compile(optimizer=optimizer) gpdre.fit(X_test, X_train, epochs=epochs, batch_size=batch_size, buffer_size=buffer_size) for prop_name, prop in props.items(): r_prop = gpdre.ratio(X_train, convert_to_tensor_fn=prop) acc = metric(X_train, y_train, X_test, y_test, sample_weight=r_prop.numpy(), random_state=seed) rows.append( dict(weight=prop_name, acc=acc, seed=seed, dataset_seed=seed)) data = pd.DataFrame(rows) data.to_csv(str(summary_path.joinpath(f"{name}.csv"))) return 0
X2, X1 = np.mgrid[-4:9:200j, -6:8:200j] X_grid = np.dstack((X1, X2)) # %% def class_posterior(x1, x2): return 0.5 * (1 + tf.tanh(x1 - tf.nn.relu(-x2))) # %% r = SugiyamaKrauledatMuellerDensityRatioMarginals() (X_train, y_train), (X_test, y_test) = r.make_covariate_shift_dataset( num_test, num_train, class_posterior_fn=class_posterior, threshold=0.5, seed=dataset_seed) X, s = make_classification_dataset(X_test, X_train) # %% fig, ax = plt.subplots() ax.set_title(r"Train $p_{\mathrm{tr}}(\mathbf{x})$ and " r"test $p_{\mathrm{te}}(\mathbf{x})$ distributions") contours_train = ax.contour(X1, X2, r.bot.prob(X_grid), cmap="Blues") contours_test = ax.contour(X1, X2, r.top.prob(X_grid), cmap="Oranges", linestyles="--") ax.clabel(contours_train, fmt="%.2f") ax.clabel(contours_test, fmt="%.2f") ax.set_xlabel(r'$x_1$')
def main(name, width, aspect, extension, output_dir): figsize = size(width, aspect) suffix = f"{width:.0f}x{width/aspect:.0f}" # preamble rc = { "figure.figsize": figsize, "font.serif": ["Times New Roman"], "text.usetex": True, } sns.set(context="paper", style="ticks", palette="colorblind", font="serif", rc=rc) output_path = Path(output_dir).joinpath(name) output_path.mkdir(parents=True, exist_ok=True) # /preamble xmin, xmax = -6, 8 ymin, ymax = -4, 9 X2, X1 = np.mgrid[ymin:ymax:200j, xmin:xmax:200j] X_grid = np.dstack((X1, X2)) r = SugiyamaKrauledatMuellerDensityRatioMarginals() (X_train, y_train), (X_test, y_test) = r.make_covariate_shift_dataset( NUM_TEST, NUM_TRAIN, class_posterior_fn=class_posterior, seed=DATASET_SEED) X, s = make_classification_dataset(X_test, X_train) # Figure 3 fig, ax = plt.subplots() ax.set_title(r"$f(\mathbf{x}) = " r"\log p_{\mathrm{te}}(\mathbf{x}) - " r"\log p_{\mathrm{tr}}(\mathbf{x})$") contours = ax.contour(X1, X2, r.logit(X_grid).numpy(), cmap="PuOr") fig.colorbar(contours, ax=ax) ax.clabel(contours, fmt="%.2f") ax.set_xlabel(r'$x_1$') ax.set_ylabel(r'$x_2$') ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) for ext in extension: fig.savefig(output_path.joinpath(f"logit_{suffix}.{ext}"), bbox_inches="tight") plt.show() # Figure 4 fig, ax = plt.subplots() ax.set_title(r"$r(\mathbf{x}) = \exp(f(\mathbf{x}))$") contours = ax.contour(X1, X2, r.ratio(X_grid).numpy(), cmap="PuOr") fig.colorbar(contours, ax=ax) ax.clabel(contours, fmt="%.2f") ax.set_xlabel(r'$x_1$') ax.set_ylabel(r'$x_2$') ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) for ext in extension: fig.savefig(output_path.joinpath(f"ratio_{suffix}.{ext}"), bbox_inches="tight") plt.show() # Figure 5 fig, ax = plt.subplots() ax.set_title(r"$P(s=1|\mathbf{x}) = \sigma(f(\mathbf{x}))$") contours = ax.contour(X1, X2, r.prob(X_grid).numpy(), cmap="PuOr") ax.scatter(*X.T, c=r.prob(X).numpy(), cmap="PuOr", alpha=0.6) fig.colorbar(contours, ax=ax) ax.clabel(contours, fmt="%.2f") ax.set_xlabel(r'$x_1$') ax.set_ylabel(r'$x_2$') ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) for ext in extension: fig.savefig(output_path.joinpath(f"prob_{suffix}.{ext}"), bbox_inches="tight") plt.show() model_uniform_train = LogisticRegression(C=1.0, random_state=SEED) model_uniform_train.fit(X_train, y_train) model_uniform_test = LogisticRegression(C=1.0, random_state=SEED) model_uniform_test.fit(X_test, y_test) model_exact_train = LogisticRegression(C=1.0, random_state=SEED) model_exact_train.fit(X_train, y_train, sample_weight=r.ratio(X_train).numpy()) p_grid_uniform_train = model_uniform_train.predict_proba(X_grid.reshape(-1, 2)) \ .reshape(200, 200, 2) p_grid_uniform_test = model_uniform_test.predict_proba(X_grid.reshape(-1, 2)) \ .reshape(200, 200, 2) p_grid_exact_train = model_exact_train.predict_proba(X_grid.reshape(-1, 2)) \ .reshape(200, 200, 2) # Figure 7 fig, ax = plt.subplots() ax.set_title("Without importance sampling (uniform weights)") contours = ax.contour(X1, X2, p_grid_uniform_train[..., -1], cmap="RdYlBu") fig.colorbar(contours, ax=ax) ax.clabel(contours, fmt="%.2f") ax.scatter(*X_train.T, c=y_train, cmap="RdYlBu", alpha=0.8, label="train") ax.scatter(*X_test.T, marker='x', c=y_test, cmap="RdYlBu", alpha=0.2, label="test") ax.legend() ax.set_xlabel(r'$x_1$') ax.set_ylabel(r'$x_2$') ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) for ext in extension: fig.savefig(output_path.joinpath(f"logistic_uniform_{suffix}.{ext}"), bbox_inches="tight") plt.show() # Figure 8 fig, ax = plt.subplots() ax.set_title("With importance sampling (exact density ratio)") contours = ax.contour(X1, X2, p_grid_exact_train[..., -1], cmap="RdYlBu") fig.colorbar(contours, ax=ax) ax.clabel(contours, fmt="%.2f") ax.scatter(*X_train.T, c=y_train, s=r.ratio(X_train).numpy(), cmap="RdYlBu", alpha=0.8, label="train") ax.scatter( *X_test.T, marker='x', c=y_test, # s=r.ratio(X_test).numpy(), cmap="RdYlBu", alpha=0.2, label="test") ax.legend() ax.set_xlabel(r'$x_1$') ax.set_ylabel(r'$x_2$') ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) for ext in extension: fig.savefig(output_path.joinpath(f"logistic_exact_{suffix}.{ext}"), bbox_inches="tight") plt.show() # Figure 9 fig, ax = plt.subplots() contours1 = ax.contour(X1, X2, p_grid_uniform_train[..., -1], linestyles="solid", levels=1, zorder=-1, cmap="Blues") contours2 = ax.contour(X1, X2, p_grid_uniform_test[..., -1], linestyles="dashed", levels=1, zorder=-1, cmap="Oranges") contours3 = ax.contour(X1, X2, p_grid_exact_train[..., -1], linestyles="dotted", levels=1, zorder=-1, cmap="Greens") contours4 = ax.contour(X1, X2, class_posterior(X1, X2).numpy(), linestyles="dashdot", levels=1, zorder=-1, cmap="Purples") # fig.colorbar(contours, ax=ax) # ax.clabel(contours1, fmt="unweighted", fontsize="smaller") # ax.clabel(contours2, fmt="test", fontsize="smaller") # ax.clabel(contours3, fmt="ideal", fontsize="smaller") ax.scatter(*X_train.T, c=y_train, cmap="RdYlBu", alpha=0.8, label="train") ax.scatter(*X_test.T, marker='x', c=y_test, cmap="RdYlBu", alpha=0.2, label="test") ax.legend() ax.set_xlabel(r'$x_1$') ax.set_ylabel(r'$x_2$') ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) for ext in extension: fig.savefig(output_path.joinpath(f"dataset_{suffix}.{ext}"), bbox_inches="tight") plt.show() # Figure 2 fig, ax = plt.subplots() # ax.set_title(r"Train $p_{\mathrm{tr}}(\mathbf{x})$ and " # r"test $p_{\mathrm{te}}(\mathbf{x})$ distributions") contours_train = ax.contour(X1, X2, r.bot.prob(X_grid), cmap="Oranges") contours_test = ax.contour(X1, X2, r.top.prob(X_grid), cmap="Purples", linestyles="--") contours = ax.contour(X1, X2, class_posterior(X1, X2).numpy(), levels=1, zorder=-1, linestyles="dashdot", cmap="Purples") ax.clabel(contours_train, fmt="%.2f") ax.clabel(contours_test, fmt="%.2f") ax.set_xlabel(r'$x_1$') ax.set_ylabel(r'$x_2$') ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) for ext in extension: fig.savefig(output_path.joinpath(f"distribution_{suffix}.{ext}"), bbox_inches="tight") plt.show() return 0
def main(myname, summary_dir, seed): summary_path = Path(summary_dir).joinpath("liacc") summary_path.mkdir(parents=True, exist_ok=True) names = [ "abalone", "ailerons", "bank32", "bank8", "cali", "cpuact", "elevators", "puma8" ][4:] rows = [] for name in names: output_path = summary_path.joinpath(f"{name}") output_path.mkdir(parents=True, exist_ok=True) data_path = get_data_path(name) data_mat = loadmat(data_path) results_path = get_results_path(name) results_mat = loadmat(results_path) X_trains = get_splits(data_mat, key="X") y_trains = get_splits(data_mat, key="Y") X_tests = get_splits(data_mat, key="Xtest") y_tests = get_splits(data_mat, key="Ytest") weights = get_splits(data_mat, key="ideal") # X_train = X_trains[-1] # Y_train = y_trains[-1] # X_test = X_tests[-1] # Y_test = y_tests[-1] # weight = weights[-1] # proj = results_mat["W"] # X_train_low = X_train.dot(proj) # X_test_low = X_test.dot(proj) for split, (X_train, Y_train, X_test, Y_test, weight) in \ enumerate(zip(X_trains, y_trains, X_tests, y_tests, weights)): X, _ = make_classification_dataset(X_test, X_train) # X_low, _ = make_classification_dataset(X_test_low, X_train_low) num_features = X.shape[-1] # num_features_low = X_low.shape[-1] y_train = Y_train.squeeze(axis=-1) y_test = Y_test.squeeze(axis=-1) # sample_weight = weight.squeeze(axis=-1) # error = regression_metric(X_train, y_train, X_test, y_test) # rows.append(dict(weight="uniform", name=name, error=error, projection="none")) # # rows.append(dict(weight="uniform", name=name, split=split, error=error)) # error = regression_metric(X_train, y_train, X_test, y_test, # sample_weight=sample_weight) # rows.append(dict(weight="exact", name=name, error=error, projection="none")) # rows.append(dict(weight="exact", name=name, split=split, error=error)) # # # RuLSIF # # r_rulsif = RuLSIFDensityRatioEstimator(alpha=1e-6) # # r_rulsif.fit(X_test, X_train) # # sample_weight = np.maximum(1e-6, r_rulsif.ratio(X_train)) # # error = regression_metric(X_train, y_train, X_test, y_test, # # sample_weight=sample_weight) # # rows.append(dict(weight="rulsif", name=name, split=split, error=error)) # # # # KLIEP # # # r_kliep = KLIEPDensityRatioEstimator(seed=seed) # # # r_kliep.fit(X_test, X_train) # # # sample_weight = np.maximum(1e-6, r_kliep.ratio(X_train)) # # # error = regression_metric(X_train, y_train, X_test, y_test, # # # sample_weight=sample_weight) # # # rows.append(dict(weight="kliep", name=name, split=split, error=error)) # # # KMM # # r_kmm = KMMDensityRatioEstimator(B=1000.0) # # r_kmm.fit(X_test, X_train) # # sample_weight = np.maximum(1e-6, r_kmm.ratio(X_train)) # # error = regression_metric(X_train, y_train, X_test, y_test, # # sample_weight=sample_weight) # # rows.append(dict(weight="kmm", name=name, split=split, error=error)) # # # Logistic Regression (Linear) # # r_logreg = LogisticRegressionDensityRatioEstimator(C=1.0, seed=seed) # # r_logreg.fit(X_test, X_train) # # sample_weight = np.maximum(1e-6, r_logreg.ratio(X_train).numpy()) # # error = regression_metric(X_train, y_train, X_test, y_test, # # sample_weight=sample_weight) # # rows.append(dict(weight="logreg", name=name, split=split, error=error)) # # # Logistic Regression (MLP) # # r_mlp = MLPDensityRatioEstimator(num_layers=2, num_units=32, # # activation="relu", seed=seed) # # r_mlp.compile(optimizer=optimizer, metrics=["accuracy"]) # # r_mlp.fit(X_test, X_train, epochs=epochs, batch_size=batch_size) # # sample_weight = np.maximum(1e-6, r_mlp.ratio(X_train).numpy()) # # error = regression_metric(X_train, y_train, X_test, y_test, # # sample_weight=sample_weight) # # rows.append(dict(weight="mlp", name=name, split=split, error=error)) # for whiten in [True]: for kernel_name, kernel_cls in kernels.items(): for num_inducing_points in [100, 300, 500]: for use_ard in [False, True]: # # Gaussian Processes (low-dimensional) # gpdre = GaussianProcessDensityRatioEstimator( # input_dim=num_features_low, # kernel_cls=kernel_cls, # num_inducing_points=num_inducing_points, # inducing_index_points_initializer=KMeans(X_low, seed=9), # vgp_cls=SVGP, # whiten=True, # jitter=jitter, # seed=9) # gpdre.compile(optimizer=optimizer) # gpdre.fit(X_test_low, X_train_low, # epochs=epochs, # batch_size=batch_size, # buffer_size=buffer_size) # for prop_name, prop in props.items(): # r_prop = gpdre.ratio(X_train_low, convert_to_tensor_fn=prop) # error = regression_metric(X_train, y_train, X_test, y_test, # sample_weight=r_prop.numpy()) # rows.append(dict(name=name, error=error, projection="low", # kernel_name=kernel_name, # # whiten=whiten, # weight=prop_name)) # Gaussian Processes gpdre = GaussianProcessDensityRatioEstimator( input_dim=num_features, kernel_cls=kernel_cls, use_ard=use_ard, num_inducing_points=num_inducing_points, inducing_index_points_initializer=KMeans( X, seed=split), vgp_cls=SVGP, whiten=True, jitter=jitter, seed=split) gpdre.compile(optimizer=optimizer) gpdre.fit(X_test, X_train, epochs=epochs, batch_size=batch_size, buffer_size=buffer_size) for prop_name, prop in props.items(): r_prop = gpdre.ratio(X_train, convert_to_tensor_fn=prop) error = regression_metric( X_train, y_train, X_test, y_test, sample_weight=r_prop.numpy()) rows.append( dict( name=name, error=error, projection="none", kernel_name=kernel_name, split=split, # whiten=whiten, use_ard=use_ard, epochs=epochs, num_inducing_points=num_inducing_points, weight=prop_name)) data = pd.DataFrame(rows) data.to_csv(str(summary_path.joinpath(f"{myname}.csv"))) # data = pd.DataFrame(rows) # data.to_csv(str(summary_path.joinpath(f"{myname}.csv"))) return 0
def main(filename, dataset_filename, num_inducing_points, noise_variance, num_epochs, batch_size, quadrature_size, learning_rate, beta1, beta2, jitter, seed): random_state = np.random.RandomState(seed) # Don't get confused -- train and test refer to those of the downstream # prediction task. Both train and test are used for training of the DRE # here. (X_train, y_train), (X_test, y_test) = load_hdf5(dataset_filename) X, y = make_classification_dataset(X_test, X_train) num_train, input_dim = X.shape model = build_model(input_dim=input_dim, inputs=X, num_inducing_points=num_inducing_points, jitter=jitter) optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, beta_1=beta1, beta_2=beta2) @tf.function def nelbo(X_batch, y_batch): qf = model(X_batch) ell = qf.surrogate_posterior_expected_log_likelihood( observations=y_batch, log_likelihood_fn=log_likelihood, quadrature_size=quadrature_size) kl = qf.surrogate_posterior_kl_divergence_prior() kl_weight = get_kl_weight(num_train, batch_size) return -ell + kl_weight * kl @tf.function def train_step(X_batch, y_batch): with tf.GradientTape() as tape: loss = nelbo(X_batch, y_batch) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) return loss dataset = tf.data.Dataset.from_tensor_slices((X, y)) \ .shuffle(seed=seed, buffer_size=SHUFFLE_BUFFER_SIZE) \ .batch(batch_size, drop_remainder=True) keys = [ "inducing_index_points", "variational_inducing_observations_loc", "variational_inducing_observations_scale", "log_observation_noise_variance", "log_amplitude", "log_length_scale" ] history = defaultdict(list) with trange(num_epochs, unit="epoch") as range_epochs: for epoch in range_epochs: for step, (X_batch, y_batch) in enumerate(dataset): loss = train_step(X_batch, y_batch) range_epochs.set_postfix(loss=to_numpy(loss)) history["loss"].append(loss.numpy()) for key, tensor in zip(keys, model.get_weights()): history[key].append(tensor) dre = make_dre(model) importance_weights = dre(X_train).mode().numpy() with h5py.File(filename, 'w') as f: f.create_dataset("importance_weights", data=importance_weights) return 0
def main(myname, summary_dir, seed): summary_path = Path(summary_dir).joinpath("liacc") summary_path.mkdir(parents=True, exist_ok=True) names = [ "abalone", "ailerons", "bank32", "bank8", "cali", "cpuact", "elevators", "puma8" ] rows = [] for name in names: output_path = summary_path.joinpath(f"{name}") output_path.mkdir(parents=True, exist_ok=True) data_path = get_path(name, kind="data", data_home="results/") data_mat = loadmat(data_path) X_trains = get_splits(data_mat, key="X") Y_trains = get_splits(data_mat, key="Y") X_tests = get_splits(data_mat, key="Xtest") Y_tests = get_splits(data_mat, key="Ytest") sample_weights = get_splits(data_mat, key="ideal") result_path = get_path(name, kind="results_CV", data_home="results/20200530/") results_mat = loadmat(result_path) projs = get_splits(results_mat, key="all_W") for split, (X_train, Y_train, X_test, Y_test, proj, sample_weight) in \ enumerate(zip(X_trains, Y_trains, X_tests, Y_tests, projs, sample_weights)): y_train = Y_train.squeeze(axis=-1) y_test = Y_test.squeeze(axis=-1) # X, _ = make_classification_dataset(X_test, X_train) # num_features = X.shape[-1] X_train_low = X_train.dot(proj) X_test_low = X_test.dot(proj) X_low, _ = make_classification_dataset(X_test_low, X_train_low) num_features_low = X_low.shape[-1] error = regression_metric(X_train_low, y_train, X_test_low, y_test) rows.append( dict(weight="uniform", name=name, split=split, error=error, projection="low")) error = regression_metric( X_train_low, y_train, X_test_low, y_test, sample_weight=sample_weight.squeeze(axis=-1)) rows.append( dict(weight="exact", name=name, split=split, error=error, projection="low")) # # Gaussian Processes (full-dimensional) # gpdre = GaussianProcessDensityRatioEstimator( # input_dim=num_features, kernel_cls=kernels["sqr_exp"], # num_inducing_points=num_inducing_points, # inducing_index_points_initializer=KMeans(X, seed=split), # vgp_cls=SVGP, whiten=True, jitter=jitter, seed=split) # gpdre.compile(optimizer=optimizer) # gpdre.fit(X_test, X_train, # epochs=epochs, # batch_size=batch_size, # buffer_size=buffer_size) # for prop_name, prop in props.items(): # r_prop = gpdre.ratio(X_train, convert_to_tensor_fn=prop) # error = regression_metric(X_train, y_train, X_test, y_test, # sample_weight=r_prop.numpy()) # rows.append(dict(name=name, split=split, error=error, # projection="none", kernel_name="sqr_exp", # use_ard=True, epochs=epochs, weight=prop_name, # num_inducing_points=num_inducing_points, # num_features=num_features)) # Gaussian Processes (low-dimensional) gpdre = GaussianProcessDensityRatioEstimator( input_dim=num_features_low, kernel_cls=kernels["sqr_exp"], num_inducing_points=num_inducing_points, inducing_index_points_initializer=KMeans(X_low, seed=split), vgp_cls=SVGP, whiten=True, jitter=jitter, seed=split) gpdre.compile(optimizer=optimizer) gpdre.fit(X_test_low, X_train_low, epochs=epochs, batch_size=batch_size, buffer_size=buffer_size) for prop_name, prop in props.items(): r_prop = gpdre.ratio(X_train_low, convert_to_tensor_fn=prop) error = regression_metric(X_train_low, y_train, X_test_low, y_test, sample_weight=r_prop.numpy()) rows.append( dict(name=name, split=split, error=error, projection="low", kernel_name="sqr_exp", use_ard=True, epochs=epochs, weight=prop_name, num_inducing_points=num_inducing_points, num_features=num_features_low)) data = pd.DataFrame(rows) data.to_csv(str(summary_path.joinpath(f"{myname}.csv"))) # data = pd.DataFrame(rows) # data.to_csv(str(summary_path.joinpath(f"{myname}.csv"))) return 0