def main(filename, dataset_filename, optimizer, num_layers, num_units,
activation, epochs, batch_size, l1_factor, l2_factor, seed):
random_state = np.random.RandomState(seed)
# Don't get confused -- train and test refer to those of the downstream
# prediction task. Both train and test are used for training of the DRE
# here.
(X_train, y_train), (X_test, y_test) = load_hdf5(dataset_filename)
X, y = make_classification_dataset(X_test, X_train)
# Model specification
model = build_model(output_dim=1,
num_layers=num_layers,
num_units=num_units,
activation=activation,
kernel_regularizer=l1_l2(l1_factor, l2_factor),
seed=seed)
model.compile(loss=binary_crossentropy_from_logits,
optimizer=optimizer,
metrics=["accuracy"])
hist = model.fit(X, y, batch_size=batch_size, epochs=epochs, shuffle=True)
importance_weights = np.exp(model.predict(X_train)).squeeze()
with h5py.File(filename, 'w') as f:
f.create_dataset("importance_weights", data=importance_weights)
return 0
def main(name, summary_dir, seed):
summary_path = Path(summary_dir) # .joinpath(name)
summary_path.mkdir(parents=True, exist_ok=True)
r = SugiyamaKrauledatMuellerDensityRatioMarginals()
rows = []
for seed in range(num_seeds):
# (X_train, y_train), (X_test, y_test) = r.train_test_split(X, y, seed=seed)
(X_train, y_train), (X_test, y_test) = r.make_covariate_shift_dataset(
num_test, num_train, class_posterior_fn=class_posterior, threshold=0.5,
seed=seed)
X, s = make_classification_dataset(X_test, X_train)
# # Uniform
# acc = metric(X_train, y_train, X_test, y_test, random_state=seed)
# rows.append(dict(weight="uniform", acc=acc, seed=seed))
# # Exact
# acc = metric(X_train, y_train, X_test, y_test,
# sample_weight=r.ratio(X_train).numpy(), random_state=seed)
# rows.append(dict(weight="exact", acc=acc, seed=seed))
for epochs in [500, 1000, 1500, 2000]:
# for sparsity_factor in sparsity_factors:
num_inducing_points = int(len(X) * sparsity_factor)
# Gaussian Processes
gpdre = GaussianProcessDensityRatioEstimator(
input_dim=num_features,
kernel_cls=Matern52,
use_ard=use_ard,
num_inducing_points=num_inducing_points,
inducing_index_points_initializer=KMeans(X, seed=seed),
vgp_cls=SVGP,
whiten=True,
jitter=jitter,
seed=seed)
gpdre.compile(optimizer=optimizer)
gpdre.fit(X_test, X_train, epochs=epochs, batch_size=batch_size,
buffer_size=buffer_size)
for prop_name, prop in props.items():
r_prop = gpdre.ratio(X_train, convert_to_tensor_fn=prop)
acc = metric(X_train, y_train, X_test, y_test,
sample_weight=r_prop.numpy(), random_state=seed)
rows.append(dict(weight=prop_name, acc=acc, seed=seed,
sparsity_factor=sparsity_factor,
use_ard=use_ard, epochs=epochs))
data = pd.DataFrame(rows)
data.to_csv(str(summary_path.joinpath(f"{name}.csv")))
return 0
fig, ax = plt.subplots()
ax.plot(X_grid, r.ratio(X_grid), c='k', label=r"$r(x) = \exp{f(x)}$")
ax.set_xlabel('$x$')
ax.set_ylabel('$r(x)$')
ax.legend()
plt.show()
# %%
# Create classification dataset.
X_p, X_q = r.make_dataset(num_train, seed=dataset_seed)
X_train, y_train = make_classification_dataset(X_p, X_q)
# %%
# Dataset visualized against the Bayes optimal classifier.
fig, ax = plt.subplots()
ax.plot(X_grid, r.prob(X_grid), c='k',
label=r"$\pi(x) = \sigma(f(x))$")
ax.scatter(X_train, y_train, c=y_train, s=12.**2,
marker='s', alpha=0.1, cmap="coolwarm_r")
ax.set_yticks([0, 1])
ax.set_yticklabels([r"$x_q \sim q(x)$", r"$x_p \sim p(x)$"])
ax.set_xlabel('$x$')
ax.legend()
def main(name, summary_dir, seed):
summary_path = Path(summary_dir).joinpath("sugiyama")
summary_path.mkdir(parents=True, exist_ok=True)
r = SugiyamaKrauledatMuellerDensityRatioMarginals()
rows = []
for seed in range(num_seeds):
# (X_train, y_train), (X_test, y_test) = r.train_test_split(X, y, seed=seed)
(X_train, y_train), (X_test, y_test) = r.make_covariate_shift_dataset(
num_test,
num_train,
class_posterior_fn=class_posterior,
threshold=0.5,
seed=seed)
X, s = make_classification_dataset(X_test, X_train)
# Uniform
acc = metric(X_train, y_train, X_test, y_test, random_state=seed)
rows.append(
dict(weight="uniform", acc=acc, seed=seed, dataset_seed=seed))
# Exact
acc = metric(X_train,
y_train,
X_test,
y_test,
sample_weight=r.ratio(X_train).numpy(),
random_state=seed)
rows.append(dict(weight="exact", acc=acc, seed=seed,
dataset_seed=seed))
# RuLSIF
r_rulsif = RuLSIFDensityRatioEstimator(alpha=1e-6)
r_rulsif.fit(X_test, X_train)
sample_weight = np.maximum(1e-6, r_rulsif.ratio(X_train))
acc = metric(X_train,
y_train,
X_test,
y_test,
sample_weight=sample_weight,
random_state=seed)
rows.append(
dict(weight="rulsif", acc=acc, seed=seed, dataset_seed=seed))
# KLIEP
# sigmas = [0.1, 0.25, 0.5, 0.75, 1.0]
sigmas = list(np.maximum(0.25 * np.arange(5), 0.1))
r_kliep = KLIEPDensityRatioEstimator(sigmas=sigmas, seed=seed)
r_kliep.fit(X_test, X_train)
sample_weight = np.maximum(1e-6, r_kliep.ratio(X_train))
acc = metric(X_train,
y_train,
X_test,
y_test,
sample_weight=sample_weight,
random_state=seed)
rows.append(dict(weight="kliep", acc=acc, seed=seed,
dataset_seed=seed))
# KMM
r_kmm = KMMDensityRatioEstimator(B=1000.0)
r_kmm.fit(X_test, X_train)
sample_weight = np.maximum(1e-6, r_kmm.ratio(X_train))
acc = metric(X_train,
y_train,
X_test,
y_test,
sample_weight=sample_weight,
random_state=seed)
rows.append(dict(weight="kmm", acc=acc, seed=seed, dataset_seed=seed))
# Logistic Regression (Linear)
r_logreg = LogisticRegressionDensityRatioEstimator(seed=seed)
r_logreg.fit(X_test, X_train)
sample_weight = np.maximum(1e-6, r_logreg.ratio(X_train).numpy())
acc = metric(X_train,
y_train,
X_test,
y_test,
sample_weight=sample_weight,
random_state=seed)
rows.append(
dict(weight="logreg", acc=acc, seed=seed, dataset_seed=seed))
# Logistic Regression (MLP)
r_mlp = MLPDensityRatioEstimator(num_layers=1,
num_units=8,
activation="tanh",
seed=seed)
r_mlp.compile(optimizer=optimizer, metrics=["accuracy"])
r_mlp.fit(X_test, X_train, epochs=epochs, batch_size=batch_size)
sample_weight = np.maximum(1e-6, r_mlp.ratio(X_train).numpy())
acc = metric(X_train,
y_train,
X_test,
y_test,
sample_weight=sample_weight,
random_state=seed)
rows.append(dict(weight="mlp", acc=acc, seed=seed, dataset_seed=seed))
# Gaussian Processes
gpdre = GaussianProcessDensityRatioEstimator(
input_dim=num_features,
kernel_cls=Matern52,
num_inducing_points=num_inducing_points,
inducing_index_points_initializer=KMeans(X, seed=seed),
vgp_cls=SVGP,
whiten=True,
jitter=jitter,
seed=seed)
gpdre.compile(optimizer=optimizer)
gpdre.fit(X_test,
X_train,
epochs=epochs,
batch_size=batch_size,
buffer_size=buffer_size)
for prop_name, prop in props.items():
r_prop = gpdre.ratio(X_train, convert_to_tensor_fn=prop)
acc = metric(X_train,
y_train,
X_test,
y_test,
sample_weight=r_prop.numpy(),
random_state=seed)
rows.append(
dict(weight=prop_name, acc=acc, seed=seed, dataset_seed=seed))
data = pd.DataFrame(rows)
data.to_csv(str(summary_path.joinpath(f"{name}.csv")))
return 0
X2, X1 = np.mgrid[-4:9:200j, -6:8:200j]
X_grid = np.dstack((X1, X2))
# %%
def class_posterior(x1, x2):
return 0.5 * (1 + tf.tanh(x1 - tf.nn.relu(-x2)))
# %%
r = SugiyamaKrauledatMuellerDensityRatioMarginals()
(X_train, y_train), (X_test, y_test) = r.make_covariate_shift_dataset(
num_test, num_train, class_posterior_fn=class_posterior, threshold=0.5,
seed=dataset_seed)
X, s = make_classification_dataset(X_test, X_train)
# %%
fig, ax = plt.subplots()
ax.set_title(r"Train $p_{\mathrm{tr}}(\mathbf{x})$ and "
r"test $p_{\mathrm{te}}(\mathbf{x})$ distributions")
contours_train = ax.contour(X1, X2, r.bot.prob(X_grid), cmap="Blues")
contours_test = ax.contour(X1, X2, r.top.prob(X_grid), cmap="Oranges",
linestyles="--")
ax.clabel(contours_train, fmt="%.2f")
ax.clabel(contours_test, fmt="%.2f")
ax.set_xlabel(r'$x_1$')
def main(name, width, aspect, extension, output_dir):
figsize = size(width, aspect)
suffix = f"{width:.0f}x{width/aspect:.0f}"
# preamble
rc = {
"figure.figsize": figsize,
"font.serif": ["Times New Roman"],
"text.usetex": True,
}
sns.set(context="paper",
style="ticks",
palette="colorblind",
font="serif",
rc=rc)
output_path = Path(output_dir).joinpath(name)
output_path.mkdir(parents=True, exist_ok=True)
# /preamble
xmin, xmax = -6, 8
ymin, ymax = -4, 9
X2, X1 = np.mgrid[ymin:ymax:200j, xmin:xmax:200j]
X_grid = np.dstack((X1, X2))
r = SugiyamaKrauledatMuellerDensityRatioMarginals()
(X_train, y_train), (X_test, y_test) = r.make_covariate_shift_dataset(
NUM_TEST,
NUM_TRAIN,
class_posterior_fn=class_posterior,
seed=DATASET_SEED)
X, s = make_classification_dataset(X_test, X_train)
# Figure 3
fig, ax = plt.subplots()
ax.set_title(r"$f(\mathbf{x}) = "
r"\log p_{\mathrm{te}}(\mathbf{x}) - "
r"\log p_{\mathrm{tr}}(\mathbf{x})$")
contours = ax.contour(X1, X2, r.logit(X_grid).numpy(), cmap="PuOr")
fig.colorbar(contours, ax=ax)
ax.clabel(contours, fmt="%.2f")
ax.set_xlabel(r'$x_1$')
ax.set_ylabel(r'$x_2$')
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
for ext in extension:
fig.savefig(output_path.joinpath(f"logit_{suffix}.{ext}"),
bbox_inches="tight")
plt.show()
# Figure 4
fig, ax = plt.subplots()
ax.set_title(r"$r(\mathbf{x}) = \exp(f(\mathbf{x}))$")
contours = ax.contour(X1, X2, r.ratio(X_grid).numpy(), cmap="PuOr")
fig.colorbar(contours, ax=ax)
ax.clabel(contours, fmt="%.2f")
ax.set_xlabel(r'$x_1$')
ax.set_ylabel(r'$x_2$')
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
for ext in extension:
fig.savefig(output_path.joinpath(f"ratio_{suffix}.{ext}"),
bbox_inches="tight")
plt.show()
# Figure 5
fig, ax = plt.subplots()
ax.set_title(r"$P(s=1|\mathbf{x}) = \sigma(f(\mathbf{x}))$")
contours = ax.contour(X1, X2, r.prob(X_grid).numpy(), cmap="PuOr")
ax.scatter(*X.T, c=r.prob(X).numpy(), cmap="PuOr", alpha=0.6)
fig.colorbar(contours, ax=ax)
ax.clabel(contours, fmt="%.2f")
ax.set_xlabel(r'$x_1$')
ax.set_ylabel(r'$x_2$')
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
for ext in extension:
fig.savefig(output_path.joinpath(f"prob_{suffix}.{ext}"),
bbox_inches="tight")
plt.show()
model_uniform_train = LogisticRegression(C=1.0, random_state=SEED)
model_uniform_train.fit(X_train, y_train)
model_uniform_test = LogisticRegression(C=1.0, random_state=SEED)
model_uniform_test.fit(X_test, y_test)
model_exact_train = LogisticRegression(C=1.0, random_state=SEED)
model_exact_train.fit(X_train,
y_train,
sample_weight=r.ratio(X_train).numpy())
p_grid_uniform_train = model_uniform_train.predict_proba(X_grid.reshape(-1, 2)) \
.reshape(200, 200, 2)
p_grid_uniform_test = model_uniform_test.predict_proba(X_grid.reshape(-1, 2)) \
.reshape(200, 200, 2)
p_grid_exact_train = model_exact_train.predict_proba(X_grid.reshape(-1, 2)) \
.reshape(200, 200, 2)
# Figure 7
fig, ax = plt.subplots()
ax.set_title("Without importance sampling (uniform weights)")
contours = ax.contour(X1, X2, p_grid_uniform_train[..., -1], cmap="RdYlBu")
fig.colorbar(contours, ax=ax)
ax.clabel(contours, fmt="%.2f")
ax.scatter(*X_train.T, c=y_train, cmap="RdYlBu", alpha=0.8, label="train")
ax.scatter(*X_test.T,
marker='x',
c=y_test,
cmap="RdYlBu",
alpha=0.2,
label="test")
ax.legend()
ax.set_xlabel(r'$x_1$')
ax.set_ylabel(r'$x_2$')
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
for ext in extension:
fig.savefig(output_path.joinpath(f"logistic_uniform_{suffix}.{ext}"),
bbox_inches="tight")
plt.show()
# Figure 8
fig, ax = plt.subplots()
ax.set_title("With importance sampling (exact density ratio)")
contours = ax.contour(X1, X2, p_grid_exact_train[..., -1], cmap="RdYlBu")
fig.colorbar(contours, ax=ax)
ax.clabel(contours, fmt="%.2f")
ax.scatter(*X_train.T,
c=y_train,
s=r.ratio(X_train).numpy(),
cmap="RdYlBu",
alpha=0.8,
label="train")
ax.scatter(
*X_test.T,
marker='x',
c=y_test, # s=r.ratio(X_test).numpy(),
cmap="RdYlBu",
alpha=0.2,
label="test")
ax.legend()
ax.set_xlabel(r'$x_1$')
ax.set_ylabel(r'$x_2$')
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
for ext in extension:
fig.savefig(output_path.joinpath(f"logistic_exact_{suffix}.{ext}"),
bbox_inches="tight")
plt.show()
# Figure 9
fig, ax = plt.subplots()
contours1 = ax.contour(X1,
X2,
p_grid_uniform_train[..., -1],
linestyles="solid",
levels=1,
zorder=-1,
cmap="Blues")
contours2 = ax.contour(X1,
X2,
p_grid_uniform_test[..., -1],
linestyles="dashed",
levels=1,
zorder=-1,
cmap="Oranges")
contours3 = ax.contour(X1,
X2,
p_grid_exact_train[..., -1],
linestyles="dotted",
levels=1,
zorder=-1,
cmap="Greens")
contours4 = ax.contour(X1,
X2,
class_posterior(X1, X2).numpy(),
linestyles="dashdot",
levels=1,
zorder=-1,
cmap="Purples")
# fig.colorbar(contours, ax=ax)
# ax.clabel(contours1, fmt="unweighted", fontsize="smaller")
# ax.clabel(contours2, fmt="test", fontsize="smaller")
# ax.clabel(contours3, fmt="ideal", fontsize="smaller")
ax.scatter(*X_train.T, c=y_train, cmap="RdYlBu", alpha=0.8, label="train")
ax.scatter(*X_test.T,
marker='x',
c=y_test,
cmap="RdYlBu",
alpha=0.2,
label="test")
ax.legend()
ax.set_xlabel(r'$x_1$')
ax.set_ylabel(r'$x_2$')
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
for ext in extension:
fig.savefig(output_path.joinpath(f"dataset_{suffix}.{ext}"),
bbox_inches="tight")
plt.show()
# Figure 2
fig, ax = plt.subplots()
# ax.set_title(r"Train $p_{\mathrm{tr}}(\mathbf{x})$ and "
# r"test $p_{\mathrm{te}}(\mathbf{x})$ distributions")
contours_train = ax.contour(X1, X2, r.bot.prob(X_grid), cmap="Oranges")
contours_test = ax.contour(X1,
X2,
r.top.prob(X_grid),
cmap="Purples",
linestyles="--")
contours = ax.contour(X1,
X2,
class_posterior(X1, X2).numpy(),
levels=1,
zorder=-1,
linestyles="dashdot",
cmap="Purples")
ax.clabel(contours_train, fmt="%.2f")
ax.clabel(contours_test, fmt="%.2f")
ax.set_xlabel(r'$x_1$')
ax.set_ylabel(r'$x_2$')
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
for ext in extension:
fig.savefig(output_path.joinpath(f"distribution_{suffix}.{ext}"),
bbox_inches="tight")
plt.show()
return 0
def main(myname, summary_dir, seed):
summary_path = Path(summary_dir).joinpath("liacc")
summary_path.mkdir(parents=True, exist_ok=True)
names = [
"abalone", "ailerons", "bank32", "bank8", "cali", "cpuact",
"elevators", "puma8"
][4:]
rows = []
for name in names:
output_path = summary_path.joinpath(f"{name}")
output_path.mkdir(parents=True, exist_ok=True)
data_path = get_data_path(name)
data_mat = loadmat(data_path)
results_path = get_results_path(name)
results_mat = loadmat(results_path)
X_trains = get_splits(data_mat, key="X")
y_trains = get_splits(data_mat, key="Y")
X_tests = get_splits(data_mat, key="Xtest")
y_tests = get_splits(data_mat, key="Ytest")
weights = get_splits(data_mat, key="ideal")
# X_train = X_trains[-1]
# Y_train = y_trains[-1]
# X_test = X_tests[-1]
# Y_test = y_tests[-1]
# weight = weights[-1]
# proj = results_mat["W"]
# X_train_low = X_train.dot(proj)
# X_test_low = X_test.dot(proj)
for split, (X_train, Y_train, X_test, Y_test, weight) in \
enumerate(zip(X_trains, y_trains, X_tests, y_tests, weights)):
X, _ = make_classification_dataset(X_test, X_train)
# X_low, _ = make_classification_dataset(X_test_low, X_train_low)
num_features = X.shape[-1]
# num_features_low = X_low.shape[-1]
y_train = Y_train.squeeze(axis=-1)
y_test = Y_test.squeeze(axis=-1)
# sample_weight = weight.squeeze(axis=-1)
# error = regression_metric(X_train, y_train, X_test, y_test)
# rows.append(dict(weight="uniform", name=name, error=error, projection="none"))
# # rows.append(dict(weight="uniform", name=name, split=split, error=error))
# error = regression_metric(X_train, y_train, X_test, y_test,
# sample_weight=sample_weight)
# rows.append(dict(weight="exact", name=name, error=error, projection="none"))
# rows.append(dict(weight="exact", name=name, split=split, error=error))
# # # RuLSIF
# # r_rulsif = RuLSIFDensityRatioEstimator(alpha=1e-6)
# # r_rulsif.fit(X_test, X_train)
# # sample_weight = np.maximum(1e-6, r_rulsif.ratio(X_train))
# # error = regression_metric(X_train, y_train, X_test, y_test,
# # sample_weight=sample_weight)
# # rows.append(dict(weight="rulsif", name=name, split=split, error=error))
# # # # KLIEP
# # # r_kliep = KLIEPDensityRatioEstimator(seed=seed)
# # # r_kliep.fit(X_test, X_train)
# # # sample_weight = np.maximum(1e-6, r_kliep.ratio(X_train))
# # # error = regression_metric(X_train, y_train, X_test, y_test,
# # # sample_weight=sample_weight)
# # # rows.append(dict(weight="kliep", name=name, split=split, error=error))
# # # KMM
# # r_kmm = KMMDensityRatioEstimator(B=1000.0)
# # r_kmm.fit(X_test, X_train)
# # sample_weight = np.maximum(1e-6, r_kmm.ratio(X_train))
# # error = regression_metric(X_train, y_train, X_test, y_test,
# # sample_weight=sample_weight)
# # rows.append(dict(weight="kmm", name=name, split=split, error=error))
# # # Logistic Regression (Linear)
# # r_logreg = LogisticRegressionDensityRatioEstimator(C=1.0, seed=seed)
# # r_logreg.fit(X_test, X_train)
# # sample_weight = np.maximum(1e-6, r_logreg.ratio(X_train).numpy())
# # error = regression_metric(X_train, y_train, X_test, y_test,
# # sample_weight=sample_weight)
# # rows.append(dict(weight="logreg", name=name, split=split, error=error))
# # # Logistic Regression (MLP)
# # r_mlp = MLPDensityRatioEstimator(num_layers=2, num_units=32,
# # activation="relu", seed=seed)
# # r_mlp.compile(optimizer=optimizer, metrics=["accuracy"])
# # r_mlp.fit(X_test, X_train, epochs=epochs, batch_size=batch_size)
# # sample_weight = np.maximum(1e-6, r_mlp.ratio(X_train).numpy())
# # error = regression_metric(X_train, y_train, X_test, y_test,
# # sample_weight=sample_weight)
# # rows.append(dict(weight="mlp", name=name, split=split, error=error))
# for whiten in [True]:
for kernel_name, kernel_cls in kernels.items():
for num_inducing_points in [100, 300, 500]:
for use_ard in [False, True]:
# # Gaussian Processes (low-dimensional)
# gpdre = GaussianProcessDensityRatioEstimator(
# input_dim=num_features_low,
# kernel_cls=kernel_cls,
# num_inducing_points=num_inducing_points,
# inducing_index_points_initializer=KMeans(X_low, seed=9),
# vgp_cls=SVGP,
# whiten=True,
# jitter=jitter,
# seed=9)
# gpdre.compile(optimizer=optimizer)
# gpdre.fit(X_test_low, X_train_low,
# epochs=epochs,
# batch_size=batch_size,
# buffer_size=buffer_size)
# for prop_name, prop in props.items():
# r_prop = gpdre.ratio(X_train_low, convert_to_tensor_fn=prop)
# error = regression_metric(X_train, y_train, X_test, y_test,
# sample_weight=r_prop.numpy())
# rows.append(dict(name=name, error=error, projection="low",
# kernel_name=kernel_name,
# # whiten=whiten,
# weight=prop_name))
# Gaussian Processes
gpdre = GaussianProcessDensityRatioEstimator(
input_dim=num_features,
kernel_cls=kernel_cls,
use_ard=use_ard,
num_inducing_points=num_inducing_points,
inducing_index_points_initializer=KMeans(
X, seed=split),
vgp_cls=SVGP,
whiten=True,
jitter=jitter,
seed=split)
gpdre.compile(optimizer=optimizer)
gpdre.fit(X_test,
X_train,
epochs=epochs,
batch_size=batch_size,
buffer_size=buffer_size)
for prop_name, prop in props.items():
r_prop = gpdre.ratio(X_train,
convert_to_tensor_fn=prop)
error = regression_metric(
X_train,
y_train,
X_test,
y_test,
sample_weight=r_prop.numpy())
rows.append(
dict(
name=name,
error=error,
projection="none",
kernel_name=kernel_name,
split=split,
# whiten=whiten,
use_ard=use_ard,
epochs=epochs,
num_inducing_points=num_inducing_points,
weight=prop_name))
data = pd.DataFrame(rows)
data.to_csv(str(summary_path.joinpath(f"{myname}.csv")))
# data = pd.DataFrame(rows)
# data.to_csv(str(summary_path.joinpath(f"{myname}.csv")))
return 0
def main(filename, dataset_filename, num_inducing_points, noise_variance,
num_epochs, batch_size, quadrature_size, learning_rate, beta1, beta2,
jitter, seed):
random_state = np.random.RandomState(seed)
# Don't get confused -- train and test refer to those of the downstream
# prediction task. Both train and test are used for training of the DRE
# here.
(X_train, y_train), (X_test, y_test) = load_hdf5(dataset_filename)
X, y = make_classification_dataset(X_test, X_train)
num_train, input_dim = X.shape
model = build_model(input_dim=input_dim,
inputs=X,
num_inducing_points=num_inducing_points,
jitter=jitter)
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate,
beta_1=beta1,
beta_2=beta2)
@tf.function
def nelbo(X_batch, y_batch):
qf = model(X_batch)
ell = qf.surrogate_posterior_expected_log_likelihood(
observations=y_batch,
log_likelihood_fn=log_likelihood,
quadrature_size=quadrature_size)
kl = qf.surrogate_posterior_kl_divergence_prior()
kl_weight = get_kl_weight(num_train, batch_size)
return -ell + kl_weight * kl
@tf.function
def train_step(X_batch, y_batch):
with tf.GradientTape() as tape:
loss = nelbo(X_batch, y_batch)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients,
model.trainable_variables))
return loss
dataset = tf.data.Dataset.from_tensor_slices((X, y)) \
.shuffle(seed=seed, buffer_size=SHUFFLE_BUFFER_SIZE) \
.batch(batch_size, drop_remainder=True)
keys = [
"inducing_index_points", "variational_inducing_observations_loc",
"variational_inducing_observations_scale",
"log_observation_noise_variance", "log_amplitude", "log_length_scale"
]
history = defaultdict(list)
with trange(num_epochs, unit="epoch") as range_epochs:
for epoch in range_epochs:
for step, (X_batch, y_batch) in enumerate(dataset):
loss = train_step(X_batch, y_batch)
range_epochs.set_postfix(loss=to_numpy(loss))
history["loss"].append(loss.numpy())
for key, tensor in zip(keys, model.get_weights()):
history[key].append(tensor)
dre = make_dre(model)
importance_weights = dre(X_train).mode().numpy()
with h5py.File(filename, 'w') as f:
f.create_dataset("importance_weights", data=importance_weights)
return 0
def main(myname, summary_dir, seed):
summary_path = Path(summary_dir).joinpath("liacc")
summary_path.mkdir(parents=True, exist_ok=True)
names = [
"abalone", "ailerons", "bank32", "bank8", "cali", "cpuact",
"elevators", "puma8"
]
rows = []
for name in names:
output_path = summary_path.joinpath(f"{name}")
output_path.mkdir(parents=True, exist_ok=True)
data_path = get_path(name, kind="data", data_home="results/")
data_mat = loadmat(data_path)
X_trains = get_splits(data_mat, key="X")
Y_trains = get_splits(data_mat, key="Y")
X_tests = get_splits(data_mat, key="Xtest")
Y_tests = get_splits(data_mat, key="Ytest")
sample_weights = get_splits(data_mat, key="ideal")
result_path = get_path(name,
kind="results_CV",
data_home="results/20200530/")
results_mat = loadmat(result_path)
projs = get_splits(results_mat, key="all_W")
for split, (X_train, Y_train, X_test, Y_test, proj, sample_weight) in \
enumerate(zip(X_trains, Y_trains, X_tests, Y_tests, projs, sample_weights)):
y_train = Y_train.squeeze(axis=-1)
y_test = Y_test.squeeze(axis=-1)
# X, _ = make_classification_dataset(X_test, X_train)
# num_features = X.shape[-1]
X_train_low = X_train.dot(proj)
X_test_low = X_test.dot(proj)
X_low, _ = make_classification_dataset(X_test_low, X_train_low)
num_features_low = X_low.shape[-1]
error = regression_metric(X_train_low, y_train, X_test_low, y_test)
rows.append(
dict(weight="uniform",
name=name,
split=split,
error=error,
projection="low"))
error = regression_metric(
X_train_low,
y_train,
X_test_low,
y_test,
sample_weight=sample_weight.squeeze(axis=-1))
rows.append(
dict(weight="exact",
name=name,
split=split,
error=error,
projection="low"))
# # Gaussian Processes (full-dimensional)
# gpdre = GaussianProcessDensityRatioEstimator(
# input_dim=num_features, kernel_cls=kernels["sqr_exp"],
# num_inducing_points=num_inducing_points,
# inducing_index_points_initializer=KMeans(X, seed=split),
# vgp_cls=SVGP, whiten=True, jitter=jitter, seed=split)
# gpdre.compile(optimizer=optimizer)
# gpdre.fit(X_test, X_train,
# epochs=epochs,
# batch_size=batch_size,
# buffer_size=buffer_size)
# for prop_name, prop in props.items():
# r_prop = gpdre.ratio(X_train, convert_to_tensor_fn=prop)
# error = regression_metric(X_train, y_train, X_test, y_test,
# sample_weight=r_prop.numpy())
# rows.append(dict(name=name, split=split, error=error,
# projection="none", kernel_name="sqr_exp",
# use_ard=True, epochs=epochs, weight=prop_name,
# num_inducing_points=num_inducing_points,
# num_features=num_features))
# Gaussian Processes (low-dimensional)
gpdre = GaussianProcessDensityRatioEstimator(
input_dim=num_features_low,
kernel_cls=kernels["sqr_exp"],
num_inducing_points=num_inducing_points,
inducing_index_points_initializer=KMeans(X_low, seed=split),
vgp_cls=SVGP,
whiten=True,
jitter=jitter,
seed=split)
gpdre.compile(optimizer=optimizer)
gpdre.fit(X_test_low,
X_train_low,
epochs=epochs,
batch_size=batch_size,
buffer_size=buffer_size)
for prop_name, prop in props.items():
r_prop = gpdre.ratio(X_train_low, convert_to_tensor_fn=prop)
error = regression_metric(X_train_low,
y_train,
X_test_low,
y_test,
sample_weight=r_prop.numpy())
rows.append(
dict(name=name,
split=split,
error=error,
projection="low",
kernel_name="sqr_exp",
use_ard=True,
epochs=epochs,
weight=prop_name,
num_inducing_points=num_inducing_points,
num_features=num_features_low))
data = pd.DataFrame(rows)
data.to_csv(str(summary_path.joinpath(f"{myname}.csv")))
# data = pd.DataFrame(rows)
# data.to_csv(str(summary_path.joinpath(f"{myname}.csv")))
return 0
