def bgan_synth(synth_dataset,
z_dim,
batch_size=64,
numz=5,
num_iter=1000,
wasserstein=False,
rpath="synth_results",
gen_observed=10,
base_learning_rate=1e-2,
lr_decay=3.,
save_weights=False):
bgan = BGAN(
[synth_dataset.x_dim],
z_dim,
synth_dataset.N,
batch_size=batch_size,
prior_std=10.0,
alpha=1e-3,
J=numz,
M=1,
ml=(numz == 1),
num_classes=1,
wasserstein=wasserstein, # unsupervised only
gen_observed=gen_observed)
print "Starting session"
session = get_session()
tf.global_variables_initializer().run()
print "Starting training loop"
num_train_iter = num_iter
sample_z = np.random.uniform(-1, 1, size=(batch_size, z_dim))
all_aics_fake, all_data_fake, all_dists = [], [], []
for train_iter in range(num_train_iter):
learning_rate = base_learning_rate * np.exp(-lr_decay * min(
1.0, (train_iter * batch_size) / float(synth_dataset.N)))
print learning_rate
batch_z = np.random.uniform(-1, 1, [batch_size, z_dim])
input_batch = synth_dataset.next_batch(batch_size)
_, d_loss = session.run(
[bgan.d_optim, bgan.d_loss],
feed_dict={
bgan.inputs: input_batch,
bgan.z: batch_z,
bgan.d_learning_rate: learning_rate
})
if wasserstein:
session.run(bgan.clip_d, feed_dict={})
g_losses = []
for gi in xrange(bgan.num_gen):
# compute g_sample loss
batch_z = np.random.uniform(-1, 1, [batch_size, z_dim])
_, g_loss = session.run(
[bgan.g_optims[gi], bgan.generation["g_losses"][gi]],
feed_dict={
bgan.z: batch_z,
bgan.g_learning_rate: learning_rate
})
g_losses.append(g_loss)
print "Disc loss = %.2f, Gen loss = %s" % (d_loss, ", ".join(
["%.2f" % gl for gl in g_losses]))
if (train_iter + 1) % 100 == 0:
print "Disc loss = %.2f, Gen loss = %s" % (d_loss, ", ".join(
["%.2f" % gl for gl in g_losses]))
print "Running GMM on sampled data"
fake_data = []
for num_samples in xrange(10):
for gi in xrange(bgan.num_gen):
# collect sample
sample_z = np.random.uniform(-1,
1,
size=(batch_size, z_dim))
sampled_data = session.run(
bgan.generation["gen_samplers"][gi],
feed_dict={bgan.z: sample_z})
fake_data.append(sampled_data)
X_real = synth_dataset.X
X_sample = np.concatenate(fake_data)
all_data_fake.append(X_sample)
aics_fake = gmm_ms(X_sample)
print "Fake number of clusters (AIC estimate):", aics_fake.argmin()
dist, X_trans_real, X_trans_fake = analyze_div(X_real, X_sample)
print "JS div:", dist
fp = FigPrinter((1, 2))
xmin1 = np.min(X_trans_real[:, 0]) - 1.0
xmax1 = np.max(X_trans_real[:, 0]) + 1.0
xmin2 = np.min(X_trans_real[:, 1]) - 1.0
xmax2 = np.max(X_trans_real[:, 1]) + 1.0
fp.ax_arr[0].plot(X_trans_real[:, 0], X_trans_real[:, 1], '.r')
fp.ax_arr[0].set_xlim([xmin1, xmax1])
fp.ax_arr[0].set_ylim([xmin2, xmax2])
fp.ax_arr[1].plot(X_trans_fake[:, 0], X_trans_fake[:, 1], '.g')
fp.ax_arr[1].set_xlim([xmin1, xmax1])
fp.ax_arr[1].set_ylim([xmin2, xmax2])
fp.ax_arr[0].set_aspect('equal', adjustable='box')
fp.ax_arr[1].set_aspect('equal', adjustable='box')
fp.ax_arr[1].set_title("Iter %i" % (train_iter + 1))
fp.print_to_file(
os.path.join(
rpath,
"pca_distribution_%i_%i.png" % (numz, train_iter + 1)))
all_aics_fake.append(aics_fake)
all_dists.append(dist)
if save_weights:
var_dict = {}
for var in tf.trainable_variables():
var_dict[var.name] = session.run(var.name)
np.savez_compressed(
os.path.join(rpath, "weights_%i.npz" % train_iter),
**var_dict)
return {
"data_fake": all_data_fake,
"data_real": synth_dataset.X,
"z_dim": z_dim,
"numz": numz,
"num_iter": num_iter,
"divergences": all_dists,
"all_aics_fake": np.array(all_aics_fake)
}
def bgan_synth(synth_dataset,
z_dim,
batch_size=64,
numz=5,
num_iter=10000,
num_hidden_generator=128,
wasserstein=False,
rpath="synth_results",
base_learning_rate=1e-2,
lr_decay=3,
save_weights=False):
bgan = BGAN_good(
x_dim=[synth_dataset.x_dim],
z_dim=z_dim,
hidden_size=num_hidden_generator,
batch_size=batch_size,
prior_std=10.0,
alpha=1e-1,
J=numz,
M=1,
ml=(numz == 1),
num_classes=1,
)
print("Starting session")
session = get_session()
tf.global_variables_initializer().run()
print("Starting training loop")
num_train_iter = num_iter
# sample_z = np.random.uniform(-1, 1, size=(batch_size, z_dim))
all_aics_fake, all_data_fake, all_dists = [], [], []
for train_iter in range(num_train_iter):
learning_rate = base_learning_rate * np.exp(
-lr_decay * min(1.0, (train_iter * batch_size) /
(num_train_iter * batch_size)))
batch_z = np.random.normal(0, 1, [batch_size, z_dim])
input_batch = synth_dataset.next_batch(batch_size)
_, d_loss = session.run(
[bgan.d_optim, bgan.d_loss],
feed_dict={
bgan.inputs: input_batch,
bgan.z: batch_z,
bgan.d_learning_rate: learning_rate
})
if wasserstein:
session.run(bgan.clip_d, feed_dict={})
g_losses = []
for gi in range(bgan.num_gen):
# compute g_sample loss
batch_z = np.random.normal(0, 1, [batch_size, z_dim])
_, g_loss = session.run(
[bgan.g_optims[gi], bgan.generation["g_losses"][gi]],
feed_dict={
bgan.z: batch_z,
bgan.g_learning_rate: learning_rate
})
g_losses.append(g_loss)
if (train_iter + 1) % 100 == 0:
print("Disc loss = %.2f, Gen loss = %s, lr = %.6f" %
(d_loss, ", ".join(["%.2f" % gl
for gl in g_losses]), learning_rate))
if (train_iter + 1) % 500 == 1:
fake_data = []
for gi in range(bgan.num_gen):
samples = []
for num_samples in range(10):
# collect sample
sample_z = np.random.normal(0, 1, size=(batch_size, z_dim))
sampled_data = session.run(
bgan.generation["gen_samplers"][gi],
feed_dict={bgan.z: sample_z})
samples.append(sampled_data)
fake_data.append(np.concatenate(samples))
X_fake = np.concatenate(fake_data)
X_real = synth_dataset.X
all_data_fake.append(X_fake)
# dist, X_trans_real, X_trans_fake = analyze_div(X_real, X_sample)
# print("JS div:", dist)
fp = FigPrinter((1, 1))
fp.ax_arr.plot(X_real[:, 0], X_real[:, 1], 'ko', alpha=0.2)
num_generator = len(fake_data)
cmap = get_cmap('jet')
for i, X_fake in enumerate(fake_data):
fp.ax_arr.plot(X_fake[:, 0],
X_fake[:, 1],
'o',
color=cmap(i * 1.0 /
max(1, num_generator - 1))[:3],
alpha=0.2)
# fp.ax_arr[0].set_xlim([xmin1, xmax1]);
# fp.ax_arr[0].set_ylim([xmin2, xmax2])
fp.ax_arr.set_aspect('equal', adjustable='box')
fp.ax_arr.set_title("Iter %i" % (train_iter + 1))
fp.print_to_file(
os.path.join(rpath, "distribution_%i_%i.png" %
(numz, train_iter + 1)))
print('PNG saved.')
all_aics_fake.append(0)
all_dists.append(0)
if save_weights:
var_dict = {}
for var in tf.trainable_variables():
var_dict[var.name] = session.run(var.name)
np.savez_compressed(
os.path.join(rpath, "weights_%i.npz" % train_iter),
**var_dict)
return {
"data_fake": all_data_fake,
"data_real": X_real,
"z_dim": z_dim,
"numz": numz,
"num_iter": num_iter,
# "divergences": all_dists,
# "all_aics_fake": np.array(all_aics_fake),
}
def bgan_synth(synth_dataset,
z_dim,
batch_size=64,
numz=5,
num_iter=1000,
wasserstein=False,
rpath="synth_results",
base_learning_rate=1e-2,
lr_decay=3.,
save_weights=False):
bgan = BGAN(
[synth_dataset.x_dim],
z_dim,
synth_dataset.N,
batch_size=batch_size,
prior_std=10.0,
alpha=1e-3,
J=numz,
M=1,
ml=(numz == 1),
num_classes=1,
wasserstein=wasserstein, # unsupervised only
)
print("Starting session")
session = get_session()
tf.global_variables_initializer().run()
print("Starting training loop")
num_train_iter = num_iter
all_aics_fake, all_data_fake, all_dists = [], [], []
it_loss, it_acc, acc = [], [], []
mean = np.random.uniform(-1, 1, size=z_dim)
cov = np.random.uniform(-1, 1, size=(z_dim, z_dim))
cov = np.dot(cov, cov.transpose())
for train_iter in range(num_train_iter):
learning_rate = base_learning_rate * np.exp(-lr_decay * min(
1.0, (train_iter * batch_size) / float(synth_dataset.N)))
print(learning_rate)
batch_z = np.random.uniform(-1, 1, [batch_size, z_dim])
#batch_z = np.random.multivariate_normal(mean, cov, size=batch_size)
#batch_z = np.random.normal(0, 1.0, size=(batch_size, z_dim))
_, input_batch = synth_dataset.next_batch(batch_size)
_, d_loss = session.run(
[bgan.d_optim, bgan.d_loss],
feed_dict={
bgan.inputs: input_batch,
bgan.z: batch_z,
bgan.d_learning_rate: learning_rate
})
if wasserstein:
session.run(bgan.clip_d, feed_dict={})
g_losses = []
for gi in range(bgan.num_gen):
# compute g_sample loss
batch_z = np.random.uniform(-1, 1, [batch_size, z_dim])
#batch_z = np.random.multivariate_normal(mean, cov, size=batch_size)
#batch_z = np.random.normal(0, 1.0, size=(batch_size, z_dim))
_, g_loss, prior = session.run([
bgan.g_optims[gi], bgan.generation["g_losses"][gi],
bgan.generation["g_prior"][gi]
],
feed_dict={
bgan.z: batch_z,
bgan.g_learning_rate:
learning_rate
})
print(prior, gi)
g_losses.append(g_loss)
print("Disc loss = %.2f, Gen loss = %s" %
(d_loss, ", ".join(["%.2f" % gl for gl in g_losses])))
if (train_iter + 1) % 100 == 0:
print("Disc loss = %.2f, Gen loss = %s" %
(d_loss, ", ".join(["%.2f" % gl for gl in g_losses])))
print("Running GMM on sampled data")
fake_data = []
for num_samples in range(10):
for gi in range(bgan.num_gen):
# collect sample
sample_z = np.random.uniform(-1,
1,
size=(batch_size, z_dim))
#sample_z = np.random.multivariate_normal(mean, cov, size=batch_size)
#sample_z = np.random.normal(0, 1.0, size=(batch_size, z_dim))
sampled_data = session.run(
bgan.generation["gen_samplers"][gi],
feed_dict={bgan.z: sample_z})
fake_data.append(sampled_data)
X_real = synth_dataset.X
X_sample = np.concatenate(fake_data)
all_data_fake.append(X_sample)
"""aics_fake = gmm_ms(X_sample)
print ("Fake number of clusters (AIC estimate):", aics_fake.argmin())"""
dist, X_trans_real, X_trans_fake = analyze_div(X_real, X_sample)
print("JS div:", dist)
"""fp = FigPrinter((1,2))
xmin1 = np.min(X_trans_real[:, 0]) - 1.0
xmax1 = np.max(X_trans_real[:, 0]) + 1.0
xmin2 = np.min(X_trans_real[:, 1]) - 1.0
xmax2 = np.max(X_trans_real[:, 1]) + 1.0
fp.ax_arr[0].plot(X_trans_real[:, 0], X_trans_real[:, 1], '.r')
fp.ax_arr[0].set_xlim([xmin1, xmax1]); fp.ax_arr[0].set_ylim([xmin2, xmax2])
fp.ax_arr[1].plot(X_trans_fake[:, 0], X_trans_fake[:, 1], '.g')
fp.ax_arr[1].set_xlim([xmin1, xmax1]); fp.ax_arr[1].set_ylim([xmin2, xmax2])
fp.ax_arr[0].set_aspect('equal', adjustable='box')
fp.ax_arr[1].set_aspect('equal', adjustable='box')
fp.ax_arr[1].set_title("Iter %i" % (train_iter+1))
fp.print_to_file(os.path.join(rpath, "pca_distribution_%i_%i.png" % (numz, train_iter+1)))"""
all_dists.append(dist)
it_loss.append(train_iter + 1)
if save_weights:
var_dict = {}
for var in tf.trainable_variables():
var_dict[var.name] = session.run(var.name)
np.savez_compressed(
os.path.join(rpath, "weights_%i.npz" % train_iter),
**var_dict)
if (train_iter + 1) % 200 == 0:
count_pos = 0
count = 0
for i in range(int(synth_dataset.N_test / batch_size)):
d_logit = session.run(
[bgan.D],
feed_dict={
bgan.inputs:
synth_dataset.X_test[i * batch_size:(i + 1) *
batch_size]
})
d_logit = d_logit[0]
count_pos += np.sum(np.argmax(d_logit, 1))
count += batch_size
acc.append(100.0 * count_pos / count)
it_acc.append(train_iter + 1)
fp = FigPrinter((1, 2))
fp.ax_arr[0].plot(it_loss, all_dists)
fp.ax_arr[0].set_title("JS Divergence")
fp.ax_arr[1].plot(it_acc, acc)
fp.ax_arr[1].set_title("Test Accuracy")
fp.print_to_file(os.path.join(rpath, "divergence_disc.png"))
return {
"data_fake": all_data_fake,
"data_real": synth_dataset.X,
"z_dim": z_dim,
"numz": numz,
"num_iter": num_iter,
"divergences": all_dists,
"it_loss": it_loss,
"acc": acc,
"it_acc": it_acc
}
def bgan_synth(dataset,
synth_dataset,
z_dim,
clip,
batch_size=128,
num_iter=15000,
rpath="synth_results",
base_learning_rate=0.00015,
lr_decay=3.,
save_weights=False):
bgan = BGAN([synth_dataset.x_dim],
z_dim,
synth_dataset.N,
batch_size=batch_size,
lr=0.0002,
alpha=0.00012)
print("Starting session")
session = get_session()
tf.global_variables_initializer().run()
print("Starting training loop")
num_train_iter = num_iter
learning_rate = base_learning_rate
np_samples = []
start_time = time.time()
for train_iter in range(num_train_iter):
batch_z = np.random.normal(0., 1, [batch_size, z_dim])
input_batch = synth_dataset.next_batch(batch_size)
_, d_loss, d_real = session.run(
[bgan.d_optim_adam, bgan.d_loss, bgan.D],
feed_dict={
bgan.inputs: input_batch,
bgan.z: batch_z,
bgan.drop_prob: 0.99,
bgan.d_learning_rate: learning_rate
})
g_losses = []
for gi in range(2):
z = np.random.normal(0., 1, [batch_size, z_dim])
_, gl, d_fake = session.run(
[bgan.g_optims_adam, bgan.g_loss, bgan.D_],
feed_dict={
bgan.z: batch_z,
bgan.drop_prob: 0.99,
bgan.g_learning_rate: learning_rate
})
g_losses.append(gl)
if (train_iter + 1) % 10000 == 0:
fake_data = []
for i in range(10):
sample_z = np.random.normal(0., 1, size=(1000, z_dim))
sampled_data = session.run(bgan.generation["generators"][0],
feed_dict={
bgan.z: sample_z,
bgan.drop_prob: 0.99
})
fake_data.append(sampled_data)
X_real = synth_dataset.next_batch(10000)
X_sample = np.concatenate(fake_data)
if (dataset == 'synth'):
pca = PCA(2)
X_real = pca.fit_transform(X_real)
X_sample = pca.transform(X_sample)
'''np_samples_ = X_sample[::1]
fp = FigPrinter((1,2))
xmin1 = np.min(X_real[:, 0]) - 0.5
xmax1 = np.max(X_real[:, 0]) + 0.5
xmin2 = np.min(X_real[:, 1]) - 0.5
xmax2 = np.max(X_real[:, 1]) + 0.5
fp.ax_arr[0].plot(X_real[:, 0], X_real[:, 1], '.r')
fp.ax_arr[0].set_xlim([xmin1, xmax1]); fp.ax_arr[0].set_ylim([xmin2, xmax2])
fp.ax_arr[1].plot(X_sample[:, 0], X_sample[:, 1], '.g')
fp.ax_arr[1].set_xlim([xmin1, xmax1]); fp.ax_arr[1].set_ylim([xmin2, xmax2])
fp.ax_arr[0].set_aspect('equal', adjustable='box')
fp.ax_arr[1].set_aspect('equal', adjustable='box')
fp.ax_arr[1].set_title("Iter %i" % (train_iter+1))
fp.print_to_file(os.path.join(rpath, "train_results_%i.png" % (train_iter+1)))
bg_color = sns.color_palette('Greens', n_colors=256)[0]
ax2 = sns.kdeplot(np_samples_[:, 0], np_samples_[:, 1], shade=True, cmap='Greens', n_levels=20, clip=clip)
ax2.set_facecolor(bg_color)'''
#plt.savefig(os.path.join(rpath, "train_density_plot_bdgan_{}_{}.png".format(dataset,train_iter+1)))
samples = []
end_time = time.time()
train_time = end_time - start_time
print(train_time)
counter = 0
other_counter = 0
flag = 1
while (True):
sample_z = np.random.normal(0., 1, size=(1, z_dim))
ai, bi, sample = session.run([bgan.h0, bgan.drop, bgan.x],
feed_dict={
bgan.z: sample_z,
bgan.drop_prob: 0.99
})
other_counter = other_counter + 1
if (np.all((bi == 0))):
continue
if (np.count_nonzero(bi) > 1 or flag == 0):
samples.append(sample)
if (len(samples) == 9600):
flag = 0
if (len(samples) >= 10000):
break
counter = counter + 1
samples2 = np.concatenate(samples)
#print(counter)
#print(other_counter)
np.savetxt('original_{}.txt'.format(dataset),
synth_dataset.next_batch(10000),
delimiter=',')
np.savetxt('bdgan_samples3_{}.txt'.format(dataset),
samples2,
delimiter=',')
bg_color = sns.color_palette('Greens', n_colors=256)[0]
ax2 = sns.kdeplot(samples2[:, 0],
samples2[:, 1],
shade=True,
cmap='Greens',
n_levels=20,
clip=clip)
ax2.set_facecolor(bg_color)
plt.savefig(
os.path.join(rpath, "density_plot_bdgan_{}.png".format(dataset)))
g = (sns.jointplot(samples2[:, 0],
samples2[:, 1], color="k").plot_joint(sns.kdeplot,
zorder=0,
n_levels=6))
plt.savefig(
os.path.join(rpath, "density_sc_plot_bdgan_{}.png".format(dataset)))
fp = FigPrinter((1, 2))
xmin1 = np.min(X_real[:, 0]) - 0.5
xmax1 = np.max(X_real[:, 0]) + 0.5
xmin2 = np.min(X_real[:, 1]) - 0.5
xmax2 = np.max(X_real[:, 1]) + 0.5
fp.ax_arr[0].plot(X_real[:, 0], X_real[:, 1], '.r')
fp.ax_arr[0].set_xlim([xmin1, xmax1])
fp.ax_arr[0].set_ylim([xmin2, xmax2])
fp.ax_arr[1].plot(samples2[:, 0], samples2[:, 1], '.g')
fp.ax_arr[1].set_xlim([xmin1, xmax1])
fp.ax_arr[1].set_ylim([xmin2, xmax2])
fp.ax_arr[0].set_aspect('equal', adjustable='box')
fp.ax_arr[1].set_aspect('equal', adjustable='box')
fp.ax_arr[1].set_title("Iter %i" % (train_iter + 1))
fp.print_to_file(os.path.join(rpath, "results_%i.png" % (10 + 1)))
return {"data_real": synth_dataset.X, "z_dim": z_dim, "num_iter": num_iter}