def main():
mus = np.concatenate([np.identity(10), np.zeros((10, z_dim - 10))], 1)
covs = np.stack([np.identity(z_dim) * args.sigma] * 10, 0)
max_iter_step = 60000
trainset = make_dataset(trainx, trainy)
testset = make_dataset(testx, testy)
with tf.device(device):
unlabeled_data, z, opt_c_z, opt_c_x, opt_e, opt_g, fake_x, c_loss_z, c_loss_x, e_loss, g_loss, unlabeled_z = build_graph(
)
merged_all = tf.summary.merge_all()
saver = tf.train.Saver()
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = False
config.gpu_options.per_process_gpu_memory_fraction = 0.95
with tf.Session(config=config) as sess:
def sample_z(gmm):
if gmm is None:
return np.random.normal(0, 1,
[batch_size, z_dim]).astype(np.float32)
ys = np.random.randint(10, size=batch_size)
# np.random.multinomial(1, [1/10.]*10, size=batch_size)
return np.concatenate([
np.random.multivariate_normal(mus[y], covs[y], 1)
for y in ys
], 0)
else:
return gmm.sample(batch_size)[0]
def sample_z_given_y(gmm, inx):
if gmm is None:
return np.random.normal(0, 1, [1, z_dim]).astype(np.float32)
return np.random.multivariate_normal(mus[inx], covs[inx], 1)
else:
return np.random.multivariate_normal(gmm.means_[inx],
gmm.covariances_[inx], 1)
sess.run(tf.global_variables_initializer())
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
gmm = None
print("Training clustering wgan...")
for i in range(max_iter_step):
if i < 25 or i % 500 == 0:
citers = 100
else:
citers = Citers
for j in range(citers):
if i % 100 == 99 and j == 0:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
_, _, merged, loss_c_z_, loss_c_x_ = sess.run(
[opt_c_z, opt_c_x, merged_all, c_loss_z, c_loss_x],
feed_dict={
z: sample_z(gmm),
unlabeled_data: trainset.next_batch(batch_size)[0]
},
options=run_options,
run_metadata=run_metadata)
summary_writer.add_summary(merged, i)
summary_writer.add_run_metadata(
run_metadata, 'critic_metadata {}'.format(i), i)
else:
_, _ = sess.run(
[opt_c_z, opt_c_x],
feed_dict={
unlabeled_data: trainset.next_batch(batch_size)[0],
z: sample_z(gmm)
})
if i % 100 == 99:
_, _, merged, loss_e_, loss_g_ = sess.run(
[opt_g, opt_e, merged_all, e_loss, g_loss],
feed_dict={
z: sample_z(gmm),
unlabeled_data: trainset.next_batch(batch_size)[0]
},
options=run_options,
run_metadata=run_metadata)
summary_writer.add_summary(merged, i)
summary_writer.add_run_metadata(
run_metadata,
'generator_and_encoder_metadata {}'.format(i), i)
else:
_, _ = sess.run(
[opt_g, opt_e],
feed_dict={
z: sample_z(gmm),
unlabeled_data: trainset.next_batch(batch_size)[0]
})
if i % 100 == 99:
print(
"Training ite %d, c_loss_z: %f, c_loss_x: %f, e_loss: %f, g_loss: %f"
% (i, loss_c_z_, loss_c_x_, loss_e_, loss_g_))
batch_y = []
for j in range(10):
for k in range(10):
batch_y.append(j)
batch_z = np.concatenate(
[sample_z_given_y(gmm, y) for y in batch_y[:64]], 0)
bx = sess.run(fake_x, feed_dict={z: batch_z})
batch_z = np.concatenate(
[sample_z_given_y(gmm, y) for y in batch_y[36:]], 0)
bx1 = sess.run(fake_x, feed_dict={z: batch_z})
bx = np.concatenate([bx, bx1[28:]], 0)
fig = plt.figure(image_dir + '.clustering-wgan')
grid_show(fig, (bx + 1) / 2, [32, 32, channel])
if not os.path.exists('./logs/{}/{}'.format(
image_dir, args.logdir)):
os.makedirs('./logs/{}/{}'.format(image_dir, args.logdir))
fig.savefig('./logs/{}/{}/{}.png'.format(
image_dir, args.logdir, (i - 99) / 100))
if i % 500 == 0:
trainset._index_in_epoch = 0
true_zs = np.zeros(
(trainset.num_examples / batch_size * batch_size, z_dim))
gts = np.zeros(
(trainset.num_examples / batch_size * batch_size))
for j in range(trainset.num_examples / batch_size):
train_img, train_label = trainset.next_batch(batch_size)
bz = sess.run(unlabeled_z,
feed_dict={unlabeled_data: train_img})
true_zs[j * batch_size:(j + 1) * batch_size] = bz
gts[j * batch_size:(j + 1) * batch_size] = train_label
gmm = mixture.BayesianGaussianMixture(
n_components=10, covariance_type='full').fit(true_zs)
preds = gmm.predict(true_zs)
print("Training ite %d, acc: %f, nmi: %f" %
(i, cluster_acc(preds, gts), cluster_nmi(preds, gts)))
#preds1 = np.argmax(np.sum(np.square(true_zs[:, np.newaxis, :] - mus[np.newaxis, :, :]), 2), 1)
#print("Training ite %d, new gmm, acc: %f, nmi: %f; prior gmm, acc: %f, nmi: %f" % (i, cluster_acc(preds, gts), cluster_nmi(preds, gts), cluster_acc(preds1, gts), cluster_nmi(preds1, gts)))
if i % 1000 == 999:
saver.save(sess,
os.path.join(ckpt_dir, "model.ckpt"),
global_step=i)
def main():
mus = np.concatenate([np.identity(10), np.zeros((10, z_dim - 10))], 1)
covs = np.stack([np.identity(z_dim) * args.sigma] * 10, 0)
max_iter_step = 60000
trainset = make_dataset(np.concatenate([trainx, testx], 0),
np.concatenate([trainy, testy], 0))
testset = make_dataset(testx, testy)
with tf.device(device):
unlabeled_data, z, opt_c_z, opt_c_x, opt_e, opt_g, fake_x, c_loss_z, c_loss_x, e_loss, g_loss, unlabeled_z, opt_prior, prior_loss, sample_pi, tau = build_graph(
)
merged_all = tf.summary.merge_all()
saver = tf.train.Saver()
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = False
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
tau0 = 1.0 # initial temperature
np_temp = tau0
ANNEAL_RATE = 0.00003
MIN_TEMP = 0.5
print("Training unsupervised wgan...")
for i in range(max_iter_step):
for j in range(Citers):
if i % 100 == 99 and j == 0:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
_, _, merged, loss_c_z_, loss_c_x_ = sess.run(
[opt_c_z, opt_c_x, merged_all, c_loss_z, c_loss_x],
feed_dict={
unlabeled_data: trainset.next_batch(batch_size)[0],
tau: np_temp
},
options=run_options,
run_metadata=run_metadata)
summary_writer.add_summary(merged, i)
summary_writer.add_run_metadata(
run_metadata, 'critic_metadata {}'.format(i), i)
else:
_, _ = sess.run(
[opt_c_z, opt_c_x],
feed_dict={
tau: np_temp,
unlabeled_data: trainset.next_batch(batch_size)[0]
})
if i % 100 == 99:
_, _, merged, loss_g_, loss_prior_ = sess.run(
[opt_g, opt_prior, merged_all, g_loss, prior_loss],
feed_dict={
unlabeled_data: trainset.next_batch(batch_size)[0],
tau: np_temp
},
options=run_options,
run_metadata=run_metadata)
summary_writer.add_summary(merged, i)
summary_writer.add_run_metadata(
run_metadata, 'generator_and_prior_metadata {}'.format(i),
i)
_, merged, loss_e_ = sess.run(
[opt_e, merged_all, e_loss],
feed_dict={
unlabeled_data: trainset.next_batch(batch_size)[0],
tau: np_temp
},
options=run_options,
run_metadata=run_metadata)
summary_writer.add_summary(merged, i)
summary_writer.add_run_metadata(
run_metadata, 'encoder_metadata {}'.format(i), i)
else:
_, _ = sess.run(
[opt_g, opt_prior],
feed_dict={
tau: np_temp,
unlabeled_data: trainset.next_batch(batch_size)[0]
})
_ = sess.run(
[opt_e],
feed_dict={
tau: np_temp,
unlabeled_data: trainset.next_batch(batch_size)[0]
})
if i % 100 == 99:
# pi_ = sess.run(sample_pi, feed_dict={tau:np_temp})
# print(pi_)
print(
"Training ite %d, c_loss_z: %f, c_loss_x: %f, prior_loss: %f, e_loss: %f, g_loss: %f"
% (i, loss_c_z_, loss_c_x_, loss_prior_, loss_e_, loss_g_))
# batch_y = []
# for j in range(10):
# for k in range(10):
# batch_y.append(j)
# batch_z = np.concatenate([sample_z_given_y(gmm, y) for y in batch_y[:batch_size]], 0)
# bx = sess.run(fake_x, feed_dict={z: batch_z})
# batch_z = np.concatenate([sample_z_given_y(gmm, y) for y in batch_y[100-batch_size:]], 0)
# bx1 = sess.run(fake_x, feed_dict={z: batch_z})
# bx = np.concatenate([bx, bx1[2*batch_size-100:batch_size]], 0)
# fig = plt.figure(image_dir + '.clustering-wgan')
# grid_show(fig, (bx + 1) / 2, [32, 32, channel])
# if not os.path.exists('./logs/{}/{}'.format(image_dir, args.logdir)):
# os.makedirs('./logs/{}/{}'.format(image_dir, args.logdir))
# fig.savefig('./logs/{}/{}/{}.png'.format(image_dir, args.logdir, (i-99)/100))
if i % 1000 == 999:
np_temp = np.maximum(tau0 * np.exp(-ANNEAL_RATE * i), MIN_TEMP)
trainset.shuffle()
true_zs = np.zeros(
(trainset._num_examples / batch_size * batch_size, z_dim))
gts = np.zeros(
(trainset._num_examples / batch_size * batch_size))
for j in range(trainset._num_examples / batch_size):
train_img, train_label = trainset.next_batch(batch_size)
bz = sess.run(unlabeled_z,
feed_dict={unlabeled_data: train_img})
true_zs[j * batch_size:(j + 1) * batch_size] = bz
gts[j * batch_size:(j + 1) * batch_size] = train_label
preds = cluster.KMeans(n_clusters=10,
n_jobs=-1).fit_predict(true_zs)
print("Training ite %d, acc: %f, nmi: %f" %
(i, cluster_acc(preds, gts), cluster_nmi(preds, gts)))
saver.save(sess,
os.path.join(ckpt_dir, "model.ckpt"),
global_step=i)
def bench_desom(X_train,
y_train,
dataset,
map_size,
encoder_dims,
ae_weights=None):
print('*** {} - desom with {} map and {} autoencoder (gamma={})***'.format(
dataset, map_size, encoder_dims, gamma))
desom = DESOM(encoder_dims=encoder_dims, map_size=map_size)
save_dir = 'results/benchmark/desom-gamma{}_{}_{}_{}x{}'.format(
gamma, dataset, optimizer, map_size[0], map_size[1])
subprocess.run(['mkdir', '-p', save_dir])
for run in range(n_runs):
desom.initialize()
desom.compile(gamma=gamma, optimizer=optimizer)
if ae_weights is not None:
desom.load_ae_weights(ae_weights)
# Weights initialization by randomly sampling training points
desom.init_som_weights(X_train)
t0 = time.time()
desom.fit(X_train, y_train, None, None, iterations, som_iterations,
eval_interval, save_epochs, batch_size, Tmax, Tmin, decay,
save_dir)
dt = time.time() - t0
print('Run {}/{} (took {:f} seconds)'.format(run + 1, n_runs, dt))
y_pred = desom.predict(X_train)
pur[run] = cluster_purity(y_train, y_pred)
nmi[run] = normalized_mutual_info_score(y_train, y_pred)
ari[run] = adjusted_rand_score(y_train, y_pred)
acc[run] = cluster_acc(y_train, y_pred)
duration[run] = dt
if map_size[0] == 8:
# Post clustering in latent space
print('Post-clustering in latent space...')
prototypes = desom.prototypes
km_desom = KMeans(n_clusters=np.max(y_train),
n_jobs=-1).fit(prototypes)
km_desom_pred = km_desom.predict(desom.encode(X_train))
pur_clust[run] = cluster_purity(y_train, km_desom_pred)
nmi_clust[run] = normalized_mutual_info_score(
y_train, km_desom_pred)
ari_clust[run] = adjusted_rand_score(y_train, km_desom_pred)
acc_clust[run] = cluster_acc(y_train, km_desom_pred)
name = '{} (K={})'.format(dataset, map_size[0] * map_size[1])
results.at[name, 'pur'] = pur.mean()
results.at[name, 'pur_std'] = pur.std()
results.at[name, 'nmi'] = nmi.mean()
results.at[name, 'nmi_std'] = nmi.std()
results.at[name, 'ari'] = ari.mean()
results.at[name, 'ari_std'] = ari.std()
results.at[name, 'acc'] = acc.mean()
results.at[name, 'acc_std'] = acc.std()
results.at[name, 'duration'] = duration.mean()
if map_size[0] == 8:
# Post clustering
results.at[name, 'pur_clust'] = pur_clust.mean()
results.at[name, 'pur_clust_std'] = pur_clust.std()
results.at[name, 'nmi_clust'] = nmi_clust.mean()
results.at[name, 'nmi_clust_std'] = nmi_clust.std()
results.at[name, 'ari_clust'] = ari_clust.mean()
results.at[name, 'ari_clust_std'] = ari_clust.std()
results.at[name, 'acc_clust'] = acc_clust.mean()
results.at[name, 'acc_clust_std'] = acc_clust.std()
print(results.loc[name])
def main():
max_iter_step = 60000
trainset = make_dataset(np.concatenate([trainx, testx],0), np.concatenate([trainy, testy],0))
testset = make_dataset(testx, testy)
with tf.device(device):
unlabeled_data, z, opt_c_z, opt_c_x, opt_e, opt_g, fake_x, c_loss_z, c_loss_x, e_loss, g_loss, unlabeled_z, opt_prior, prior_loss, opt_e_y, e_loss_y, y, pretrain = build_graph()
merged_all = tf.summary.merge_all()
saver = tf.train.Saver()
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
config.gpu_options.allow_growth = False
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
print("Pretraining unsupervised wgan...")
for j in range(args.pretrain):
_, loss_e_y_ = sess.run([pretrain, e_loss_y], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size)})
if j % 100 == 99:
print("Pretraining ite %d, e_loss_y: %f" % (j , loss_e_y_))
print("Training unsupervised wgan...")
loss_prior_ = 0
for i in range(max_iter_step):
for j in range(Citers):
if i % 100 == 99 and j == 0:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
_, _, merged, loss_c_z_, loss_c_x_ = sess.run([opt_c_z, opt_c_x, merged_all, c_loss_z, c_loss_x], feed_dict={unlabeled_data: trainset.next_batch(batch_size)[0], y:np.random.multinomial(1, [1/10.]*10, size=batch_size)}, options=run_options, run_metadata=run_metadata)
summary_writer.add_summary(merged, i)
summary_writer.add_run_metadata(run_metadata, 'critic_metadata {}'.format(i), i)
else:
_, _ = sess.run([opt_c_z, opt_c_x], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size),unlabeled_data: trainset.next_batch(batch_size)[0]})
if i % 100 == 99:
_, _, merged, loss_e_, loss_g_ = sess.run([opt_g, opt_e, merged_all, e_loss, g_loss], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size), unlabeled_data: trainset.next_batch(batch_size)[0]}, options=run_options, run_metadata=run_metadata)
summary_writer.add_summary(merged, i)
summary_writer.add_run_metadata(run_metadata, 'generator_and_encoder_metadata {}'.format(i), i)
else:
_, _ = sess.run([opt_g, opt_e], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size), unlabeled_data: trainset.next_batch(batch_size)[0]})
if i % 100 == 99:
for j in range(100):
_,_, loss_e_y_, loss_prior_ = sess.run([opt_e_y, opt_prior, e_loss_y, prior_loss], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size)})
print("Training ite %d, c_loss_z: %f, c_loss_x: %f, prior_loss: %f, g_loss: %f, e_loss: %f, e_loss_y: %f" % (i, loss_c_z_, loss_c_x_, loss_prior_, loss_g_, loss_e_, loss_e_y_))
batch_y = []
for j in range(10):
for k in range(10):
batch_y.append(j)
batch_y = dense_to_one_hot(np.asarray(batch_y))
bx = sess.run(fake_x, feed_dict={y: batch_y[:batch_size]})
bx1 = sess.run(fake_x, feed_dict={y: batch_y[100-batch_size:]})
bx = np.concatenate([bx, bx1[2*batch_size-100:batch_size]], 0)
fig = plt.figure(image_dir + '.clustering-wgan')
grid_show(fig, (bx + 1) / 2, [32, 32, channel])
if not os.path.exists('./logs/{}/{}'.format(image_dir, args.logdir)):
os.makedirs('./logs/{}/{}'.format(image_dir, args.logdir))
fig.savefig('./logs/{}/{}/{}.png'.format(image_dir, args.logdir, (i-99)/100))
if i % 1000 == 999:
trainset.shuffle()
true_zs = np.zeros((trainset._num_examples / batch_size * batch_size, z_dim))
gts = np.zeros((trainset._num_examples / batch_size * batch_size))
for j in range(trainset._num_examples / batch_size):
train_img, train_label = trainset.next_batch(batch_size)
bz = sess.run(unlabeled_z, feed_dict={unlabeled_data: train_img})
true_zs[j*batch_size:(j+1)*batch_size] = bz
gts[j*batch_size:(j+1)*batch_size] = train_label
preds = cluster.KMeans(n_clusters=10, n_jobs=-1).fit_predict(true_zs)
print("Training ite %d, acc: %f, nmi: %f" % (i, cluster_acc(preds, gts), cluster_nmi(preds, gts)))
saver.save(sess, os.path.join(
ckpt_dir, "model.ckpt"), global_step=i)