def main(): mus = np.concatenate([np.identity(10), np.zeros((10, z_dim - 10))], 1) covs = np.stack([np.identity(z_dim) * args.sigma] * 10, 0) max_iter_step = 60000 trainset = make_dataset(trainx, trainy) testset = make_dataset(testx, testy) with tf.device(device): unlabeled_data, z, opt_c_z, opt_c_x, opt_e, opt_g, fake_x, c_loss_z, c_loss_x, e_loss, g_loss, unlabeled_z = build_graph( ) merged_all = tf.summary.merge_all() saver = tf.train.Saver() config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) config.gpu_options.allow_growth = False config.gpu_options.per_process_gpu_memory_fraction = 0.95 with tf.Session(config=config) as sess: def sample_z(gmm): if gmm is None: return np.random.normal(0, 1, [batch_size, z_dim]).astype(np.float32) ys = np.random.randint(10, size=batch_size) # np.random.multinomial(1, [1/10.]*10, size=batch_size) return np.concatenate([ np.random.multivariate_normal(mus[y], covs[y], 1) for y in ys ], 0) else: return gmm.sample(batch_size)[0] def sample_z_given_y(gmm, inx): if gmm is None: return np.random.normal(0, 1, [1, z_dim]).astype(np.float32) return np.random.multivariate_normal(mus[inx], covs[inx], 1) else: return np.random.multivariate_normal(gmm.means_[inx], gmm.covariances_[inx], 1) sess.run(tf.global_variables_initializer()) summary_writer = tf.summary.FileWriter(log_dir, sess.graph) gmm = None print("Training clustering wgan...") for i in range(max_iter_step): if i < 25 or i % 500 == 0: citers = 100 else: citers = Citers for j in range(citers): if i % 100 == 99 and j == 0: run_options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() _, _, merged, loss_c_z_, loss_c_x_ = sess.run( [opt_c_z, opt_c_x, merged_all, c_loss_z, c_loss_x], feed_dict={ z: sample_z(gmm), unlabeled_data: trainset.next_batch(batch_size)[0] }, options=run_options, run_metadata=run_metadata) summary_writer.add_summary(merged, i) summary_writer.add_run_metadata( run_metadata, 'critic_metadata {}'.format(i), i) else: _, _ = sess.run( [opt_c_z, opt_c_x], feed_dict={ unlabeled_data: trainset.next_batch(batch_size)[0], z: sample_z(gmm) }) if i % 100 == 99: _, _, merged, loss_e_, loss_g_ = sess.run( [opt_g, opt_e, merged_all, e_loss, g_loss], feed_dict={ z: sample_z(gmm), unlabeled_data: trainset.next_batch(batch_size)[0] }, options=run_options, run_metadata=run_metadata) summary_writer.add_summary(merged, i) summary_writer.add_run_metadata( run_metadata, 'generator_and_encoder_metadata {}'.format(i), i) else: _, _ = sess.run( [opt_g, opt_e], feed_dict={ z: sample_z(gmm), unlabeled_data: trainset.next_batch(batch_size)[0] }) if i % 100 == 99: print( "Training ite %d, c_loss_z: %f, c_loss_x: %f, e_loss: %f, g_loss: %f" % (i, loss_c_z_, loss_c_x_, loss_e_, loss_g_)) batch_y = [] for j in range(10): for k in range(10): batch_y.append(j) batch_z = np.concatenate( [sample_z_given_y(gmm, y) for y in batch_y[:64]], 0) bx = sess.run(fake_x, feed_dict={z: batch_z}) batch_z = np.concatenate( [sample_z_given_y(gmm, y) for y in batch_y[36:]], 0) bx1 = sess.run(fake_x, feed_dict={z: batch_z}) bx = np.concatenate([bx, bx1[28:]], 0) fig = plt.figure(image_dir + '.clustering-wgan') grid_show(fig, (bx + 1) / 2, [32, 32, channel]) if not os.path.exists('./logs/{}/{}'.format( image_dir, args.logdir)): os.makedirs('./logs/{}/{}'.format(image_dir, args.logdir)) fig.savefig('./logs/{}/{}/{}.png'.format( image_dir, args.logdir, (i - 99) / 100)) if i % 500 == 0: trainset._index_in_epoch = 0 true_zs = np.zeros( (trainset.num_examples / batch_size * batch_size, z_dim)) gts = np.zeros( (trainset.num_examples / batch_size * batch_size)) for j in range(trainset.num_examples / batch_size): train_img, train_label = trainset.next_batch(batch_size) bz = sess.run(unlabeled_z, feed_dict={unlabeled_data: train_img}) true_zs[j * batch_size:(j + 1) * batch_size] = bz gts[j * batch_size:(j + 1) * batch_size] = train_label gmm = mixture.BayesianGaussianMixture( n_components=10, covariance_type='full').fit(true_zs) preds = gmm.predict(true_zs) print("Training ite %d, acc: %f, nmi: %f" % (i, cluster_acc(preds, gts), cluster_nmi(preds, gts))) #preds1 = np.argmax(np.sum(np.square(true_zs[:, np.newaxis, :] - mus[np.newaxis, :, :]), 2), 1) #print("Training ite %d, new gmm, acc: %f, nmi: %f; prior gmm, acc: %f, nmi: %f" % (i, cluster_acc(preds, gts), cluster_nmi(preds, gts), cluster_acc(preds1, gts), cluster_nmi(preds1, gts))) if i % 1000 == 999: saver.save(sess, os.path.join(ckpt_dir, "model.ckpt"), global_step=i)
def main(): mus = np.concatenate([np.identity(10), np.zeros((10, z_dim - 10))], 1) covs = np.stack([np.identity(z_dim) * args.sigma] * 10, 0) max_iter_step = 60000 trainset = make_dataset(np.concatenate([trainx, testx], 0), np.concatenate([trainy, testy], 0)) testset = make_dataset(testx, testy) with tf.device(device): unlabeled_data, z, opt_c_z, opt_c_x, opt_e, opt_g, fake_x, c_loss_z, c_loss_x, e_loss, g_loss, unlabeled_z, opt_prior, prior_loss, sample_pi, tau = build_graph( ) merged_all = tf.summary.merge_all() saver = tf.train.Saver() config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) config.gpu_options.allow_growth = False with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) summary_writer = tf.summary.FileWriter(log_dir, sess.graph) tau0 = 1.0 # initial temperature np_temp = tau0 ANNEAL_RATE = 0.00003 MIN_TEMP = 0.5 print("Training unsupervised wgan...") for i in range(max_iter_step): for j in range(Citers): if i % 100 == 99 and j == 0: run_options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() _, _, merged, loss_c_z_, loss_c_x_ = sess.run( [opt_c_z, opt_c_x, merged_all, c_loss_z, c_loss_x], feed_dict={ unlabeled_data: trainset.next_batch(batch_size)[0], tau: np_temp }, options=run_options, run_metadata=run_metadata) summary_writer.add_summary(merged, i) summary_writer.add_run_metadata( run_metadata, 'critic_metadata {}'.format(i), i) else: _, _ = sess.run( [opt_c_z, opt_c_x], feed_dict={ tau: np_temp, unlabeled_data: trainset.next_batch(batch_size)[0] }) if i % 100 == 99: _, _, merged, loss_g_, loss_prior_ = sess.run( [opt_g, opt_prior, merged_all, g_loss, prior_loss], feed_dict={ unlabeled_data: trainset.next_batch(batch_size)[0], tau: np_temp }, options=run_options, run_metadata=run_metadata) summary_writer.add_summary(merged, i) summary_writer.add_run_metadata( run_metadata, 'generator_and_prior_metadata {}'.format(i), i) _, merged, loss_e_ = sess.run( [opt_e, merged_all, e_loss], feed_dict={ unlabeled_data: trainset.next_batch(batch_size)[0], tau: np_temp }, options=run_options, run_metadata=run_metadata) summary_writer.add_summary(merged, i) summary_writer.add_run_metadata( run_metadata, 'encoder_metadata {}'.format(i), i) else: _, _ = sess.run( [opt_g, opt_prior], feed_dict={ tau: np_temp, unlabeled_data: trainset.next_batch(batch_size)[0] }) _ = sess.run( [opt_e], feed_dict={ tau: np_temp, unlabeled_data: trainset.next_batch(batch_size)[0] }) if i % 100 == 99: # pi_ = sess.run(sample_pi, feed_dict={tau:np_temp}) # print(pi_) print( "Training ite %d, c_loss_z: %f, c_loss_x: %f, prior_loss: %f, e_loss: %f, g_loss: %f" % (i, loss_c_z_, loss_c_x_, loss_prior_, loss_e_, loss_g_)) # batch_y = [] # for j in range(10): # for k in range(10): # batch_y.append(j) # batch_z = np.concatenate([sample_z_given_y(gmm, y) for y in batch_y[:batch_size]], 0) # bx = sess.run(fake_x, feed_dict={z: batch_z}) # batch_z = np.concatenate([sample_z_given_y(gmm, y) for y in batch_y[100-batch_size:]], 0) # bx1 = sess.run(fake_x, feed_dict={z: batch_z}) # bx = np.concatenate([bx, bx1[2*batch_size-100:batch_size]], 0) # fig = plt.figure(image_dir + '.clustering-wgan') # grid_show(fig, (bx + 1) / 2, [32, 32, channel]) # if not os.path.exists('./logs/{}/{}'.format(image_dir, args.logdir)): # os.makedirs('./logs/{}/{}'.format(image_dir, args.logdir)) # fig.savefig('./logs/{}/{}/{}.png'.format(image_dir, args.logdir, (i-99)/100)) if i % 1000 == 999: np_temp = np.maximum(tau0 * np.exp(-ANNEAL_RATE * i), MIN_TEMP) trainset.shuffle() true_zs = np.zeros( (trainset._num_examples / batch_size * batch_size, z_dim)) gts = np.zeros( (trainset._num_examples / batch_size * batch_size)) for j in range(trainset._num_examples / batch_size): train_img, train_label = trainset.next_batch(batch_size) bz = sess.run(unlabeled_z, feed_dict={unlabeled_data: train_img}) true_zs[j * batch_size:(j + 1) * batch_size] = bz gts[j * batch_size:(j + 1) * batch_size] = train_label preds = cluster.KMeans(n_clusters=10, n_jobs=-1).fit_predict(true_zs) print("Training ite %d, acc: %f, nmi: %f" % (i, cluster_acc(preds, gts), cluster_nmi(preds, gts))) saver.save(sess, os.path.join(ckpt_dir, "model.ckpt"), global_step=i)
def bench_desom(X_train, y_train, dataset, map_size, encoder_dims, ae_weights=None): print('*** {} - desom with {} map and {} autoencoder (gamma={})***'.format( dataset, map_size, encoder_dims, gamma)) desom = DESOM(encoder_dims=encoder_dims, map_size=map_size) save_dir = 'results/benchmark/desom-gamma{}_{}_{}_{}x{}'.format( gamma, dataset, optimizer, map_size[0], map_size[1]) subprocess.run(['mkdir', '-p', save_dir]) for run in range(n_runs): desom.initialize() desom.compile(gamma=gamma, optimizer=optimizer) if ae_weights is not None: desom.load_ae_weights(ae_weights) # Weights initialization by randomly sampling training points desom.init_som_weights(X_train) t0 = time.time() desom.fit(X_train, y_train, None, None, iterations, som_iterations, eval_interval, save_epochs, batch_size, Tmax, Tmin, decay, save_dir) dt = time.time() - t0 print('Run {}/{} (took {:f} seconds)'.format(run + 1, n_runs, dt)) y_pred = desom.predict(X_train) pur[run] = cluster_purity(y_train, y_pred) nmi[run] = normalized_mutual_info_score(y_train, y_pred) ari[run] = adjusted_rand_score(y_train, y_pred) acc[run] = cluster_acc(y_train, y_pred) duration[run] = dt if map_size[0] == 8: # Post clustering in latent space print('Post-clustering in latent space...') prototypes = desom.prototypes km_desom = KMeans(n_clusters=np.max(y_train), n_jobs=-1).fit(prototypes) km_desom_pred = km_desom.predict(desom.encode(X_train)) pur_clust[run] = cluster_purity(y_train, km_desom_pred) nmi_clust[run] = normalized_mutual_info_score( y_train, km_desom_pred) ari_clust[run] = adjusted_rand_score(y_train, km_desom_pred) acc_clust[run] = cluster_acc(y_train, km_desom_pred) name = '{} (K={})'.format(dataset, map_size[0] * map_size[1]) results.at[name, 'pur'] = pur.mean() results.at[name, 'pur_std'] = pur.std() results.at[name, 'nmi'] = nmi.mean() results.at[name, 'nmi_std'] = nmi.std() results.at[name, 'ari'] = ari.mean() results.at[name, 'ari_std'] = ari.std() results.at[name, 'acc'] = acc.mean() results.at[name, 'acc_std'] = acc.std() results.at[name, 'duration'] = duration.mean() if map_size[0] == 8: # Post clustering results.at[name, 'pur_clust'] = pur_clust.mean() results.at[name, 'pur_clust_std'] = pur_clust.std() results.at[name, 'nmi_clust'] = nmi_clust.mean() results.at[name, 'nmi_clust_std'] = nmi_clust.std() results.at[name, 'ari_clust'] = ari_clust.mean() results.at[name, 'ari_clust_std'] = ari_clust.std() results.at[name, 'acc_clust'] = acc_clust.mean() results.at[name, 'acc_clust_std'] = acc_clust.std() print(results.loc[name])
def main(): max_iter_step = 60000 trainset = make_dataset(np.concatenate([trainx, testx],0), np.concatenate([trainy, testy],0)) testset = make_dataset(testx, testy) with tf.device(device): unlabeled_data, z, opt_c_z, opt_c_x, opt_e, opt_g, fake_x, c_loss_z, c_loss_x, e_loss, g_loss, unlabeled_z, opt_prior, prior_loss, opt_e_y, e_loss_y, y, pretrain = build_graph() merged_all = tf.summary.merge_all() saver = tf.train.Saver() config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) config.gpu_options.allow_growth = False with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) summary_writer = tf.summary.FileWriter(log_dir, sess.graph) print("Pretraining unsupervised wgan...") for j in range(args.pretrain): _, loss_e_y_ = sess.run([pretrain, e_loss_y], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size)}) if j % 100 == 99: print("Pretraining ite %d, e_loss_y: %f" % (j , loss_e_y_)) print("Training unsupervised wgan...") loss_prior_ = 0 for i in range(max_iter_step): for j in range(Citers): if i % 100 == 99 and j == 0: run_options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() _, _, merged, loss_c_z_, loss_c_x_ = sess.run([opt_c_z, opt_c_x, merged_all, c_loss_z, c_loss_x], feed_dict={unlabeled_data: trainset.next_batch(batch_size)[0], y:np.random.multinomial(1, [1/10.]*10, size=batch_size)}, options=run_options, run_metadata=run_metadata) summary_writer.add_summary(merged, i) summary_writer.add_run_metadata(run_metadata, 'critic_metadata {}'.format(i), i) else: _, _ = sess.run([opt_c_z, opt_c_x], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size),unlabeled_data: trainset.next_batch(batch_size)[0]}) if i % 100 == 99: _, _, merged, loss_e_, loss_g_ = sess.run([opt_g, opt_e, merged_all, e_loss, g_loss], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size), unlabeled_data: trainset.next_batch(batch_size)[0]}, options=run_options, run_metadata=run_metadata) summary_writer.add_summary(merged, i) summary_writer.add_run_metadata(run_metadata, 'generator_and_encoder_metadata {}'.format(i), i) else: _, _ = sess.run([opt_g, opt_e], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size), unlabeled_data: trainset.next_batch(batch_size)[0]}) if i % 100 == 99: for j in range(100): _,_, loss_e_y_, loss_prior_ = sess.run([opt_e_y, opt_prior, e_loss_y, prior_loss], feed_dict={y:np.random.multinomial(1, [1/10.]*10, size=batch_size)}) print("Training ite %d, c_loss_z: %f, c_loss_x: %f, prior_loss: %f, g_loss: %f, e_loss: %f, e_loss_y: %f" % (i, loss_c_z_, loss_c_x_, loss_prior_, loss_g_, loss_e_, loss_e_y_)) batch_y = [] for j in range(10): for k in range(10): batch_y.append(j) batch_y = dense_to_one_hot(np.asarray(batch_y)) bx = sess.run(fake_x, feed_dict={y: batch_y[:batch_size]}) bx1 = sess.run(fake_x, feed_dict={y: batch_y[100-batch_size:]}) bx = np.concatenate([bx, bx1[2*batch_size-100:batch_size]], 0) fig = plt.figure(image_dir + '.clustering-wgan') grid_show(fig, (bx + 1) / 2, [32, 32, channel]) if not os.path.exists('./logs/{}/{}'.format(image_dir, args.logdir)): os.makedirs('./logs/{}/{}'.format(image_dir, args.logdir)) fig.savefig('./logs/{}/{}/{}.png'.format(image_dir, args.logdir, (i-99)/100)) if i % 1000 == 999: trainset.shuffle() true_zs = np.zeros((trainset._num_examples / batch_size * batch_size, z_dim)) gts = np.zeros((trainset._num_examples / batch_size * batch_size)) for j in range(trainset._num_examples / batch_size): train_img, train_label = trainset.next_batch(batch_size) bz = sess.run(unlabeled_z, feed_dict={unlabeled_data: train_img}) true_zs[j*batch_size:(j+1)*batch_size] = bz gts[j*batch_size:(j+1)*batch_size] = train_label preds = cluster.KMeans(n_clusters=10, n_jobs=-1).fit_predict(true_zs) print("Training ite %d, acc: %f, nmi: %f" % (i, cluster_acc(preds, gts), cluster_nmi(preds, gts))) saver.save(sess, os.path.join( ckpt_dir, "model.ckpt"), global_step=i)