def setup(self, X, num_centers, alpha, save_to='dec_model'):
sep = X.shape[0]*9/10
X_train = X[:sep]
X_val = X[sep:]
ae_model = AutoEncoderModel(self.xpu, [X.shape[1],500,500,2000,10], pt_dropout=0.2)
if not os.path.exists(save_to+'_pt.arg'):
ae_model.layerwise_pretrain(X_train, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
ae_model.finetune(X_train, 256, 100000, 'sgd', l_rate=0.1, decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
ae_model.save(save_to+'_pt.arg')
logging.log(logging.INFO, "Autoencoder Training error: %f"%ae_model.eval(X_train))
logging.log(logging.INFO, "Autoencoder Validation error: %f"%ae_model.eval(X_val))
else:
ae_model.load(save_to+'_pt.arg')
self.ae_model = ae_model
self.dec_op = DECModel.DECLoss(num_centers, alpha)
label = mx.sym.Variable('label')
self.feature = self.ae_model.encoder
self.loss = self.dec_op(data=self.ae_model.encoder, label=label, name='dec')
self.args.update({k:v for k,v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()})
self.args['dec_mu'] = mx.nd.empty((num_centers, self.ae_model.dims[-1]), ctx=self.xpu)
self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k,v in self.args.items()})
self.args_mult.update({k: k.endswith('bias') and 2.0 or 1.0 for k in self.args})
self.num_centers = num_centers
def setup(self, X, num_centers, alpha, save_to='dec_model'):
sep = X.shape[0] * 9 / 10
X_train = X[:sep]
X_val = X[sep:]
ae_model = AutoEncoderModel(self.xpu, [X.shape[1], 500, 500, 2000, 10], pt_dropout=0.2)
if not os.path.exists(save_to + '_pt.arg'):
ae_model.layerwise_pretrain(X_train, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000, 0.1))
ae_model.finetune(X_train, 256, 100000, 'sgd', l_rate=0.1, decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000, 0.1))
ae_model.save(save_to + '_pt.arg')
logging.log(logging.INFO, "Autoencoder Training error: %f" % ae_model.eval(X_train))
logging.log(logging.INFO, "Autoencoder Validation error: %f" % ae_model.eval(X_val))
else:
ae_model.load(save_to + '_pt.arg')
self.ae_model = ae_model
self.dec_op = DECModel.DECLoss(num_centers, alpha)
label = mx.sym.Variable('label')
self.feature = self.ae_model.encoder
self.loss = self.dec_op(data=self.ae_model.encoder, label=label, name='dec')
self.args.update({k: v for k, v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()})
self.args['dec_mu'] = mx.nd.empty((num_centers, self.ae_model.dims[-1]), ctx=self.xpu)
self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k, v in self.args.items()})
self.args_mult.update({k: k.endswith('bias') and 2.0 or 1.0 for k in self.args})
self.num_centers = num_centers
val_X = X[split_point:]
ae_model.layerwise_pretrain(train_X,
batch_size,
pretrain_num_iter,
'sgd',
l_rate=0.001,
decay=0.0,
lr_scheduler=mx.lr_scheduler.FactorScheduler(
20000, 0.7),
print_every=print_every)
ae_model.finetune(train_X,
batch_size,
finetune_num_iter,
'sgd',
l_rate=0.1,
decay=0.0,
lr_scheduler=mx.lr_scheduler.FactorScheduler(20000, 0.1),
print_every=print_every)
ae_model.save('autoencoder.arg')
ae_model.load('autoencoder.arg')
print("Training error:", ae_model.eval(train_X))
print("Validation error:", ae_model.eval(val_X))
if visualize:
try:
from matplotlib import pyplot as plt
from model import extract_feature
# sample a random image
xpu, [X_train.shape[1], 500, 500, 2000, output_size],
pt_dropout=0.2)
## Pre-train
ae_model.layerwise_pretrain(X_train,
batch_size,
50000,
'sgd',
l_rate=0.1,
decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(
20000, 0.1))
## finetune
ae_model.finetune(X_train,
batch_size,
100000,
'sgd',
l_rate=0.1,
decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000, 0.1))
## Get train/valid error (for Generalization)
logging.log(logging.INFO,
"Autoencoder Training error: %f" % ae_model.eval(X_train))
logging.log(logging.INFO,
"Autoencoder Validation error: %f" % ae_model.eval(X_val))
# put useful metrics in a dict
outdict = {
'E_train': ae_model.eval(X_train),
'E_val': ae_model.eval(X_val),
'output_size': output_size,
'sep': sep
# pylint: skip-file
import mxnet as mx
import numpy as np
import logging
import data
from autoencoder import AutoEncoderModel
if __name__ == '__main__':
# set to INFO to see less information during training
logging.basicConfig(level=logging.DEBUG)
ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2,
internal_act='relu', output_act='relu')
X, _ = data.get_mnist()
train_X = X[:60000]
val_X = X[60000:]
ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
ae_model.finetune(train_X, 256, 100000, 'sgd', l_rate=0.1, decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
ae_model.save('mnist_pt.arg')
ae_model.load('mnist_pt.arg')
print "Training error:", ae_model.eval(train_X)
print "Validation error:", ae_model.eval(val_X)
output_act='relu')
train_X = X
#ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
# lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
#V = np.zeros((train_X.shape[0],10))
V = np.random.rand(train_X.shape[0], K) / 10
lambda_v_rt = np.ones((train_X.shape[0], K)) * sqrt(lv)
U, V, theta, BCD_loss = ae_model.finetune(
train_X,
R,
V,
lambda_v_rt,
lambda_u,
lambda_v,
dir_save,
batch_size,
num_iter,
'sgd',
l_rate=0.1,
decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000, 0.1))
#ae_model.save('cdl_pt.arg')
np.savetxt(dir_save + '/final-U.dat', U, fmt='%.5f', comments='')
np.savetxt(dir_save + '/final-V.dat', V, fmt='%.5f', comments='')
np.savetxt(dir_save + '/final-theta.dat', theta, fmt='%.5f', comments='')
#ae_model.load('cdl_pt.arg')
Recon_loss = lambda_v / lv * ae_model.eval(train_X, V, lambda_v_rt)
print "Training error: %.3f" % (BCD_loss + Recon_loss)
fp = open(dir_save + '/cdl.log', 'a')
finetune_num_iter = opt.finetune_num_iter
visualize = opt.visualize
layers = [int(i) for i in opt.num_units.split(',')]
if __name__ == '__main__':
ae_model = AutoEncoderModel(mx.cpu(0), layers, pt_dropout=0.2,
internal_act='relu', output_act='relu')
X, _ = data.get_mnist()
train_X = X[:60000]
val_X = X[60000:]
ae_model.layerwise_pretrain(train_X, batch_size, pretrain_num_iter, 'sgd', l_rate=0.1,
decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000,0.1),
print_every=print_every)
ae_model.finetune(train_X, batch_size, finetune_num_iter, 'sgd', l_rate=0.1, decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000,0.1), print_every=print_every)
ae_model.save('mnist_pt.arg')
ae_model.load('mnist_pt.arg')
print("Training error:", ae_model.eval(train_X))
print("Validation error:", ae_model.eval(val_X))
if visualize:
try:
from matplotlib import pyplot as plt
from model import extract_feature
# sample a random image
original_image = X[np.random.choice(X.shape[0]), :].reshape(1, 784)
data_iter = mx.io.NDArrayIter({'data': original_image}, batch_size=1, shuffle=False,
last_batch_handle='pad')
# reconstruct the image
reconstructed_image = extract_feature(ae_model.decoder, ae_model.args,
ae_model.auxs, data_iter, 1,
import logging
import mnist_data as data
from math import sqrt
from autoencoder import AutoEncoderModel
if __name__ == '__main__':
lv = 1e-2# lv/ln in CDL
# set to INFO to see less information during training
logging.basicConfig(level=logging.DEBUG)
#ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2,
# internal_act='relu', output_act='relu')
ae_model = AutoEncoderModel(mx.cpu(2), [784,500,500,2000,10], pt_dropout=0.2,
internal_act='relu', output_act='relu')
X, _ = data.get_mnist()
train_X = X[:60000]
val_X = X[60000:]
#ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
# lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
#V = np.zeros((train_X.shape[0],10))
V = np.random.rand(train_X.shape[0],10)/10
lambda_v_rt = np.ones((train_X.shape[0],10))*sqrt(lv)
ae_model.finetune(train_X, V, lambda_v_rt, 256,
20, 'sgd', l_rate=0.1, decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
ae_model.save('mnist_pt.arg')
ae_model.load('mnist_pt.arg')
print "Training error:", ae_model.eval(train_X,V,lambda_v_rt)
#print "Validation error:", ae_model.eval(val_X)
#ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2,
# internal_act='relu', output_act='relu')
#mx.cpu() no param needed for cpu.
ae_model = AutoEncoderModel(mx.cpu(), [X.shape[1],100,K],
pt_dropout=0.2, internal_act='relu', output_act='relu')
train_X = X
#ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
# lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
#V = np.zeros((train_X.shape[0],10))
V = np.random.rand(train_X.shape[0],K)/10
lambda_v_rt = np.ones((train_X.shape[0],K))*sqrt(lv)
U, V, theta, BCD_loss = ae_model.finetune(train_X, R, V, lambda_v_rt, lambda_u,
lambda_v, dir_save, batch_size,
num_iter, 'sgd', l_rate=0.1, decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000,0.1))
#ae_model.save('cdl_pt.arg')
np.savetxt(dir_save+'/final-U.dat',U,fmt='%.5f',comments='')
np.savetxt(dir_save+'/final-V.dat',V,fmt='%.5f',comments='')
np.savetxt(dir_save+'/final-theta.dat',theta,fmt='%.5f',comments='')
#ae_model.load('cdl_pt.arg')
Recon_loss = lambda_v/lv*ae_model.eval(train_X,V,lambda_v_rt)
print "Training error: %.3f" % (BCD_loss+Recon_loss)
fp = open(dir_save+'/cdl.log','a')
fp.write("Training error: %.3f\n" % (BCD_loss+Recon_loss))
fp.close()
#print "Validation error:", ae_model.eval(val_X)
def setup(self, X, num_centers, alpha, znum, save_to='dec_model'):
self.sep = int(X.shape[0] * 0.75)
X_train = X[:self.sep]
X_val = X[self.sep:]
batch_size = 32 # 160 32*5 = update_interval*5
# Train or Read autoencoder: note is not dependent on number of clusters just on z latent size
ae_model = AutoEncoderModel(self.xpu,
[X.shape[1], 500, 500, 2000, znum],
pt_dropout=0.2)
if not os.path.exists(save_to + '_pt.arg'):
ae_model.layerwise_pretrain(X_train,
batch_size,
50000,
'sgd',
l_rate=0.1,
decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(
20000, 0.1))
ae_model.finetune(X_train,
batch_size,
100000,
'sgd',
l_rate=0.1,
decay=0.0,
lr_scheduler=mx.misc.FactorScheduler(20000, 0.1))
ae_model.save(save_to + '_pt.arg')
logging.log(
logging.INFO,
"Autoencoder Training error: %f" % ae_model.eval(X_train))
logging.log(
logging.INFO,
"Autoencoder Validation error: %f" % ae_model.eval(X_val))
else:
ae_model.load(save_to + '_pt.arg')
logging.log(
logging.INFO,
"Reading Autoencoder from file..: %s" % (save_to + '_pt.arg'))
logging.log(
logging.INFO,
"Autoencoder Training error: %f" % ae_model.eval(X_train))
logging.log(
logging.INFO,
"Autoencoder Validation error: %f" % ae_model.eval(X_val))
self.ae_model = ae_model
logging.log(logging.INFO, "finished reading Autoencoder from file..: ")
# prep model for clustering
self.dec_op = DECModel.DECLoss(num_centers, alpha)
label = mx.sym.Variable('label')
self.feature = self.ae_model.encoder
self.loss = self.dec_op(data=self.ae_model.encoder,
label=label,
name='dec')
self.args.update({
k: v
for k, v in self.ae_model.args.items()
if k in self.ae_model.encoder.list_arguments()
})
self.args['dec_mu'] = mx.nd.empty(
(num_centers, self.ae_model.dims[-1]), ctx=self.xpu)
self.args_grad.update({
k: mx.nd.empty(v.shape, ctx=self.xpu)
for k, v in self.args.items()
})
self.args_mult.update(
{k: k.endswith('bias') and 2.0 or 1.0
for k in self.args})
self.num_centers = num_centers
self.znum = znum
self.batch_size = batch_size
self.G = self.ae_model.eval(X_train) / self.ae_model.eval(X_val)
