def setup(self, X, num_centers, alpha, save_to='dec_model'): sep = X.shape[0] * 9 / 10 X_train = X[:sep] X_val = X[sep:] ae_model = AutoEncoderModel(self.xpu, [X.shape[1], 500, 500, 2000, 10], pt_dropout=0.2) if not os.path.exists(save_to + '_pt.arg'): ae_model.layerwise_pretrain(X_train, 256, 50000, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000, 0.1)) ae_model.finetune(X_train, 256, 100000, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000, 0.1)) ae_model.save(save_to + '_pt.arg') logging.log(logging.INFO, "Autoencoder Training error: %f" % ae_model.eval(X_train)) logging.log(logging.INFO, "Autoencoder Validation error: %f" % ae_model.eval(X_val)) else: ae_model.load(save_to + '_pt.arg') self.ae_model = ae_model self.dec_op = DECModel.DECLoss(num_centers, alpha) label = mx.sym.Variable('label') self.feature = self.ae_model.encoder self.loss = self.dec_op(data=self.ae_model.encoder, label=label, name='dec') self.args.update({k: v for k, v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()}) self.args['dec_mu'] = mx.nd.empty((num_centers, self.ae_model.dims[-1]), ctx=self.xpu) self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k, v in self.args.items()}) self.args_mult.update({k: k.endswith('bias') and 2.0 or 1.0 for k in self.args}) self.num_centers = num_centers
def setup(self, X, num_centers, alpha, save_to='dec_model'): sep = X.shape[0]*9/10 X_train = X[:sep] X_val = X[sep:] ae_model = AutoEncoderModel(self.xpu, [X.shape[1],500,500,2000,10], pt_dropout=0.2) if not os.path.exists(save_to+'_pt.arg'): ae_model.layerwise_pretrain(X_train, 256, 50000, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) ae_model.finetune(X_train, 256, 100000, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) ae_model.save(save_to+'_pt.arg') logging.log(logging.INFO, "Autoencoder Training error: %f"%ae_model.eval(X_train)) logging.log(logging.INFO, "Autoencoder Validation error: %f"%ae_model.eval(X_val)) else: ae_model.load(save_to+'_pt.arg') self.ae_model = ae_model self.dec_op = DECModel.DECLoss(num_centers, alpha) label = mx.sym.Variable('label') self.feature = self.ae_model.encoder self.loss = self.dec_op(data=self.ae_model.encoder, label=label, name='dec') self.args.update({k:v for k,v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()}) self.args['dec_mu'] = mx.nd.empty((num_centers, self.ae_model.dims[-1]), ctx=self.xpu) self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k,v in self.args.items()}) self.args_mult.update({k: k.endswith('bias') and 2.0 or 1.0 for k in self.args}) self.num_centers = num_centers
20000, 0.7), print_every=print_every) ae_model.finetune(train_X, batch_size, finetune_num_iter, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.lr_scheduler.FactorScheduler(20000, 0.1), print_every=print_every) ae_model.save('autoencoder.arg') ae_model.load('autoencoder.arg') print("Training error:", ae_model.eval(train_X)) print("Validation error:", ae_model.eval(val_X)) if visualize: try: from matplotlib import pyplot as plt from model import extract_feature # sample a random image #index = np.random.choice(len(X)) index = 0 original_image = X[index] #print(json.dumps(original_image)) data_iter = mx.io.NDArrayIter({'data': [original_image]}, batch_size=1, shuffle=False, last_batch_handle='pad')
l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler( 20000, 0.1)) ## finetune ae_model.finetune(X_train, batch_size, 100000, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000, 0.1)) ## Get train/valid error (for Generalization) logging.log(logging.INFO, "Autoencoder Training error: %f" % ae_model.eval(X_train)) logging.log(logging.INFO, "Autoencoder Validation error: %f" % ae_model.eval(X_val)) # put useful metrics in a dict outdict = { 'E_train': ae_model.eval(X_train), 'E_val': ae_model.eval(X_val), 'output_size': output_size, 'sep': sep } allAutoencoders.append(outdict) # to save ae_model.save( os.path.join( save_to,
# pylint: skip-file import mxnet as mx import numpy as np import logging import data from autoencoder import AutoEncoderModel if __name__ == '__main__': # set to INFO to see less information during training logging.basicConfig(level=logging.DEBUG) ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2, internal_act='relu', output_act='relu') X, _ = data.get_mnist() train_X = X[:60000] val_X = X[60000:] ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) ae_model.finetune(train_X, 256, 100000, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) ae_model.save('mnist_pt.arg') ae_model.load('mnist_pt.arg') print "Training error:", ae_model.eval(train_X) print "Validation error:", ae_model.eval(val_X)
# lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) #V = np.zeros((train_X.shape[0],10)) V = np.random.rand(train_X.shape[0], K) / 10 lambda_v_rt = np.ones((train_X.shape[0], K)) * sqrt(lv) U, V, theta, BCD_loss = ae_model.finetune( train_X, R, V, lambda_v_rt, lambda_u, lambda_v, dir_save, batch_size, num_iter, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000, 0.1)) #ae_model.save('cdl_pt.arg') np.savetxt(dir_save + '/final-U.dat', U, fmt='%.5f', comments='') np.savetxt(dir_save + '/final-V.dat', V, fmt='%.5f', comments='') np.savetxt(dir_save + '/final-theta.dat', theta, fmt='%.5f', comments='') #ae_model.load('cdl_pt.arg') Recon_loss = lambda_v / lv * ae_model.eval(train_X, V, lambda_v_rt) print "Training error: %.3f" % (BCD_loss + Recon_loss) fp = open(dir_save + '/cdl.log', 'a') fp.write("Training error: %.3f\n" % (BCD_loss + Recon_loss)) fp.close() #print "Validation error:", ae_model.eval(val_X)
import logging import mnist_data as data from math import sqrt from autoencoder import AutoEncoderModel if __name__ == '__main__': lv = 1e-2# lv/ln in CDL # set to INFO to see less information during training logging.basicConfig(level=logging.DEBUG) #ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2, # internal_act='relu', output_act='relu') ae_model = AutoEncoderModel(mx.cpu(2), [784,500,500,2000,10], pt_dropout=0.2, internal_act='relu', output_act='relu') X, _ = data.get_mnist() train_X = X[:60000] val_X = X[60000:] #ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0, # lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) #V = np.zeros((train_X.shape[0],10)) V = np.random.rand(train_X.shape[0],10)/10 lambda_v_rt = np.ones((train_X.shape[0],10))*sqrt(lv) ae_model.finetune(train_X, V, lambda_v_rt, 256, 20, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) ae_model.save('mnist_pt.arg') ae_model.load('mnist_pt.arg') print "Training error:", ae_model.eval(train_X,V,lambda_v_rt) #print "Validation error:", ae_model.eval(val_X)
#ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2, # internal_act='relu', output_act='relu') #mx.cpu() no param needed for cpu. ae_model = AutoEncoderModel(mx.cpu(), [X.shape[1],100,K], pt_dropout=0.2, internal_act='relu', output_act='relu') train_X = X #ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0, # lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) #V = np.zeros((train_X.shape[0],10)) V = np.random.rand(train_X.shape[0],K)/10 lambda_v_rt = np.ones((train_X.shape[0],K))*sqrt(lv) U, V, theta, BCD_loss = ae_model.finetune(train_X, R, V, lambda_v_rt, lambda_u, lambda_v, dir_save, batch_size, num_iter, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) #ae_model.save('cdl_pt.arg') np.savetxt(dir_save+'/final-U.dat',U,fmt='%.5f',comments='') np.savetxt(dir_save+'/final-V.dat',V,fmt='%.5f',comments='') np.savetxt(dir_save+'/final-theta.dat',theta,fmt='%.5f',comments='') #ae_model.load('cdl_pt.arg') Recon_loss = lambda_v/lv*ae_model.eval(train_X,V,lambda_v_rt) print "Training error: %.3f" % (BCD_loss+Recon_loss) fp = open(dir_save+'/cdl.log','a') fp.write("Training error: %.3f\n" % (BCD_loss+Recon_loss)) fp.close() #print "Validation error:", ae_model.eval(val_X)
def setup(self, X, num_centers, alpha, znum, save_to='dec_model'): self.sep = int(X.shape[0] * 0.75) X_train = X[:self.sep] X_val = X[self.sep:] batch_size = 32 # 160 32*5 = update_interval*5 # Train or Read autoencoder: note is not dependent on number of clusters just on z latent size ae_model = AutoEncoderModel(self.xpu, [X.shape[1], 500, 500, 2000, znum], pt_dropout=0.2) if not os.path.exists(save_to + '_pt.arg'): ae_model.layerwise_pretrain(X_train, batch_size, 50000, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler( 20000, 0.1)) ae_model.finetune(X_train, batch_size, 100000, 'sgd', l_rate=0.1, decay=0.0, lr_scheduler=mx.misc.FactorScheduler(20000, 0.1)) ae_model.save(save_to + '_pt.arg') logging.log( logging.INFO, "Autoencoder Training error: %f" % ae_model.eval(X_train)) logging.log( logging.INFO, "Autoencoder Validation error: %f" % ae_model.eval(X_val)) else: ae_model.load(save_to + '_pt.arg') logging.log( logging.INFO, "Reading Autoencoder from file..: %s" % (save_to + '_pt.arg')) logging.log( logging.INFO, "Autoencoder Training error: %f" % ae_model.eval(X_train)) logging.log( logging.INFO, "Autoencoder Validation error: %f" % ae_model.eval(X_val)) self.ae_model = ae_model logging.log(logging.INFO, "finished reading Autoencoder from file..: ") # prep model for clustering self.dec_op = DECModel.DECLoss(num_centers, alpha) label = mx.sym.Variable('label') self.feature = self.ae_model.encoder self.loss = self.dec_op(data=self.ae_model.encoder, label=label, name='dec') self.args.update({ k: v for k, v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments() }) self.args['dec_mu'] = mx.nd.empty( (num_centers, self.ae_model.dims[-1]), ctx=self.xpu) self.args_grad.update({ k: mx.nd.empty(v.shape, ctx=self.xpu) for k, v in self.args.items() }) self.args_mult.update( {k: k.endswith('bias') and 2.0 or 1.0 for k in self.args}) self.num_centers = num_centers self.znum = znum self.batch_size = batch_size self.G = self.ae_model.eval(X_train) / self.ae_model.eval(X_val)