def main(): """test for Sparse AE""" os.system('rm -rf log') T = [] T1 = SparseAE(64, 49, optimize_method='cg', max_iter=200, debug=0, verbose=True, tol=1e-8, mini_batch=64, momentum=0, momen_beta=.95, alpha=.01, adastep=1, logger='cg_good.csv') T2 = SparseAE(64, 49, optimize_method='cg', max_iter=200, debug=0, verbose=True, tol=1e-8, mini_batch=64, momentum=0, momen_beta=.95, alpha=.01, adastep=1, logger='cg_ill.csv') T3 = SparseAE(64, 49, optimize_method='cg', max_iter=200, debug=0, verbose=True, tol=1e-8, mini_batch=64, momentum=0, momen_beta=.95, alpha=.01, adastep=1, logger='cg_worse.csv') T4 = SparseAE(64, 49, optimize_method='cg', max_iter=200, debug=0, verbose=True, tol=1e-8, mini_batch=64, momentum=0, momen_beta=.95, alpha=.01, adastep=1, logger='cg_worst.csv') T.append(T1) T.append(T2) T.append(T3) T.append(T4) X = [] X1 = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=20480) X2 = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=20480) X3 = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=20480) X4 = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=20480) idx_jitter = np.random.permutation(64) idx_jitter_pos = idx_jitter[: 16] idx_jitter_neg = idx_jitter[-16: ] for i, j in zip(idx_jitter_pos[:3], idx_jitter_neg[:3]): X2[i, :] = np.squeeze(np.random.normal(0.89, 0.01, [20480, ])) X2[j, :] = np.squeeze(np.random.normal(0.11, 0.01, [20480, ])) for i, j in zip(idx_jitter_pos[:8], idx_jitter_neg[:8]): X3[i, :] = np.squeeze(np.random.normal(0.899, 0.0001, [20480, ])) X3[j, :] = np.squeeze(np.random.normal(0.101, 0.0001, [20480, ])) for i, j in zip(idx_jitter_pos, idx_jitter_neg): X4[i, :] = np.squeeze(np.random.normal(0.89999, 0.000001, [20480, ])) X4[j, :] = np.squeeze(np.random.normal(0.10001, 0.000001, [20480, ])) X.extend([X1, X2, X3, X4]) print "cond(X1): ", np.linalg.cond(X1) print "cond(X2): ", np.linalg.cond(X2) print "cond(X3): ", np.linalg.cond(X3) print "cond(X4): ", np.linalg.cond(X4) for i in range(len(T)): try: T[i].train(X[i]) except: print 'Training shut down\n' pass
def main(): """test for Sparse AE""" os.system('rm -rf log') T = [] T1 = SparseAE(64, 49, optimize_method='bfgs', max_iter=100, debug=0, verbose=True, tol=1e-8, mini_batch=32, logger='bfgs.csv') T2 = SparseAE(64, 49, optimize_method='cg', max_iter=100, debug=0, verbose=True, tol=1e-8, mini_batch=32, logger='cg.csv') T3 = SparseAE(64, 49, optimize_method='sgd', max_iter=10, debug=0, verbose=True, tol=1e-8, mini_batch=64, momentum=True, momen_beta=.95, alpha=.01, adastep=1, logger='sgd.csv') T.append(T1) T.append(T2) T.append(T3) X = vs.load_sample('IMAGES25.mat', patch_size=8, n_patches=81920) name = ['bfgs.png', 'cg.png', 'sgd.png'] for i in range(3): try: T[i].train(X) except: pass T[i].devec_theta() vs.disp_effect(T[i].w1, fname=name[i])
def load_test(): a = vs.load_sample('IMAGES.mat', patch_size=4, n_patches=400) if a.shape == (16, 400): print 'Load test testing correct.' return else: print 'Load test testing WRONG.' return
def main(): """Tuning for SparseAE""" # First layer T = SparseAE(64, 25, optimize_method='cg', max_iter=400, debug=0, verbose=True, tol=1e-8, mini_batch=32) X = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=10000) T.train(X) T.devec_theta() # vs.disp_effect(T.w1, '1.jpg') # Second layer rho = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100] beta = [3e-3, 3e-2, 9e-2, 3e-1, 9e-1, 3, 9, 30, 90, 300] lamb = [1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10] param = product(rho, beta, lamb) param_str = ['rho', 'sparse_beta', 'lamb'] param = map(lambda x: dict(zip(param_str, x)), param) X = activate(np.dot(T.w1, X) + T.b1) if not os.path.isdir('./imgs'): os.system('mkdir imgs') for idx, param_elem in enumerate(param): import warnings warnings.filterwarnings('error') try: S = SparseAE(25, 9, optimize_method='cg', max_iter=400, debug=0, verbose=True, tol=1e-8, mini_batch=32, **param_elem) S.train(X) S.devec_theta() fname = 'imgs/' + str(idx) + '.jpg' vs.disp_effect(S.w1, fname=fname) except: fname = 'imgs/' + 'log' fid = open(fname, 'w') fid.write('Exception: ' + str(idx) + '\n') fid.close()
def main(): """test for Sparse AE""" os.system('rm -rf log') T = [] T1 = SparseAE(64, 49, optimize_method='sgd', max_iter=2, debug=0, verbose=True, tol=1e-8, mini_batch=256, momentum=True, momen_beta=.99, alpha=.01, adastep=1, logger='sgd99.csv') T2 = SparseAE(64, 49, optimize_method='sgd', max_iter=2, debug=0, verbose=True, tol=1e-8, mini_batch=256, momentum=True, momen_beta=.95, alpha=.01, adastep=1, logger='sgd95.csv') T3 = SparseAE(64, 49, optimize_method='sgd', max_iter=2, debug=0, verbose=True, tol=1e-8, mini_batch=256, momentum=True, momen_beta=.90, alpha=.01, adastep=1, logger='sgd90.csv') T4 = SparseAE(64, 49, optimize_method='sgd', max_iter=2, debug=0, verbose=True, tol=1e-8, mini_batch=256, momentum=True, momen_beta=.80, alpha=.01, adastep=1, logger='sgd80.csv') T5 = SparseAE(64, 49, optimize_method='sgd', max_iter=2, debug=0, verbose=True, tol=1e-8, mini_batch=256, momentum=True, momen_beta=.60, alpha=.01, adastep=1, logger='sgd60.csv') T6 = SparseAE(64, 49, optimize_method='sgd', max_iter=2, debug=0, verbose=True, tol=1e-8, mini_batch=256, momentum=True, momen_beta=.40, alpha=.01, adastep=1, logger='sgd40.csv') T7 = SparseAE(64, 49, optimize_method='sgd', max_iter=2, debug=0, verbose=True, tol=1e-8, mini_batch=256, momentum=True, momen_beta=.20, alpha=.01, adastep=1, logger='sgd20.csv') T8 = SparseAE(64, 49, optimize_method='sgd', max_iter=2, debug=0, verbose=True, tol=1e-8, mini_batch=256, momentum=False, momen_beta=0, alpha=.01, adastep=1, logger='sgd0.csv') T.append(T1) T.append(T2) T.append(T3) T.append(T4) T.append(T5) T.append(T6) T.append(T7) T.append(T8) X = vs.load_sample('IMAGES25.mat', patch_size=8, n_patches=102400) for i in range(len(T)): try: T[i].train(X) except: pass
def main(): """Tuning for SparseAE""" # First layer T = SparseAE(64, 49, optimize_method='cg', max_iter=400, debug=0, verbose=True, tol=1e-8, mini_batch=32) X = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=10000) T.train(X) T.devec_theta() vs.disp_effect(T.w1, fname='Fst_lyr.jpg') # Second layer rho = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100] beta = [3e-3, 3e-2, 9e-2, 3e-1, 9e-1, 3, 9, 30, 90, 300] lamb = [1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10] param = product(rho, beta, lamb) param_str = ['rho', 'sparse_beta', 'lamb'] param = map(lambda x: dict(zip(param_str, x)), param) X = activate(np.dot(T.w1, X) + T.b1) if not os.path.isdir('./imgs'): os.system('mkdir imgs') ''' for idx, param_elem in enumerate(param): import warnings warnings.filterwarnings('error') try: S = SparseAE(49, 36, optimize_method='cg', max_iter=400, debug=0, verbose=True, tol=1e-8, mini_batch=32, **param_elem) S.train(X) S.devec_theta() fname = 'imgs/' + str(idx) + '.jpg' vs.disp_effect(S.w1, fname=fname) except: fname = 'imgs/' + 'log' fid = open(fname, 'w') fid.write('Exception: ' + str(idx) + '\n') fid.close() ''' # Break point re-computing fls = os.listdir('./imgs/weight') fls = map(lambda x: int(x[:x.find('.')]), fls) for fl in fls: param = param[:fl] + param[fl + 1:] # dpark parallel computing dpark_ctx = DparkContext('process') dpark_n_length = len(param) dpark_n_block = 50 if not os.path.isdir('./imgs/weight'): os.system('mkdir imgs/weight') print '%d models await training.' % dpark_n_length def map_iter(param_enum): idx = param_enum[0][0] * int(ceil(dpark_n_length / dpark_n_block)) +\ param_enum[0][1] import warnings warnings.filterwarnings('error') try: S = SparseAE(49, 36, optimize_method='cg', max_iter=400, debug=0, verbose=True, tol=1e-8, mini_batch=32, **param_enum[1]) S.train( np.array(X)) # dpark converts X, 'np.ndarray' to 'instance' S.devec_theta() fname = 'imgs/weight/' + str(idx) + '.csv' # vs.disp_effect(S.w1, fname=fname) # dpark doesn't support plt.savefig() np.savetxt(fname, S.w1, delimiter=',') except: import traceback traceback.print_exc() fname = 'imgs/' + 'log' fid = open(fname, 'w') fid.write('Training exception: ' + str(idx) + '\n') fid.close() dpark_ctx.makeRDD(param, dpark_n_block).enumerate().foreach(map_iter) print 'Done.' # Visualizing for i in range(len(param)): fname = 'imgs/weight/' + str(i) + '.csv' if not os.path.isfile(fname): continue w = np.loadtxt(fname, delimiter=',') fname_img = 'imgs/' + str(i) + '.jpg' vs.disp_effect(w, fname=fname_img) print i, 'visualization done.'
def main(): """test for Sparse AE""" os.system('rm -rf log') T = [] T1 = SparseAE(64, 49, optimize_method='sgd', max_iter=10, debug=0, verbose=True, tol=1e-8, mini_batch=4096, momentum=True, momen_beta=.95, alpha=.01, adastep=0, logger='sgd_mb4096.csv') T2 = SparseAE(64, 49, optimize_method='sgd', max_iter=10, debug=0, verbose=True, tol=1e-8, mini_batch=1024, momentum=True, momen_beta=.95, alpha=.01, adastep=1, logger='sgd_mb1024.csv') T3 = SparseAE(64, 49, optimize_method='sgd', max_iter=10, debug=0, verbose=True, tol=1e-8, mini_batch=256, momentum=True, momen_beta=.95, alpha=.01, adastep=0, logger='sgd_mb256.csv') T4 = SparseAE(64, 49, optimize_method='sgd', max_iter=10, debug=0, verbose=True, tol=1e-8, mini_batch=64, momentum=True, momen_beta=.95, alpha=.01, adastep=1, logger='sgd_mb64.csv') T5 = SparseAE(64, 49, optimize_method='sgd', max_iter=10, debug=0, verbose=True, tol=1e-8, mini_batch=0, momentum=True, momen_beta=.95, alpha=.01, adastep=1, logger='sgd_mb0.csv') T.append(T1) T.append(T2) T.append(T3) T.append(T4) T.append(T5) X = vs.load_sample('IMAGES25.mat', patch_size=8, n_patches=102400) for i in range(len(T)): try: T[i].train(X) except: pass