def main():
"""test for Sparse AE"""
os.system('rm -rf log')
T = []
T1 = SparseAE(64, 49, optimize_method='cg', max_iter=200,
debug=0, verbose=True, tol=1e-8, mini_batch=64,
momentum=0, momen_beta=.95, alpha=.01, adastep=1,
logger='cg_good.csv')
T2 = SparseAE(64, 49, optimize_method='cg', max_iter=200,
debug=0, verbose=True, tol=1e-8, mini_batch=64,
momentum=0, momen_beta=.95, alpha=.01, adastep=1,
logger='cg_ill.csv')
T3 = SparseAE(64, 49, optimize_method='cg', max_iter=200,
debug=0, verbose=True, tol=1e-8, mini_batch=64,
momentum=0, momen_beta=.95, alpha=.01, adastep=1,
logger='cg_worse.csv')
T4 = SparseAE(64, 49, optimize_method='cg', max_iter=200,
debug=0, verbose=True, tol=1e-8, mini_batch=64,
momentum=0, momen_beta=.95, alpha=.01, adastep=1,
logger='cg_worst.csv')
T.append(T1)
T.append(T2)
T.append(T3)
T.append(T4)
X = []
X1 = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=20480)
X2 = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=20480)
X3 = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=20480)
X4 = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=20480)
idx_jitter = np.random.permutation(64)
idx_jitter_pos = idx_jitter[: 16]
idx_jitter_neg = idx_jitter[-16: ]
for i, j in zip(idx_jitter_pos[:3], idx_jitter_neg[:3]):
X2[i, :] = np.squeeze(np.random.normal(0.89, 0.01, [20480, ]))
X2[j, :] = np.squeeze(np.random.normal(0.11, 0.01, [20480, ]))
for i, j in zip(idx_jitter_pos[:8], idx_jitter_neg[:8]):
X3[i, :] = np.squeeze(np.random.normal(0.899, 0.0001, [20480, ]))
X3[j, :] = np.squeeze(np.random.normal(0.101, 0.0001, [20480, ]))
for i, j in zip(idx_jitter_pos, idx_jitter_neg):
X4[i, :] = np.squeeze(np.random.normal(0.89999, 0.000001, [20480, ]))
X4[j, :] = np.squeeze(np.random.normal(0.10001, 0.000001, [20480, ]))
X.extend([X1, X2, X3, X4])
print "cond(X1): ", np.linalg.cond(X1)
print "cond(X2): ", np.linalg.cond(X2)
print "cond(X3): ", np.linalg.cond(X3)
print "cond(X4): ", np.linalg.cond(X4)
for i in range(len(T)):
try:
T[i].train(X[i])
except:
print 'Training shut down\n'
pass
def main():
"""test for Sparse AE"""
os.system('rm -rf log')
T = []
T1 = SparseAE(64, 49, optimize_method='bfgs', max_iter=100,
debug=0, verbose=True, tol=1e-8, mini_batch=32,
logger='bfgs.csv')
T2 = SparseAE(64, 49, optimize_method='cg', max_iter=100,
debug=0, verbose=True, tol=1e-8, mini_batch=32,
logger='cg.csv')
T3 = SparseAE(64, 49, optimize_method='sgd', max_iter=10,
debug=0, verbose=True, tol=1e-8, mini_batch=64,
momentum=True, momen_beta=.95, alpha=.01, adastep=1,
logger='sgd.csv')
T.append(T1)
T.append(T2)
T.append(T3)
X = vs.load_sample('IMAGES25.mat', patch_size=8, n_patches=81920)
name = ['bfgs.png', 'cg.png', 'sgd.png']
for i in range(3):
try:
T[i].train(X)
except:
pass
T[i].devec_theta()
vs.disp_effect(T[i].w1, fname=name[i])
def load_test():
a = vs.load_sample('IMAGES.mat', patch_size=4, n_patches=400)
if a.shape == (16, 400):
print 'Load test testing correct.'
return
else:
print 'Load test testing WRONG.'
return
def main():
"""Tuning for SparseAE"""
# First layer
T = SparseAE(64,
25,
optimize_method='cg',
max_iter=400,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=32)
X = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=10000)
T.train(X)
T.devec_theta()
# vs.disp_effect(T.w1, '1.jpg')
# Second layer
rho = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100]
beta = [3e-3, 3e-2, 9e-2, 3e-1, 9e-1, 3, 9, 30, 90, 300]
lamb = [1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10]
param = product(rho, beta, lamb)
param_str = ['rho', 'sparse_beta', 'lamb']
param = map(lambda x: dict(zip(param_str, x)), param)
X = activate(np.dot(T.w1, X) + T.b1)
if not os.path.isdir('./imgs'):
os.system('mkdir imgs')
for idx, param_elem in enumerate(param):
import warnings
warnings.filterwarnings('error')
try:
S = SparseAE(25,
9,
optimize_method='cg',
max_iter=400,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=32,
**param_elem)
S.train(X)
S.devec_theta()
fname = 'imgs/' + str(idx) + '.jpg'
vs.disp_effect(S.w1, fname=fname)
except:
fname = 'imgs/' + 'log'
fid = open(fname, 'w')
fid.write('Exception: ' + str(idx) + '\n')
fid.close()
def main():
"""test for Sparse AE"""
os.system('rm -rf log')
T = []
T1 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=2,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=256,
momentum=True,
momen_beta=.99,
alpha=.01,
adastep=1,
logger='sgd99.csv')
T2 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=2,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=256,
momentum=True,
momen_beta=.95,
alpha=.01,
adastep=1,
logger='sgd95.csv')
T3 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=2,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=256,
momentum=True,
momen_beta=.90,
alpha=.01,
adastep=1,
logger='sgd90.csv')
T4 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=2,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=256,
momentum=True,
momen_beta=.80,
alpha=.01,
adastep=1,
logger='sgd80.csv')
T5 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=2,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=256,
momentum=True,
momen_beta=.60,
alpha=.01,
adastep=1,
logger='sgd60.csv')
T6 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=2,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=256,
momentum=True,
momen_beta=.40,
alpha=.01,
adastep=1,
logger='sgd40.csv')
T7 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=2,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=256,
momentum=True,
momen_beta=.20,
alpha=.01,
adastep=1,
logger='sgd20.csv')
T8 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=2,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=256,
momentum=False,
momen_beta=0,
alpha=.01,
adastep=1,
logger='sgd0.csv')
T.append(T1)
T.append(T2)
T.append(T3)
T.append(T4)
T.append(T5)
T.append(T6)
T.append(T7)
T.append(T8)
X = vs.load_sample('IMAGES25.mat', patch_size=8, n_patches=102400)
for i in range(len(T)):
try:
T[i].train(X)
except:
pass
def main():
"""Tuning for SparseAE"""
# First layer
T = SparseAE(64,
49,
optimize_method='cg',
max_iter=400,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=32)
X = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=10000)
T.train(X)
T.devec_theta()
vs.disp_effect(T.w1, fname='Fst_lyr.jpg')
# Second layer
rho = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100]
beta = [3e-3, 3e-2, 9e-2, 3e-1, 9e-1, 3, 9, 30, 90, 300]
lamb = [1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10]
param = product(rho, beta, lamb)
param_str = ['rho', 'sparse_beta', 'lamb']
param = map(lambda x: dict(zip(param_str, x)), param)
X = activate(np.dot(T.w1, X) + T.b1)
if not os.path.isdir('./imgs'):
os.system('mkdir imgs')
'''
for idx, param_elem in enumerate(param):
import warnings
warnings.filterwarnings('error')
try:
S = SparseAE(49, 36, optimize_method='cg', max_iter=400,
debug=0, verbose=True, tol=1e-8, mini_batch=32,
**param_elem)
S.train(X)
S.devec_theta()
fname = 'imgs/' + str(idx) + '.jpg'
vs.disp_effect(S.w1, fname=fname)
except:
fname = 'imgs/' + 'log'
fid = open(fname, 'w')
fid.write('Exception: ' + str(idx) + '\n')
fid.close()
'''
# Break point re-computing
fls = os.listdir('./imgs/weight')
fls = map(lambda x: int(x[:x.find('.')]), fls)
for fl in fls:
param = param[:fl] + param[fl + 1:]
# dpark parallel computing
dpark_ctx = DparkContext('process')
dpark_n_length = len(param)
dpark_n_block = 50
if not os.path.isdir('./imgs/weight'):
os.system('mkdir imgs/weight')
print '%d models await training.' % dpark_n_length
def map_iter(param_enum):
idx = param_enum[0][0] * int(ceil(dpark_n_length / dpark_n_block)) +\
param_enum[0][1]
import warnings
warnings.filterwarnings('error')
try:
S = SparseAE(49,
36,
optimize_method='cg',
max_iter=400,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=32,
**param_enum[1])
S.train(
np.array(X)) # dpark converts X, 'np.ndarray' to 'instance'
S.devec_theta()
fname = 'imgs/weight/' + str(idx) + '.csv'
# vs.disp_effect(S.w1, fname=fname) # dpark doesn't support plt.savefig()
np.savetxt(fname, S.w1, delimiter=',')
except:
import traceback
traceback.print_exc()
fname = 'imgs/' + 'log'
fid = open(fname, 'w')
fid.write('Training exception: ' + str(idx) + '\n')
fid.close()
dpark_ctx.makeRDD(param, dpark_n_block).enumerate().foreach(map_iter)
print 'Done.'
# Visualizing
for i in range(len(param)):
fname = 'imgs/weight/' + str(i) + '.csv'
if not os.path.isfile(fname):
continue
w = np.loadtxt(fname, delimiter=',')
fname_img = 'imgs/' + str(i) + '.jpg'
vs.disp_effect(w, fname=fname_img)
print i, 'visualization done.'
def main():
"""test for Sparse AE"""
os.system('rm -rf log')
T = []
T1 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=10,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=4096,
momentum=True,
momen_beta=.95,
alpha=.01,
adastep=0,
logger='sgd_mb4096.csv')
T2 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=10,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=1024,
momentum=True,
momen_beta=.95,
alpha=.01,
adastep=1,
logger='sgd_mb1024.csv')
T3 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=10,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=256,
momentum=True,
momen_beta=.95,
alpha=.01,
adastep=0,
logger='sgd_mb256.csv')
T4 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=10,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=64,
momentum=True,
momen_beta=.95,
alpha=.01,
adastep=1,
logger='sgd_mb64.csv')
T5 = SparseAE(64,
49,
optimize_method='sgd',
max_iter=10,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=0,
momentum=True,
momen_beta=.95,
alpha=.01,
adastep=1,
logger='sgd_mb0.csv')
T.append(T1)
T.append(T2)
T.append(T3)
T.append(T4)
T.append(T5)
X = vs.load_sample('IMAGES25.mat', patch_size=8, n_patches=102400)
for i in range(len(T)):
try:
T[i].train(X)
except:
pass
