def main():
"""test for Sparse AE"""
os.system('rm -rf log')
T = []
T1 = SparseAE(64, 49, optimize_method='bfgs', max_iter=100,
debug=0, verbose=True, tol=1e-8, mini_batch=32,
logger='bfgs.csv')
T2 = SparseAE(64, 49, optimize_method='cg', max_iter=100,
debug=0, verbose=True, tol=1e-8, mini_batch=32,
logger='cg.csv')
T3 = SparseAE(64, 49, optimize_method='sgd', max_iter=10,
debug=0, verbose=True, tol=1e-8, mini_batch=64,
momentum=True, momen_beta=.95, alpha=.01, adastep=1,
logger='sgd.csv')
T.append(T1)
T.append(T2)
T.append(T3)
X = vs.load_sample('IMAGES25.mat', patch_size=8, n_patches=81920)
name = ['bfgs.png', 'cg.png', 'sgd.png']
for i in range(3):
try:
T[i].train(X)
except:
pass
T[i].devec_theta()
vs.disp_effect(T[i].w1, fname=name[i])
def plot_weight():
weight_sgd_mb4096 = listdir_sort('log/epoch/sgd_mb4096')
weight_sgd_mb1024 = listdir_sort('log/epoch/sgd_mb1024')
weight_sgd_mb256 = listdir_sort('log/epoch/sgd_mb256')
weight_sgd_mb64 = listdir_sort('log/epoch/sgd_mb64')
weight_sgd_mb0 = listdir_sort('log/epoch/sgd_mb0')
for weight_file_elem in weight_sgd_mb4096:
_weight_file_elem = 'log/epoch/sgd_mb4096/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd_mb4096/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_mb1024:
_weight_file_elem = 'log/epoch/sgd_mb1024/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd_mb1024/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_mb256:
_weight_file_elem = 'log/epoch/sgd_mb256/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd_mb256/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_mb64:
_weight_file_elem = 'log/epoch/sgd_mb64/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd_mb64/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_mb0:
_weight_file_elem = 'log/epoch/sgd_mb0/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd_mb0/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
print 'weights plotting done.'
def plot_weight():
weight_sgd_good = listdir_sort('log/timer/sgd_good')
weight_sgd_ill = listdir_sort('log/timer/sgd_ill')
weight_sgd_worse = listdir_sort('log/timer/sgd_worse')
weight_sgd_worst = listdir_sort('log/timer/sgd_worst')
for weight_file_elem in weight_sgd_good:
_weight_file_elem = 'log/timer/sgd_good/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/timer/sgd_good/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_ill:
_weight_file_elem = 'log/timer/sgd_ill/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/timer/sgd_ill/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_worse:
_weight_file_elem = 'log/timer/sgd_worse/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/timer/sgd_worse/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_worst:
_weight_file_elem = 'log/timer/sgd_worst/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/timer/sgd_worst/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
print 'weights plotting done.'
def plot_weight():
weight_sgd_ada = listdir_sort('log/epoch/sgd_ada')
weight_sgd_Nada = listdir_sort('log/epoch/sgd_Nada')
weight_sgd_ada95 = listdir_sort('log/epoch/sgd_ada95')
weight_sgd_Nada95 = listdir_sort('log/epoch/sgd_Nada95')
for weight_file_elem in weight_sgd_ada:
_weight_file_elem = 'log/epoch/sgd_ada/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd_ada/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_Nada:
_weight_file_elem = 'log/epoch/sgd_Nada/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd_Nada/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_ada95:
_weight_file_elem = 'log/epoch/sgd_ada95/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd_ada95/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd_Nada95:
_weight_file_elem = 'log/epoch/sgd_Nada95/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd_Nada95/' + weight_file_elem.zfill(
4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
print 'weights plotting done.'
def main():
"""Tuning for SparseAE"""
# First layer
T = SparseAE(64,
25,
optimize_method='cg',
max_iter=400,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=32)
X = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=10000)
T.train(X)
T.devec_theta()
# vs.disp_effect(T.w1, '1.jpg')
# Second layer
rho = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100]
beta = [3e-3, 3e-2, 9e-2, 3e-1, 9e-1, 3, 9, 30, 90, 300]
lamb = [1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10]
param = product(rho, beta, lamb)
param_str = ['rho', 'sparse_beta', 'lamb']
param = map(lambda x: dict(zip(param_str, x)), param)
X = activate(np.dot(T.w1, X) + T.b1)
if not os.path.isdir('./imgs'):
os.system('mkdir imgs')
for idx, param_elem in enumerate(param):
import warnings
warnings.filterwarnings('error')
try:
S = SparseAE(25,
9,
optimize_method='cg',
max_iter=400,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=32,
**param_elem)
S.train(X)
S.devec_theta()
fname = 'imgs/' + str(idx) + '.jpg'
vs.disp_effect(S.w1, fname=fname)
except:
fname = 'imgs/' + 'log'
fid = open(fname, 'w')
fid.write('Exception: ' + str(idx) + '\n')
fid.close()
def plot_weight(num):
weight_bfgs_file = listdir_sort('cmp' + str(num) + '/log/epoch/bfgs')
weight_cg_file = listdir_sort('cmp' + str(num) + '/log/epoch/cg')
weight_sgd_file = listdir_sort('cmp' + str(num) + '/log/epoch/sgd')
for weight_bfgs_file_elem in weight_bfgs_file:
_weight_bfgs_file_elem = 'cmp' + str(
num) + '/log/epoch/bfgs/' + weight_bfgs_file_elem
weight_bfgs = np.loadtxt(_weight_bfgs_file_elem, delimiter=',')
file_elem = 'cmp' + str(
num) + '/log/epoch/bfgs/' + weight_bfgs_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_bfgs, fname=file_elem)
for weight_cg_file_elem in weight_cg_file:
_weight_cg_file_elem = 'cmp' + str(
num) + '/log/epoch/cg/' + weight_cg_file_elem
weight_cg = np.loadtxt(_weight_cg_file_elem, delimiter=',')
file_elem = 'cmp' + str(
num) + '/log/epoch/cg/' + weight_cg_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_cg, fname=file_elem)
for weight_sgd_file_elem in weight_sgd_file:
_weight_sgd_file_elem = 'cmp' + str(
num) + '/log/epoch/sgd/' + weight_sgd_file_elem
weight_sgd = np.loadtxt(_weight_sgd_file_elem, delimiter=',')
file_elem = 'cmp' + str(
num) + '/log/epoch/sgd/' + weight_sgd_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
print 'weights plotting done.'
def disp_test():
wgt1 = np.random.rand(49, 81)
vs.disp_effect(wgt1)
wgt2 = np.random.rand(48, 150)
vs.disp_effect(wgt2)
def main():
"""Tuning for SparseAE"""
# First layer
T = SparseAE(64,
49,
optimize_method='cg',
max_iter=400,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=32)
X = vs.load_sample('IMAGES.mat', patch_size=8, n_patches=10000)
T.train(X)
T.devec_theta()
vs.disp_effect(T.w1, fname='Fst_lyr.jpg')
# Second layer
rho = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100]
beta = [3e-3, 3e-2, 9e-2, 3e-1, 9e-1, 3, 9, 30, 90, 300]
lamb = [1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10]
param = product(rho, beta, lamb)
param_str = ['rho', 'sparse_beta', 'lamb']
param = map(lambda x: dict(zip(param_str, x)), param)
X = activate(np.dot(T.w1, X) + T.b1)
if not os.path.isdir('./imgs'):
os.system('mkdir imgs')
'''
for idx, param_elem in enumerate(param):
import warnings
warnings.filterwarnings('error')
try:
S = SparseAE(49, 36, optimize_method='cg', max_iter=400,
debug=0, verbose=True, tol=1e-8, mini_batch=32,
**param_elem)
S.train(X)
S.devec_theta()
fname = 'imgs/' + str(idx) + '.jpg'
vs.disp_effect(S.w1, fname=fname)
except:
fname = 'imgs/' + 'log'
fid = open(fname, 'w')
fid.write('Exception: ' + str(idx) + '\n')
fid.close()
'''
# Break point re-computing
fls = os.listdir('./imgs/weight')
fls = map(lambda x: int(x[:x.find('.')]), fls)
for fl in fls:
param = param[:fl] + param[fl + 1:]
# dpark parallel computing
dpark_ctx = DparkContext('process')
dpark_n_length = len(param)
dpark_n_block = 50
if not os.path.isdir('./imgs/weight'):
os.system('mkdir imgs/weight')
print '%d models await training.' % dpark_n_length
def map_iter(param_enum):
idx = param_enum[0][0] * int(ceil(dpark_n_length / dpark_n_block)) +\
param_enum[0][1]
import warnings
warnings.filterwarnings('error')
try:
S = SparseAE(49,
36,
optimize_method='cg',
max_iter=400,
debug=0,
verbose=True,
tol=1e-8,
mini_batch=32,
**param_enum[1])
S.train(
np.array(X)) # dpark converts X, 'np.ndarray' to 'instance'
S.devec_theta()
fname = 'imgs/weight/' + str(idx) + '.csv'
# vs.disp_effect(S.w1, fname=fname) # dpark doesn't support plt.savefig()
np.savetxt(fname, S.w1, delimiter=',')
except:
import traceback
traceback.print_exc()
fname = 'imgs/' + 'log'
fid = open(fname, 'w')
fid.write('Training exception: ' + str(idx) + '\n')
fid.close()
dpark_ctx.makeRDD(param, dpark_n_block).enumerate().foreach(map_iter)
print 'Done.'
# Visualizing
for i in range(len(param)):
fname = 'imgs/weight/' + str(i) + '.csv'
if not os.path.isfile(fname):
continue
w = np.loadtxt(fname, delimiter=',')
fname_img = 'imgs/' + str(i) + '.jpg'
vs.disp_effect(w, fname=fname_img)
print i, 'visualization done.'
def plot_weight():
weight_sgd99 = listdir_sort('log/epoch/sgd99')
weight_sgd95 = listdir_sort('log/epoch/sgd95')
weight_sgd90 = listdir_sort('log/epoch/sgd90')
weight_sgd80 = listdir_sort('log/epoch/sgd80')
weight_sgd60 = listdir_sort('log/epoch/sgd60')
weight_sgd40 = listdir_sort('log/epoch/sgd40')
weight_sgd20 = listdir_sort('log/epoch/sgd20')
weight_sgd0 = listdir_sort('log/epoch/sgd0')
for weight_file_elem in weight_sgd99:
_weight_file_elem = 'log/epoch/sgd99/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd99/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd95:
_weight_file_elem = 'log/epoch/sgd95/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd95/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd90:
_weight_file_elem = 'log/epoch/sgd90/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd90/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd80:
_weight_file_elem = 'log/epoch/sgd80/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd80/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd60:
_weight_file_elem = 'log/epoch/sgd60/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd60/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd40:
_weight_file_elem = 'log/epoch/sgd40/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd40/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd20:
_weight_file_elem = 'log/epoch/sgd20/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd20/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
for weight_file_elem in weight_sgd0:
_weight_file_elem = 'log/epoch/sgd0/' + weight_file_elem
weight_sgd = np.loadtxt(_weight_file_elem, delimiter=',')
file_elem = 'log/epoch/sgd0/' + weight_file_elem.zfill(4) + '.png'
vs.disp_effect(weight_sgd, fname=file_elem)
print 'weights plotting done.'
