tile_shape1 = (n_batch_w, 1)
tile_shape2 = (n_batch_w, n_y)
else:
tile_shape1 = (1, n_batch_w)
tile_shape2 = (n_y, n_batch_w)
for sample in range(idxs.shape[1]):
# Get some random testset datapoints
idx = idxs[:,sample]
np.savetxt(logdir+'/'+str(sample)+'_idx.txt', idx, fmt='%5u')
#with open(, "w") as text_file:
# text_file.write("Purchase Amount: %s" % TotalAmount)
human_x = test_x[:,idx].astype(np.float32)
human_y = test_y[:,idx].astype(np.float32)
image = paramgraphics.mat_to_img(human_x, dim_input, colorImg=colorImg, tile_shape=tile_shape1)
fname = logdir+'/'+str(sample)+'_human.png'
print('Saving to '+fname)
image.save(fname, 'PNG')
# Infer corresponding 'z'
A = np.ones((1, n_batch_w)).astype(np.float32)
q_mean, q_logvar = model.dist_qz['z'](f_enc(human_x).astype(np.float32), human_y, A)
z = q_mean
# set 'y'
y = np.zeros((n_y, n_y*n_batch_w))
for i in range(n_y):
if draw_rows: y[i,i::n_y] = 1
else: y[i,(n_batch_w*i):(n_batch_w*(i+1))] = 1
start = int(pertub_prob)
end = int(pertub_prob1)
data_perturbed = np.zeros(data.shape)
tmp_a = np.ones(width)
tmp_a[start:end] = 0
#print tmp_a.shape
#print tmp_a
tmp_b = np.tile(tmp_a, height)
print tmp_b.shape
print pertub_label.shape
pertub_label = (pertub_label.T*tmp_b).T
data_perturbed = pertub_label*data+(1-pertub_label)*data_perturbed
if pertub_type == 4:
sio.savemat('data_imputation/type_'+str(pertub_type)+'_params_'+str(int(pertub_prob*100))+'_noise_rawdata.mat', {'z_train' : x_train.T, 'z_test_original' : data, 'z_test' : data_perturbed, 'pertub_label' : pertub_label})
#print data_perturbed[:,:25].shape
image = paramgraphics.mat_to_img(data_perturbed[:,:25], (28,28), colorImg=False, scale=True)
image.save('data_imputation/test_noise_4_'+str(pertub_prob)+'.png', 'PNG')
elif pertub_type == 3:
sio.savemat('data_imputation/type_'+str(pertub_type)+'_params_'+str(pertub_prob)+'_noise_rawdata.mat', {'z_train' : x_train.T, 'z_test_original' : data, 'z_test' : data_perturbed, 'pertub_label' : pertub_label})
#print data_perturbed[:,:25].shape
image = paramgraphics.mat_to_img(data_perturbed[:,:25], (28,28), colorImg=False, scale=True)
image.save('data_imputation/test_noise_3_'+str(pertub_prob)+'.png', 'PNG')
elif pertub_type == 5:
sio.savemat('data_imputation/type_'+str(pertub_type)+'_params_'+str(start)+'_'+str(end)+'_noise_rawdata.mat', {'z_train' : x_train.T, 'z_test_original' : data, 'z_test' : data_perturbed, 'pertub_label' : pertub_label})
#print data_perturbed[:,:25].shape
image = paramgraphics.mat_to_img(data_perturbed[:,:25], (28,28), colorImg=False, scale=True)
image.save('data_imputation/test_noise_5_'+str(start)+'_'+str(end)+'.png', 'PNG')
def hook(epoch, t, ll):
if epoch%10 != 0:
return
ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat)
if math.isnan(ll_valid):
print "NaN detected. Reverting to saved best parameters"
ndict.set_value(model.v, ndict.loadz(logdir+'v.ndict.tar.gz'))
ndict.set_value(model.w, ndict.loadz(logdir+'w.ndict.tar.gz'))
return
if ll_valid > ll_valid_stats[0]:
ll_valid_stats[0] = ll_valid
ll_valid_stats[1] = 0
ndict.savez(ndict.get_value(model.v), logdir+'v_best')
ndict.savez(ndict.get_value(model.w), logdir+'w_best')
else:
ll_valid_stats[1] += 1
# Stop when not improving validation set performance in 100 iterations
if False and ll_valid_stats[1] > 1000:
print "Finished"
with open(logdir+'hook.txt', 'a') as f:
print >>f, "Finished"
exit()
# Log
ndict.savez(ndict.get_value(model.v), logdir+'v')
ndict.savez(ndict.get_value(model.w), logdir+'w')
print epoch, t, ll, ll_valid
with open(logdir+'hook.txt', 'a') as f:
print >>f, t, ll, ll_valid
if gfx:
# Graphics
v = {i: model.v[i].get_value() for i in model.v}
w = {i: model.w[i].get_value() for i in model.w}
tail = '-'+str(epoch)+'.png'
image = paramgraphics.mat_to_img(f_dec(v['w0x'][:].T), dim_input, True, colorImg=colorImg)
image.save(logdir+'q_w0x'+tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg)
image.save(logdir+'out_w'+tail, 'PNG')
_x = {'y': np.random.multinomial(1, [1./n_y]*n_y, size=144).T}
_, _, _z_confab = model.gen_xz(_x, {}, n_batch=144)
image = paramgraphics.mat_to_img(f_dec(_z_confab['x']), dim_input, colorImg=colorImg)
image.save(logdir+'samples'+tail, 'PNG')
_, _, _z_confab = model.gen_xz(y_sample, z_sample, n_batch=144)
image = paramgraphics.mat_to_img(f_dec(_z_confab['x']), dim_input, colorImg=colorImg)
image.save(logdir+'samples_fixed'+tail, 'PNG')
if n_z == 2:
import ImageFont
import ImageDraw
n_width = 10
submosaic_offset = 15
submosaic_width = (dim_input[1]*n_width)
submosaic_height = (dim_input[0]*n_width)
mosaic = Image.new("RGB", (submosaic_width*mosaic_w, submosaic_offset+submosaic_height*mosaic_h))
for digit in range(0,n_y):
if digit >= mosaic_h*mosaic_w: continue
_x = {}
n_batch_plot = n_width*n_width
_x['y'] = np.zeros((n_y,n_batch_plot))
_x['y'][digit,:] = 1
_z = {'z':np.zeros((2,n_width**2))}
for i in range(0,n_width):
for j in range(0,n_width):
_z['z'][0,n_width*i+j] = scipy.stats.norm.ppf(float(i)/n_width+0.5/n_width)
_z['z'][1,n_width*i+j] = scipy.stats.norm.ppf(float(j)/n_width+0.5/n_width)
_x, _, _z_confab = model.gen_xz(_x, _z, n_batch=n_batch_plot)
x_samples = _z_confab['x']
image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg, tile_spacing=(0,0))
#image.save(logdir+'samples_digit_'+str(digit)+'_'+tail, 'PNG')
mosaic_x = (digit%mosaic_w)*submosaic_width
mosaic_y = submosaic_offset+int(digit/mosaic_w)*submosaic_height
mosaic.paste(image, (mosaic_x, mosaic_y))
draw = ImageDraw.Draw(mosaic)
draw.text((1,1),"Epoch #"+str(epoch)+" Loss="+str(int(ll)))
#plt.savefig(logdir+'mosaic'+tail, format='PNG')
mosaic.save(logdir+'mosaic'+tail, 'PNG')
def hook(epoch, t, ll):
if epoch%10 != 0: return
n_batch_n = n_batch
if n_batch_n > n_valid:
n_batch_n = n_valid
ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch_n, byteToFloat=byteToFloat)
ll_test = ll_valid
#if not dataset == 'mnist_binarized':
if not dataset == 'svhn':
ll_test, _ = model.est_loglik(x_test, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat)
# Log
ndict.savez(ndict.get_value(model.v), logdir+'v')
ndict.savez(ndict.get_value(model.w), logdir+'w')
def infer(data, n_batch=1000):
#print '--', n_batch
size = data['x'].shape[1]
res = np.zeros((sum(n_hidden), size))
res1 = np.zeros((n_z,size))
res2 = np.zeros((n_hidden[-1],size))
res3 = np.zeros((n_z,size))
for i in range(0, size, n_batch):
idx_to = min(size, i+n_batch)
x_batch = ndict.getCols(data, i, idx_to)
# may have bugs
nn_batch = idx_to - i
_x, _z, _z_confab = model.gen_xz(x_batch, {}, nn_batch)
x_samples = _z_confab['x']
for (hi, hidden) in enumerate(_z_confab['hidden']):
res[sum(n_hidden[:hi]):sum(n_hidden[:hi+1]),i:i+nn_batch] = hidden
res1[:,i:i+nn_batch] = _z_confab['mean']
res2[:,i:i+nn_batch] = _z_confab['hidden'][-1]
res3[:,i:i+nn_batch] = _z_confab['logvar']
#print '--'
return res, res1, res2, res3
#print '..', n_batch
#if not dataset == 'mnist_binarized':
if not dataset == 'svhn':
z_test, z_test1, z_test2, vv_test = infer(x_test)
z_train, z_train1, z_train2, vv_train = infer(x_train)
if ll_valid > ll_valid_stats[0]:
ll_valid_stats[0] = ll_valid
ll_valid_stats[1] = 0
ndict.savez(ndict.get_value(model.v), logdir+'v_best')
ndict.savez(ndict.get_value(model.w), logdir+'w_best')
#if not dataset == 'mnist_binarized':
if dataset == 'svhn':
pass
#np.save(logdir+'full_latent', ('z_test': z_test, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train))
#np.save(logdir+'last_latent', ('z_test': z_test2, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train2))
else:
sio.savemat(logdir+'full_latent.mat', {'z_test': z_test, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train})
sio.savemat(logdir+'mean_latent.mat', {'z_test': z_test1, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train1})
sio.savemat(logdir+'last_latent.mat', {'z_test': z_test2, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train2})
else:
ll_valid_stats[1] += 1
# Stop when not improving validation set performance in 100 iterations
if ll_valid_stats[1] > 1000:
print "Finished"
with open(logdir+'hook.txt', 'a') as f:
print >>f, "Finished"
exit()
print epoch, t, ll, ll_valid, ll_test, ll_valid_stats
with open(logdir+'hook.txt', 'a') as f:
print >>f, epoch, t, ll, ll_valid, ll_test, ll_valid_stats
'''
if dataset != 'svhn':
l_t, px_t, pz_t, qz_t = model.test(x_train, n_samples=1, n_batch=n_batch, byteToFloat=byteToFloat)
print 'Elogpx', px_t, 'Elogpz', pz_t, '-Elogqz', qz_t
#sigma_square = float(os.environ['sigma_square'])
print 'var', np.mean(np.exp(vv_train)), 'q', np.mean(np.abs(z_train1)), 'p', np.mean(np.abs(train_mean_prior)), 'd', np.mean(np.abs(z_train1-train_mean_prior))
with open(logdir+'hook.txt', 'a') as f:
print >>f, 'Elogpx', px_t, 'Elogpz', pz_t, '-Elogqz', qz_t
print >>f, 'var', np.mean(np.exp(vv_train)), 'q', np.mean(np.abs(z_train1)), 'p', np.mean(np.abs(train_mean_prior)), 'd', np.mean(np.abs(z_train1-train_mean_prior))
'''
# Graphics
if gfx and epoch%gfx_freq == 0:
#tail = '.png'
tail = '-'+str(epoch)+'.png'
v = {i: model.v[i].get_value() for i in model.v}
w = {i: model.w[i].get_value() for i in model.w}
if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset and 'zca' not in dataset:
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg)
image.save(logdir+'q_w0'+tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg)
image.save(logdir+'out_w'+tail, 'PNG')
if 'out_unif' in w:
image = paramgraphics.mat_to_img(f_dec(w['out_unif'].reshape((-1,1))), dim_input, True, colorImg=colorImg)
image.save(logdir+'out_unif'+tail, 'PNG')
if n_z == 2:
n_width = 10
import scipy.stats
z = {'z':np.zeros((2,n_width**2))}
for i in range(0,n_width):
for j in range(0,n_width):
z['z'][0,n_width*i+j] = scipy.stats.norm.ppf(float(i)/n_width+0.5/n_width)
z['z'][1,n_width*i+j] = scipy.stats.norm.ppf(float(j)/n_width+0.5/n_width)
x, _, _z = model.gen_xz({}, z, n_width**2)
if dataset == 'mnist':
x = 1 - _z['x']
image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input)
image.save(logdir+'2dmanifold'+tail, 'PNG')
else:
if 'norb' in dataset or dataset=='svhn':
nn_batch_nn = 64
else:
nn_batch_nn = 144
if not(os.environ.has_key('train_residual') and bool(int(os.environ['train_residual'])) == True) and (os.environ.has_key('prior') and bool(int(os.environ['prior'])) == True):
mp_in = np.random.randint(0,x_train['mean_prior'].shape[1],nn_batch_nn)
m_p = x_train['mean_prior'][:,mp_in]
s_s = 1
if os.environ.has_key('sigma_square'):
s_s = float(os.environ['sigma_square'])
x_samples = model.gen_xz_prior({}, {}, m_p, s_s, n_batch=nn_batch_nn)
x_samples = x_samples['x']
m_p1 = (np.ones((n_z, nn_batch_nn)).T * np.mean(x_train['mean_prior'], axis = 1)).T
x_samples1 = model.gen_xz_prior({}, {}, m_p1.astype(np.float32), s_s, n_batch=nn_batch_nn)
image = paramgraphics.mat_to_img(f_dec(x_samples1['x']), dim_input, colorImg=colorImg)
image.save(logdir+'mean_samples-prior'+tail, 'PNG')
x_samples11 = model.gen_xz_prior11({}, {}, m_p, s_s, n_batch=nn_batch_nn)
image = paramgraphics.mat_to_img(f_dec(x_samples11['x']), dim_input, colorImg=colorImg)
image.save(logdir+'prior-image'+tail, 'PNG')
else:
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=nn_batch_nn)
x_samples = _z_confab['x']
image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg)
image.save(logdir+'samples-prior'+tail, 'PNG')
#x_samples = _x['x']
#image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg)
#image.save(logdir+'samples2'+tail, 'PNG')
else:
# Model with preprocessing
if 'w0' in v:
tmp = f_dec(v['w0'][:].T)
#print dim_input
#print tmp.shape
if 'zca' in dataset or dataset=='svhn':
tmp = zca_dec(zca_mean, zca_winv, tmp)
image = paramgraphics.mat_to_img(tmp, dim_input, True, colorImg=colorImg)
image.save(logdir+'q_w0'+tail, 'PNG')
tmp = f_dec(w['out_w'][:])
if 'zca' in dataset:
tmp = zca_dec(zca_mean, zca_winv, tmp)
image = paramgraphics.mat_to_img(tmp, dim_input, True, colorImg=colorImg)
image.save(logdir+'out_w'+tail, 'PNG')
if dataset == 'svhn':
nn_batch_nn = 64
else:
nn_batch_nn = 144
if not(os.environ.has_key('train_residual') and bool(int(os.environ['train_residual'])) == True) and (os.environ.has_key('prior') and bool(int(os.environ['prior'])) == True):
mp_in = np.random.randint(0,x_train['mean_prior'].shape[1],nn_batch_nn)
m_p = x_train['mean_prior'][:,mp_in]
s_s = 1
if os.environ.has_key('sigma_square'):
s_s = float(os.environ['sigma_square'])
x_samples = model.gen_xz_prior({}, {}, m_p, s_s, n_batch=nn_batch_nn)
x_samples = zca_dec(zca_mean, zca_winv,x_samples['x'])
x_samples = np.minimum(np.maximum(x_samples, 0), 1)
x_samples11 = model.gen_xz_prior11({}, {}, m_p, s_s, n_batch=nn_batch_nn)
x_samples11 = zca_dec(zca_mean,zca_winv,x_samples11['x'])
x_samples11 = np.minimum(np.maximum(x_samples11, 0), 1)
image = paramgraphics.mat_to_img(x_samples11, dim_input, colorImg=colorImg)
image.save(logdir+'prior-image'+tail, 'PNG')
else:
_x, _z, _z_confab = model.gen_xz({}, {}, n_batch=nn_batch_nn)
x_samples = f_dec(_z_confab['x'])
x_samples = np.minimum(np.maximum(x_samples, 0), 1)
image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg)
image.save(logdir+'samples'+tail, 'PNG')
'''
def hook(epoch, t, ll):
"""
Documented by SetaSouto, may contains errors.
:epoch: Number of the current step.
:t: Time elapsed from the beginning.
:ll: Loglikelihood.
"""
if epoch % 10 != 0: return
ll_valid, _ = model.est_loglik(x_valid,
n_samples=L_valid,
n_batch=n_batch,
byteToFloat=byteToFloat)
# Saves the value of our actual net.
ndict.savez(ndict.get_value(model.v), logdir + 'v')
ndict.savez(ndict.get_value(model.w), logdir + 'w')
# If the actual ll of the validset is the best:
if ll_valid > ll_valid_stats[0]:
ll_valid_stats[0] = ll_valid
# Reset the numbers of iterations without improving:
ll_valid_stats[1] = 0
ndict.savez(ndict.get_value(model.v), logdir + 'v_best')
ndict.savez(ndict.get_value(model.w), logdir + 'w_best')
else:
ll_valid_stats[1] += 1
# Stop when not improving validation set performance in 100 iterations
if ll_valid_stats[1] > 100:
print("Finished")
with open(logdir + 'hook.txt', 'a') as f:
print(f, "Finished")
exit()
# This will be showing the current results and write them in a file:
with open(logdir + 'AA_results.txt', 'a') as file:
# Like a csv file:
file.write(
str(epoch) + ',' + str(t) + ',' + str(ll) + ',' +
str(ll_valid) + ',' + str(ll_valid_stats[1]) + '\n')
file.close()
print("-------------------------")
print("Current results:")
print(" ")
print("Step:", epoch)
print("Time elapsed:", t)
print("Loglikelihood minibatch:", ll)
print("Loglikelihood validSet:", ll_valid)
print("N not improving:", ll_valid_stats[1])
# Graphics: Generate images from the math
if gfx and epoch % gfx_freq == 0:
# tail = '.png'
tail = '-' + str(epoch) + '.png'
v = {i: model.v[i].get_value() for i in model.v}
w = {i: model.w[i].get_value() for i in model.w}
if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset:
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'q_w0' + tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_w' + tail, 'PNG')
if 'out_unif' in w:
image = paramgraphics.mat_to_img(f_dec(
w['out_unif'].reshape((-1, 1))),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_unif' + tail, 'PNG')
if n_z == 2:
n_width = 10
import scipy.stats
z = {'z': np.zeros((2, n_width**2))}
for i in range(0, n_width):
for j in range(0, n_width):
z['z'][0, n_width * i + j] = scipy.stats.norm.ppf(
float(i) / n_width + 0.5 / n_width)
z['z'][1, n_width * i + j] = scipy.stats.norm.ppf(
float(j) / n_width + 0.5 / n_width)
x, _, _z = model.gen_xz({}, z, n_width**2)
if dataset == 'mnist':
x = 1 - _z['x']
image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input)
image.save(logdir + '2dmanifold' + tail, 'PNG')
else:
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = _z_confab['x']
image = paramgraphics.mat_to_img(f_dec(x_samples),
dim_input,
colorImg=colorImg)
image.save(logdir + 'samples' + tail, 'PNG')
# x_samples = _x['x']
# image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg)
# image.save(logdir+'samples2'+tail, 'PNG')
else:
# Model with preprocessing
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'q_w0' + tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_w' + tail, 'PNG')
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = f_dec(_z_confab['x'])
x_samples = np.minimum(np.maximum(x_samples, 0), 1)
image = paramgraphics.mat_to_img(x_samples,
dim_input,
colorImg=colorImg)
image.save(logdir + 'samples' + tail, 'PNG')
def hook(epoch, t, ll):
if epoch % 10 != 0: return
ll_valid, _ = model.est_loglik(x_valid,
n_samples=L_valid,
n_batch=n_batch,
byteToFloat=byteToFloat)
# Log
ndict.savez(ndict.get_value(model.v), logdir + 'v')
ndict.savez(ndict.get_value(model.w), logdir + 'w')
if ll_valid > ll_valid_stats[0]:
ll_valid_stats[0] = ll_valid
ll_valid_stats[1] = 0
ndict.savez(ndict.get_value(model.v), logdir + 'v_best')
ndict.savez(ndict.get_value(model.w), logdir + 'w_best')
else:
ll_valid_stats[1] += 1
# Stop when not improving validation set performance in 100 iterations
if ll_valid_stats[1] > 1000:
print "Finished"
with open(logdir + 'hook.txt', 'a') as f:
print >> f, "Finished"
exit()
print epoch, t, ll, ll_valid, ll_valid_stats
with open(logdir + 'hook.txt', 'a') as f:
print >> f, epoch, t, ll, ll_valid, ll_valid_stats
# Graphics
if gfx and epoch % gfx_freq == 0:
#tail = '.png'
tail = '-' + str(epoch) + '.png'
v = {i: model.v[i].get_value() for i in model.v}
w = {i: model.w[i].get_value() for i in model.w}
if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset:
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'q_w0' + tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_w' + tail, 'PNG')
if 'out_unif' in w:
image = paramgraphics.mat_to_img(f_dec(
w['out_unif'].reshape((-1, 1))),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_unif' + tail, 'PNG')
if n_z == 2:
n_width = 10
import scipy.stats
z = {'z': np.zeros((2, n_width**2))}
for i in range(0, n_width):
for j in range(0, n_width):
z['z'][0, n_width * i + j] = scipy.stats.norm.ppf(
float(i) / n_width + 0.5 / n_width)
z['z'][1, n_width * i + j] = scipy.stats.norm.ppf(
float(j) / n_width + 0.5 / n_width)
x, _, _z = model.gen_xz({}, z, n_width**2)
if dataset == 'mnist':
x = 1 - _z['x']
image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input)
image.save(logdir + '2dmanifold' + tail, 'PNG')
else:
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = _z_confab['x']
image = paramgraphics.mat_to_img(f_dec(x_samples),
dim_input,
colorImg=colorImg)
image.save(logdir + 'samples' + tail, 'PNG')
#x_samples = _x['x']
#image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg)
#image.save(logdir+'samples2'+tail, 'PNG')
else:
# Model with preprocessing
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'q_w0' + tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_w' + tail, 'PNG')
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = f_dec(_z_confab['x'])
x_samples = np.minimum(np.maximum(x_samples, 0), 1)
image = paramgraphics.mat_to_img(x_samples,
dim_input,
colorImg=colorImg)
image.save(logdir + 'samples' + tail, 'PNG')
print pertub_label.shape
pertub_label = (pertub_label.T * tmp_b).T
data_perturbed = pertub_label * data + (1 - pertub_label) * data_perturbed
if pertub_type == 4:
sio.savemat(
'data_imputation/type_' + str(pertub_type) + '_params_' +
str(int(pertub_prob * 100)) + '_noise_rawdata.mat', {
'z_train': x_train.T,
'z_test_original': data,
'z_test': data_perturbed,
'pertub_label': pertub_label
})
#print data_perturbed[:,:25].shape
image = paramgraphics.mat_to_img(data_perturbed[:, :25], (28, 28),
colorImg=False,
scale=True)
image.save('data_imputation/test_noise_4_' + str(pertub_prob) + '.png',
'PNG')
elif pertub_type == 3:
sio.savemat(
'data_imputation/type_' + str(pertub_type) + '_params_' +
str(pertub_prob) + '_noise_rawdata.mat', {
'z_train': x_train.T,
'z_test_original': data,
'z_test': data_perturbed,
'pertub_label': pertub_label
})
#print data_perturbed[:,:25].shape
image = paramgraphics.mat_to_img(data_perturbed[:, :25], (28, 28),
colorImg=False,
def hook(epoch, t, ll):
if epoch%10 != 0: return
ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat)
# Log
ndict.savez(ndict.get_value(model.v), logdir+'v')
ndict.savez(ndict.get_value(model.w), logdir+'w')
if ll_valid > ll_valid_stats[0]:
ll_valid_stats[0] = ll_valid
ll_valid_stats[1] = 0
ndict.savez(ndict.get_value(model.v), logdir+'v_best')
ndict.savez(ndict.get_value(model.w), logdir+'w_best')
else:
ll_valid_stats[1] += 1
# Stop when not improving validation set performance in 100 iterations
if ll_valid_stats[1] > 1000:
print "Finished"
with open(logdir+'hook.txt', 'a') as f:
print >>f, "Finished"
exit()
print epoch, t, ll, ll_valid, ll_valid_stats
with open(logdir+'hook.txt', 'a') as f:
print >>f, epoch, t, ll, ll_valid, ll_valid_stats
# Graphics
if gfx and epoch%gfx_freq == 0:
#tail = '.png'
tail = '-'+str(epoch)+'.png'
v = {i: model.v[i].get_value() for i in model.v}
w = {i: model.w[i].get_value() for i in model.w}
if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset:
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg)
image.save(logdir+'q_w0'+tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg)
image.save(logdir+'out_w'+tail, 'PNG')
if 'out_unif' in w:
image = paramgraphics.mat_to_img(f_dec(w['out_unif'].reshape((-1,1))), dim_input, True, colorImg=colorImg)
image.save(logdir+'out_unif'+tail, 'PNG')
if n_z == 2:
n_width = 10
import scipy.stats
z = {'z':np.zeros((2,n_width**2))}
for i in range(0,n_width):
for j in range(0,n_width):
z['z'][0,n_width*i+j] = scipy.stats.norm.ppf(float(i)/n_width+0.5/n_width)
z['z'][1,n_width*i+j] = scipy.stats.norm.ppf(float(j)/n_width+0.5/n_width)
x, _, _z = model.gen_xz({}, z, n_width**2)
if dataset == 'mnist':
x = 1 - _z['x']
image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input)
image.save(logdir+'2dmanifold'+tail, 'PNG')
else:
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = _z_confab['x']
image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg)
image.save(logdir+'samples'+tail, 'PNG')
#x_samples = _x['x']
#image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg)
#image.save(logdir+'samples2'+tail, 'PNG')
else:
# Model with preprocessing
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg)
image.save(logdir+'q_w0'+tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg)
image.save(logdir+'out_w'+tail, 'PNG')
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = f_dec(_z_confab['x'])
x_samples = np.minimum(np.maximum(x_samples, 0), 1)
image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg)
image.save(logdir+'samples'+tail, 'PNG')
# Set interactive mode
plt.ion()
for i in range(2000):
# Do step of Gaussian diffusion process
z = np.sqrt(1-noise_var)*z + np.sqrt(noise_var)*np.random.standard_normal(z.shape)
# Smooth the trajectory
zsmooth += smoothingfactor*(z-zsmooth)
if draw_rows: _z = np.repeat(zsmooth,n_y,axis=1)
else: _z = np.tile(zsmooth,n_y)
_, _, _z_confab = model.gen_xz({'y':y}, {'z':_z}, n_batch=n_y*n_batch_w)
x_samples = f_dec(_z_confab['x'])
if False:
image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg, tile_shape=tile_shape)
plt.clf()
plt.imshow(image, cmap=pylab.gray(), origin='upper')
plt.show()
plt.draw()
else:
if not os.path.exists(logdir): os.makedirs(logdir)
image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg, tile_shape=tile_shape)
# Make sure the nr of rows and cols are even
width, height = image.size
if width%2==1: width += 1
if height%2==1: height += 1
image = image.resize((width, height))
# Save it
fname = logdir+'/'+str(i)+'.png'
print 'Saving to '+fname
def hook(epoch, t, ll):
if epoch==-1:
ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat)
ll_test, _ = model.est_loglik(x_test, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat)
#print 'Likelihood of the pre-trained model: ', ll_valid, ll_test
if epoch%10 != 0: return
ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat)
ll_test = ll_valid
#if not dataset == 'mnist_binarized':
if not dataset == 'svhn':
ll_test, _ = model.est_loglik(x_test, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat)
# Log
ndict.savez(ndict.get_value(model.v), logdir+'v')
ndict.savez(ndict.get_value(model.w), logdir+'w')
if True:
ndict.get_value
if ll_valid > ll_valid_stats[0]:
ll_valid_stats[0] = ll_valid
ll_valid_stats[1] = 0
ll_valid_stats[2] = epoch
ndict.savez(ndict.get_value(model.v), logdir+'v_best')
ndict.savez(ndict.get_value(model.w), logdir+'w_best')
else:
ll_valid_stats[1] += 1
# Stop when not improving validation set performance in 100 iterations
if ll_valid_stats[1] > 1000:
print "Finished"
with open(logdir+'hook.txt', 'a') as f:
print >>f, "Finished"
exit()
# Graphics
if gfx and epoch%gfx_freq == 0:
#tail = '.png'
tail = '-'+str(epoch)+'.png'
v = {i: model.v[i].get_value() for i in model.v}
w = {i: model.w[i].get_value() for i in model.w}
if True:
def infer(data, n_batch=1000):
size = data['x'].shape[1]
#res = np.zeros((sum(n_hidden), size))
#res3 = np.zeros((n_z,size))
#res1 = np.zeros((n_z,size))
predy = []
for i in range(0, size, n_batch):
idx_to = min(size, i+n_batch)
x_batch = ndict.getCols(data, i, idx_to)
# may have bugs
nn_batch = idx_to - i
_x, _z, _z_confab = model.gen_xz(x_batch, {}, nn_batch)
x_samples = _z_confab['x']
predy += list(_z_confab['predy'])
#for (hi, hidden) in enumerate(_z_confab['hidden']):
# res[sum(n_hidden[:hi]):sum(n_hidden[:hi+1]),i:i+nn_batch] = hidden
#res3[:,i:i+nn_batch] = _z_confab['logvar']
#res1[:,i:i+nn_batch] = _z_confab['mean']
stats = dict()
#if epoch == -1:
# print 'features: ', res.shape
return predy #(res, predy, _z, res1, res3)
def evaluate(data, predy):
y = np.argmax(data['y'], axis=0)
return sum([int(yi != py) for (yi, py) in zip(y, predy)]) / float(len(predy))
#if not dataset == 'mnist_binarized':
#(z_test, pred_test,_z_test,z_test1,vv_test) = infer(x_test)
pred_test = infer(x_test)
pred_train = infer(x_train)
if not dataset == 'svhn':
n_used = 1000
if n_valid < 2 * n_used:
n_used = n_valid / 2
pred_valid = infer(x_valid,n_used)
#predy_valid = predy_test
#(z_train, pred_train, _z_train,z_train1,vv_train) = infer(x_train)
#l_t, px_t, pz_t, qz_t = model.test(x_train, n_samples=1, n_batch=n_batch, byteToFloat=byteToFloat)
'''
print 'Elogpx', px_t, 'Elogpz', pz_t, '-Elogqz', qz_t
#sigma_square = float(os.environ['sigma_square'])
print 'var', np.mean(np.exp(vv_train)), 'q', np.mean(np.abs(z_train1)), 'p', np.mean(np.abs(train_mean_prior)), 'd', np.mean(np.abs(z_train1-train_mean_prior))
'''
#with open(logdir+'hook.txt', 'a') as f:
# print >>f, 'Elogpx', px_t, 'Elogpz', pz_t, '-Elogqz', qz_t
# print >>f, 'var', np.mean(np.exp(vv_train)), 'q', np.mean(np.abs(z_train1)), 'p', np.mean(np.abs(train_mean_prior)), 'd', np.mean(np.abs(z_train1-train_mean_prior))
#
pre_train = evaluate(x_train, pred_train)
pre_test = evaluate(x_test, pred_test)
pre_valid = pre_test
if not dataset == 'svhn':
pre_valid = evaluate(x_valid, pred_valid)
if pre_valid < predy_valid_stats[0]:
predy_valid_stats[0] = pre_valid
predy_valid_stats[1] = pre_test
predy_valid_stats[2] = epoch
predy_valid_stats[3] = 0
ndict.savez(ndict.get_value(model.v), logdir+'v_best_predy')
ndict.savez(ndict.get_value(model.w), logdir+'w_best_predy')
else:
predy_valid_stats[3] += 1
# Stop when not improving validation set performance in 100 iterations
if predy_valid_stats[3] > 10000 and model.param_c.get_value() > 0:
print "Finished"
with open(logdir+'hook.txt', 'a') as f:
print >>f, "Finished"
exit()
if pre_test < predy_test_stats[1]:
predy_test_stats[0] = pre_valid
predy_test_stats[1] = pre_test
predy_test_stats[2] = epoch
#print 'c = ', model.param_c.get_value()
print 'epoch', epoch, 't', t, 'll', ll, 'll_valid', ll_valid, 'll_test', ll_test#, 'valid_stats', ll_valid_stats
print 'train_err = ', pre_train, 'valid_err = ', pre_valid, 'test_err = ', pre_test
print '--best: predy_valid_stats', predy_valid_stats#, 'predy_test_stats', predy_test_stats
with open(logdir+'hook.txt', 'a') as f:
print >>f, 'epoch', epoch, 't', t, 'll', ll, 'll_valid', ll_valid,'ll_test', ll_test, ll_valid_stats
print >>f, 'train_err = ', pre_train, 'valid_err = ', pre_valid, 'test_err = ', pre_test
print >>f, '--best: predy_valid_stats', predy_valid_stats#, 'predy_test_stats', predy_test_stats
#if not dataset == 'mnist_binarized':
#sio.savemat(logdir+'latent.mat', {'z_test': z_test, 'z_train': z_train})
if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset:
#print 'lallaaa'
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg)
image.save(logdir+'q_w0'+tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg)
image.save(logdir+'out_w'+tail, 'PNG')
if 'out_unif' in w:
image = paramgraphics.mat_to_img(f_dec(w['out_unif'].reshape((-1,1))), dim_input, True, colorImg=colorImg)
image.save(logdir+'out_unif'+tail, 'PNG')
if n_z == 2:
n_width = 10
import scipy.stats
z = {'z':np.zeros((2,n_width**2))}
for i in range(0,n_width):
for j in range(0,n_width):
z['z'][0,n_width*i+j] = scipy.stats.norm.ppf(float(i)/n_width+0.5/n_width)
z['z'][1,n_width*i+j] = scipy.stats.norm.ppf(float(j)/n_width+0.5/n_width)
x, _, _z = model.gen_xz({}, z, n_width**2)
if dataset == 'mnist':
x = 1 - _z['x']
image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input)
image.save(logdir+'2dmanifold'+tail, 'PNG')
else:
if 'norb' in dataset or dataset == 'svhn':
nn_batch_nn = 64
else:
nn_batch_nn = 144
if not(os.environ.has_key('train_residual') and bool(int(os.environ['train_residual'])) == True) and (os.environ.has_key('prior') and bool(int(os.environ['prior'])) == True):
mp_in = np.random.randint(0,x_train['mean_prior'].shape[1],nn_batch_nn)
m_p = x_train['mean_prior'][:,mp_in]
s_s = 1
if os.environ.has_key('sigma_square'):
s_s = float(os.environ['sigma_square'])
x_samples = model.gen_xz_prior({}, {}, m_p, s_s, n_batch=nn_batch_nn)
x_samples = x_samples['x']
m_p1 = (np.ones((n_z, nn_batch_nn)).T * np.mean(x_train['mean_prior'], axis = 1)).T
x_samples1 = model.gen_xz_prior({}, {}, m_p1.astype(np.float32), s_s, n_batch=nn_batch_nn)
image = paramgraphics.mat_to_img(f_dec(x_samples1['x']), dim_input, colorImg=colorImg)
image.save(logdir+'mean_samples-prior'+tail, 'PNG')
x_samples11 = model.gen_xz_prior11({}, {}, m_p, s_s, n_batch=nn_batch_nn)
image = paramgraphics.mat_to_img(f_dec(x_samples11['x']), dim_input, colorImg=colorImg)
image.save(logdir+'prior-image'+tail, 'PNG')
else:
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=nn_batch_nn)
x_samples = _z_confab['x']
image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg)
image.save(logdir+'samples-prior'+tail, 'PNG')
else:
# Model with preprocessing
if 'w0' in v:
tmp = f_dec(v['w0'][:].T)
if 'zca' in dataset:
tmp = zca_dec(zca_mean, zca_winv, tmp)
image = paramgraphics.mat_to_img(tmp, dim_input, True, colorImg=colorImg)
image.save(logdir+'q_w0'+tail, 'PNG')
tmp = f_dec(w['out_w'][:])
if 'zca' in dataset:
tmp = zca_dec(zca_mean, zca_winv, tmp)
image = paramgraphics.mat_to_img(tmp, dim_input, True, colorImg=colorImg)
image.save(logdir+'out_w'+tail, 'PNG')
if dataset == 'svhn':
nn_batch_nn = 64
else:
nn_batch_nn = 144
if not(os.environ.has_key('train_residual') and bool(int(os.environ['train_residual'])) == True) and (os.environ.has_key('prior') and bool(int(os.environ['prior'])) == True):
mp_in = np.random.randint(0,x_train['mean_prior'].shape[1],nn_batch_nn)
m_p = x_train['mean_prior'][:,mp_in]
s_s = 1
if os.environ.has_key('sigma_square'):
s_s = float(os.environ['sigma_square'])
x_samples = model.gen_xz_prior({}, {}, m_p, s_s, n_batch=nn_batch_nn)
x_samples = zca_dec(zca_mean, zca_winv,x_samples['x'])
x_samples = np.minimum(np.maximum(x_samples, 0), 1)
x_samples11 = model.gen_xz_prior11({}, {}, m_p, s_s, n_batch=nn_batch_nn)
x_samples11 = zca_dec(zca_mean,zca_winv,x_samples11['x'])
x_samples11 = np.minimum(np.maximum(x_samples11, 0), 1)
image = paramgraphics.mat_to_img(x_samples11, dim_input, colorImg=colorImg)
image.save(logdir+'prior-image'+tail, 'PNG')
else:
_x, _z, _z_confab = model.gen_xz({}, {}, n_batch=nn_batch_nn)
x_samples = f_dec(_z_confab['x'])
x_samples = np.minimum(np.maximum(x_samples, 0), 1)
image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg)
image.save(logdir+'samples'+tail, 'PNG')
