def hook(epoch, t, ll): if epoch%10 != 0: return ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat) if math.isnan(ll_valid): print "NaN detected. Reverting to saved best parameters" ndict.set_value(model.v, ndict.loadz(logdir+'v.ndict.tar.gz')) ndict.set_value(model.w, ndict.loadz(logdir+'w.ndict.tar.gz')) return if ll_valid > ll_valid_stats[0]: ll_valid_stats[0] = ll_valid ll_valid_stats[1] = 0 ndict.savez(ndict.get_value(model.v), logdir+'v_best') ndict.savez(ndict.get_value(model.w), logdir+'w_best') else: ll_valid_stats[1] += 1 # Stop when not improving validation set performance in 100 iterations if False and ll_valid_stats[1] > 1000: print "Finished" with open(logdir+'hook.txt', 'a') as f: print >>f, "Finished" exit() # Log ndict.savez(ndict.get_value(model.v), logdir+'v') ndict.savez(ndict.get_value(model.w), logdir+'w') print epoch, t, ll, ll_valid with open(logdir+'hook.txt', 'a') as f: print >>f, t, ll, ll_valid if gfx: # Graphics v = {i: model.v[i].get_value() for i in model.v} w = {i: model.w[i].get_value() for i in model.w} tail = '-'+str(epoch)+'.png' image = paramgraphics.mat_to_img(f_dec(v['w0x'][:].T), dim_input, True, colorImg=colorImg) image.save(logdir+'q_w0x'+tail, 'PNG') image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg) image.save(logdir+'out_w'+tail, 'PNG') _x = {'y': np.random.multinomial(1, [1./n_y]*n_y, size=144).T} _, _, _z_confab = model.gen_xz(_x, {}, n_batch=144) image = paramgraphics.mat_to_img(f_dec(_z_confab['x']), dim_input, colorImg=colorImg) image.save(logdir+'samples'+tail, 'PNG') _, _, _z_confab = model.gen_xz(y_sample, z_sample, n_batch=144) image = paramgraphics.mat_to_img(f_dec(_z_confab['x']), dim_input, colorImg=colorImg) image.save(logdir+'samples_fixed'+tail, 'PNG') if n_z == 2: import ImageFont import ImageDraw n_width = 10 submosaic_offset = 15 submosaic_width = (dim_input[1]*n_width) submosaic_height = (dim_input[0]*n_width) mosaic = Image.new("RGB", (submosaic_width*mosaic_w, submosaic_offset+submosaic_height*mosaic_h)) for digit in range(0,n_y): if digit >= mosaic_h*mosaic_w: continue _x = {} n_batch_plot = n_width*n_width _x['y'] = np.zeros((n_y,n_batch_plot)) _x['y'][digit,:] = 1 _z = {'z':np.zeros((2,n_width**2))} for i in range(0,n_width): for j in range(0,n_width): _z['z'][0,n_width*i+j] = scipy.stats.norm.ppf(float(i)/n_width+0.5/n_width) _z['z'][1,n_width*i+j] = scipy.stats.norm.ppf(float(j)/n_width+0.5/n_width) _x, _, _z_confab = model.gen_xz(_x, _z, n_batch=n_batch_plot) x_samples = _z_confab['x'] image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg, tile_spacing=(0,0)) #image.save(logdir+'samples_digit_'+str(digit)+'_'+tail, 'PNG') mosaic_x = (digit%mosaic_w)*submosaic_width mosaic_y = submosaic_offset+int(digit/mosaic_w)*submosaic_height mosaic.paste(image, (mosaic_x, mosaic_y)) draw = ImageDraw.Draw(mosaic) draw.text((1,1),"Epoch #"+str(epoch)+" Loss="+str(int(ll))) #plt.savefig(logdir+'mosaic'+tail, format='PNG') mosaic.save(logdir+'mosaic'+tail, 'PNG')
def hook(epoch, t, ll): """ Documented by SetaSouto, may contains errors. :epoch: Number of the current step. :t: Time elapsed from the beginning. :ll: Loglikelihood. """ if epoch % 10 != 0: return ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat) # Saves the value of our actual net. ndict.savez(ndict.get_value(model.v), logdir + 'v') ndict.savez(ndict.get_value(model.w), logdir + 'w') # If the actual ll of the validset is the best: if ll_valid > ll_valid_stats[0]: ll_valid_stats[0] = ll_valid # Reset the numbers of iterations without improving: ll_valid_stats[1] = 0 ndict.savez(ndict.get_value(model.v), logdir + 'v_best') ndict.savez(ndict.get_value(model.w), logdir + 'w_best') else: ll_valid_stats[1] += 1 # Stop when not improving validation set performance in 100 iterations if ll_valid_stats[1] > 100: print("Finished") with open(logdir + 'hook.txt', 'a') as f: print(f, "Finished") exit() # This will be showing the current results and write them in a file: with open(logdir + 'AA_results.txt', 'a') as file: # Like a csv file: file.write( str(epoch) + ',' + str(t) + ',' + str(ll) + ',' + str(ll_valid) + ',' + str(ll_valid_stats[1]) + '\n') file.close() print("-------------------------") print("Current results:") print(" ") print("Step:", epoch) print("Time elapsed:", t) print("Loglikelihood minibatch:", ll) print("Loglikelihood validSet:", ll_valid) print("N not improving:", ll_valid_stats[1]) # Graphics: Generate images from the math if gfx and epoch % gfx_freq == 0: # tail = '.png' tail = '-' + str(epoch) + '.png' v = {i: model.v[i].get_value() for i in model.v} w = {i: model.w[i].get_value() for i in model.w} if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset: if 'w0' in v: image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg) image.save(logdir + 'q_w0' + tail, 'PNG') image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg) image.save(logdir + 'out_w' + tail, 'PNG') if 'out_unif' in w: image = paramgraphics.mat_to_img(f_dec( w['out_unif'].reshape((-1, 1))), dim_input, True, colorImg=colorImg) image.save(logdir + 'out_unif' + tail, 'PNG') if n_z == 2: n_width = 10 import scipy.stats z = {'z': np.zeros((2, n_width**2))} for i in range(0, n_width): for j in range(0, n_width): z['z'][0, n_width * i + j] = scipy.stats.norm.ppf( float(i) / n_width + 0.5 / n_width) z['z'][1, n_width * i + j] = scipy.stats.norm.ppf( float(j) / n_width + 0.5 / n_width) x, _, _z = model.gen_xz({}, z, n_width**2) if dataset == 'mnist': x = 1 - _z['x'] image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input) image.save(logdir + '2dmanifold' + tail, 'PNG') else: _x, _, _z_confab = model.gen_xz({}, {}, n_batch=144) x_samples = _z_confab['x'] image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg) image.save(logdir + 'samples' + tail, 'PNG') # x_samples = _x['x'] # image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg) # image.save(logdir+'samples2'+tail, 'PNG') else: # Model with preprocessing if 'w0' in v: image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg) image.save(logdir + 'q_w0' + tail, 'PNG') image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg) image.save(logdir + 'out_w' + tail, 'PNG') _x, _, _z_confab = model.gen_xz({}, {}, n_batch=144) x_samples = f_dec(_z_confab['x']) x_samples = np.minimum(np.maximum(x_samples, 0), 1) image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg) image.save(logdir + 'samples' + tail, 'PNG')
def hook(epoch, t, ll): if epoch%10 != 0: return n_batch_n = n_batch if n_batch_n > n_valid: n_batch_n = n_valid ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch_n, byteToFloat=byteToFloat) ll_test = ll_valid #if not dataset == 'mnist_binarized': if not dataset == 'svhn': ll_test, _ = model.est_loglik(x_test, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat) # Log ndict.savez(ndict.get_value(model.v), logdir+'v') ndict.savez(ndict.get_value(model.w), logdir+'w') def infer(data, n_batch=1000): #print '--', n_batch size = data['x'].shape[1] res = np.zeros((sum(n_hidden), size)) res1 = np.zeros((n_z,size)) res2 = np.zeros((n_hidden[-1],size)) res3 = np.zeros((n_z,size)) for i in range(0, size, n_batch): idx_to = min(size, i+n_batch) x_batch = ndict.getCols(data, i, idx_to) # may have bugs nn_batch = idx_to - i _x, _z, _z_confab = model.gen_xz(x_batch, {}, nn_batch) x_samples = _z_confab['x'] for (hi, hidden) in enumerate(_z_confab['hidden']): res[sum(n_hidden[:hi]):sum(n_hidden[:hi+1]),i:i+nn_batch] = hidden res1[:,i:i+nn_batch] = _z_confab['mean'] res2[:,i:i+nn_batch] = _z_confab['hidden'][-1] res3[:,i:i+nn_batch] = _z_confab['logvar'] #print '--' return res, res1, res2, res3 #print '..', n_batch #if not dataset == 'mnist_binarized': if not dataset == 'svhn': z_test, z_test1, z_test2, vv_test = infer(x_test) z_train, z_train1, z_train2, vv_train = infer(x_train) if ll_valid > ll_valid_stats[0]: ll_valid_stats[0] = ll_valid ll_valid_stats[1] = 0 ndict.savez(ndict.get_value(model.v), logdir+'v_best') ndict.savez(ndict.get_value(model.w), logdir+'w_best') #if not dataset == 'mnist_binarized': if dataset == 'svhn': pass #np.save(logdir+'full_latent', ('z_test': z_test, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train)) #np.save(logdir+'last_latent', ('z_test': z_test2, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train2)) else: sio.savemat(logdir+'full_latent.mat', {'z_test': z_test, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train}) sio.savemat(logdir+'mean_latent.mat', {'z_test': z_test1, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train1}) sio.savemat(logdir+'last_latent.mat', {'z_test': z_test2, 'train_y':train_y, 'test_y':test_y, 'z_train': z_train2}) else: ll_valid_stats[1] += 1 # Stop when not improving validation set performance in 100 iterations if ll_valid_stats[1] > 1000: print "Finished" with open(logdir+'hook.txt', 'a') as f: print >>f, "Finished" exit() print epoch, t, ll, ll_valid, ll_test, ll_valid_stats with open(logdir+'hook.txt', 'a') as f: print >>f, epoch, t, ll, ll_valid, ll_test, ll_valid_stats ''' if dataset != 'svhn': l_t, px_t, pz_t, qz_t = model.test(x_train, n_samples=1, n_batch=n_batch, byteToFloat=byteToFloat) print 'Elogpx', px_t, 'Elogpz', pz_t, '-Elogqz', qz_t #sigma_square = float(os.environ['sigma_square']) print 'var', np.mean(np.exp(vv_train)), 'q', np.mean(np.abs(z_train1)), 'p', np.mean(np.abs(train_mean_prior)), 'd', np.mean(np.abs(z_train1-train_mean_prior)) with open(logdir+'hook.txt', 'a') as f: print >>f, 'Elogpx', px_t, 'Elogpz', pz_t, '-Elogqz', qz_t print >>f, 'var', np.mean(np.exp(vv_train)), 'q', np.mean(np.abs(z_train1)), 'p', np.mean(np.abs(train_mean_prior)), 'd', np.mean(np.abs(z_train1-train_mean_prior)) ''' # Graphics if gfx and epoch%gfx_freq == 0: #tail = '.png' tail = '-'+str(epoch)+'.png' v = {i: model.v[i].get_value() for i in model.v} w = {i: model.w[i].get_value() for i in model.w} if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset and 'zca' not in dataset: if 'w0' in v: image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg) image.save(logdir+'q_w0'+tail, 'PNG') image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg) image.save(logdir+'out_w'+tail, 'PNG') if 'out_unif' in w: image = paramgraphics.mat_to_img(f_dec(w['out_unif'].reshape((-1,1))), dim_input, True, colorImg=colorImg) image.save(logdir+'out_unif'+tail, 'PNG') if n_z == 2: n_width = 10 import scipy.stats z = {'z':np.zeros((2,n_width**2))} for i in range(0,n_width): for j in range(0,n_width): z['z'][0,n_width*i+j] = scipy.stats.norm.ppf(float(i)/n_width+0.5/n_width) z['z'][1,n_width*i+j] = scipy.stats.norm.ppf(float(j)/n_width+0.5/n_width) x, _, _z = model.gen_xz({}, z, n_width**2) if dataset == 'mnist': x = 1 - _z['x'] image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input) image.save(logdir+'2dmanifold'+tail, 'PNG') else: if 'norb' in dataset or dataset=='svhn': nn_batch_nn = 64 else: nn_batch_nn = 144 if not(os.environ.has_key('train_residual') and bool(int(os.environ['train_residual'])) == True) and (os.environ.has_key('prior') and bool(int(os.environ['prior'])) == True): mp_in = np.random.randint(0,x_train['mean_prior'].shape[1],nn_batch_nn) m_p = x_train['mean_prior'][:,mp_in] s_s = 1 if os.environ.has_key('sigma_square'): s_s = float(os.environ['sigma_square']) x_samples = model.gen_xz_prior({}, {}, m_p, s_s, n_batch=nn_batch_nn) x_samples = x_samples['x'] m_p1 = (np.ones((n_z, nn_batch_nn)).T * np.mean(x_train['mean_prior'], axis = 1)).T x_samples1 = model.gen_xz_prior({}, {}, m_p1.astype(np.float32), s_s, n_batch=nn_batch_nn) image = paramgraphics.mat_to_img(f_dec(x_samples1['x']), dim_input, colorImg=colorImg) image.save(logdir+'mean_samples-prior'+tail, 'PNG') x_samples11 = model.gen_xz_prior11({}, {}, m_p, s_s, n_batch=nn_batch_nn) image = paramgraphics.mat_to_img(f_dec(x_samples11['x']), dim_input, colorImg=colorImg) image.save(logdir+'prior-image'+tail, 'PNG') else: _x, _, _z_confab = model.gen_xz({}, {}, n_batch=nn_batch_nn) x_samples = _z_confab['x'] image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg) image.save(logdir+'samples-prior'+tail, 'PNG') #x_samples = _x['x'] #image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg) #image.save(logdir+'samples2'+tail, 'PNG') else: # Model with preprocessing if 'w0' in v: tmp = f_dec(v['w0'][:].T) #print dim_input #print tmp.shape if 'zca' in dataset or dataset=='svhn': tmp = zca_dec(zca_mean, zca_winv, tmp) image = paramgraphics.mat_to_img(tmp, dim_input, True, colorImg=colorImg) image.save(logdir+'q_w0'+tail, 'PNG') tmp = f_dec(w['out_w'][:]) if 'zca' in dataset: tmp = zca_dec(zca_mean, zca_winv, tmp) image = paramgraphics.mat_to_img(tmp, dim_input, True, colorImg=colorImg) image.save(logdir+'out_w'+tail, 'PNG') if dataset == 'svhn': nn_batch_nn = 64 else: nn_batch_nn = 144 if not(os.environ.has_key('train_residual') and bool(int(os.environ['train_residual'])) == True) and (os.environ.has_key('prior') and bool(int(os.environ['prior'])) == True): mp_in = np.random.randint(0,x_train['mean_prior'].shape[1],nn_batch_nn) m_p = x_train['mean_prior'][:,mp_in] s_s = 1 if os.environ.has_key('sigma_square'): s_s = float(os.environ['sigma_square']) x_samples = model.gen_xz_prior({}, {}, m_p, s_s, n_batch=nn_batch_nn) x_samples = zca_dec(zca_mean, zca_winv,x_samples['x']) x_samples = np.minimum(np.maximum(x_samples, 0), 1) x_samples11 = model.gen_xz_prior11({}, {}, m_p, s_s, n_batch=nn_batch_nn) x_samples11 = zca_dec(zca_mean,zca_winv,x_samples11['x']) x_samples11 = np.minimum(np.maximum(x_samples11, 0), 1) image = paramgraphics.mat_to_img(x_samples11, dim_input, colorImg=colorImg) image.save(logdir+'prior-image'+tail, 'PNG') else: _x, _z, _z_confab = model.gen_xz({}, {}, n_batch=nn_batch_nn) x_samples = f_dec(_z_confab['x']) x_samples = np.minimum(np.maximum(x_samples, 0), 1) image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg) image.save(logdir+'samples'+tail, 'PNG') '''
def hook(epoch, t, ll): if epoch % 10 != 0: return ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat) # Log ndict.savez(ndict.get_value(model.v), logdir + 'v') ndict.savez(ndict.get_value(model.w), logdir + 'w') if ll_valid > ll_valid_stats[0]: ll_valid_stats[0] = ll_valid ll_valid_stats[1] = 0 ndict.savez(ndict.get_value(model.v), logdir + 'v_best') ndict.savez(ndict.get_value(model.w), logdir + 'w_best') else: ll_valid_stats[1] += 1 # Stop when not improving validation set performance in 100 iterations if ll_valid_stats[1] > 1000: print "Finished" with open(logdir + 'hook.txt', 'a') as f: print >> f, "Finished" exit() print epoch, t, ll, ll_valid, ll_valid_stats with open(logdir + 'hook.txt', 'a') as f: print >> f, epoch, t, ll, ll_valid, ll_valid_stats # Graphics if gfx and epoch % gfx_freq == 0: #tail = '.png' tail = '-' + str(epoch) + '.png' v = {i: model.v[i].get_value() for i in model.v} w = {i: model.w[i].get_value() for i in model.w} if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset: if 'w0' in v: image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg) image.save(logdir + 'q_w0' + tail, 'PNG') image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg) image.save(logdir + 'out_w' + tail, 'PNG') if 'out_unif' in w: image = paramgraphics.mat_to_img(f_dec( w['out_unif'].reshape((-1, 1))), dim_input, True, colorImg=colorImg) image.save(logdir + 'out_unif' + tail, 'PNG') if n_z == 2: n_width = 10 import scipy.stats z = {'z': np.zeros((2, n_width**2))} for i in range(0, n_width): for j in range(0, n_width): z['z'][0, n_width * i + j] = scipy.stats.norm.ppf( float(i) / n_width + 0.5 / n_width) z['z'][1, n_width * i + j] = scipy.stats.norm.ppf( float(j) / n_width + 0.5 / n_width) x, _, _z = model.gen_xz({}, z, n_width**2) if dataset == 'mnist': x = 1 - _z['x'] image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input) image.save(logdir + '2dmanifold' + tail, 'PNG') else: _x, _, _z_confab = model.gen_xz({}, {}, n_batch=144) x_samples = _z_confab['x'] image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg) image.save(logdir + 'samples' + tail, 'PNG') #x_samples = _x['x'] #image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg) #image.save(logdir+'samples2'+tail, 'PNG') else: # Model with preprocessing if 'w0' in v: image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg) image.save(logdir + 'q_w0' + tail, 'PNG') image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg) image.save(logdir + 'out_w' + tail, 'PNG') _x, _, _z_confab = model.gen_xz({}, {}, n_batch=144) x_samples = f_dec(_z_confab['x']) x_samples = np.minimum(np.maximum(x_samples, 0), 1) image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg) image.save(logdir + 'samples' + tail, 'PNG')
def hook(epoch, t, ll): if epoch%10 != 0: return ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat) # Log ndict.savez(ndict.get_value(model.v), logdir+'v') ndict.savez(ndict.get_value(model.w), logdir+'w') if ll_valid > ll_valid_stats[0]: ll_valid_stats[0] = ll_valid ll_valid_stats[1] = 0 ndict.savez(ndict.get_value(model.v), logdir+'v_best') ndict.savez(ndict.get_value(model.w), logdir+'w_best') else: ll_valid_stats[1] += 1 # Stop when not improving validation set performance in 100 iterations if ll_valid_stats[1] > 1000: print "Finished" with open(logdir+'hook.txt', 'a') as f: print >>f, "Finished" exit() print epoch, t, ll, ll_valid, ll_valid_stats with open(logdir+'hook.txt', 'a') as f: print >>f, epoch, t, ll, ll_valid, ll_valid_stats # Graphics if gfx and epoch%gfx_freq == 0: #tail = '.png' tail = '-'+str(epoch)+'.png' v = {i: model.v[i].get_value() for i in model.v} w = {i: model.w[i].get_value() for i in model.w} if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset: if 'w0' in v: image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg) image.save(logdir+'q_w0'+tail, 'PNG') image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg) image.save(logdir+'out_w'+tail, 'PNG') if 'out_unif' in w: image = paramgraphics.mat_to_img(f_dec(w['out_unif'].reshape((-1,1))), dim_input, True, colorImg=colorImg) image.save(logdir+'out_unif'+tail, 'PNG') if n_z == 2: n_width = 10 import scipy.stats z = {'z':np.zeros((2,n_width**2))} for i in range(0,n_width): for j in range(0,n_width): z['z'][0,n_width*i+j] = scipy.stats.norm.ppf(float(i)/n_width+0.5/n_width) z['z'][1,n_width*i+j] = scipy.stats.norm.ppf(float(j)/n_width+0.5/n_width) x, _, _z = model.gen_xz({}, z, n_width**2) if dataset == 'mnist': x = 1 - _z['x'] image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input) image.save(logdir+'2dmanifold'+tail, 'PNG') else: _x, _, _z_confab = model.gen_xz({}, {}, n_batch=144) x_samples = _z_confab['x'] image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg) image.save(logdir+'samples'+tail, 'PNG') #x_samples = _x['x'] #image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg) #image.save(logdir+'samples2'+tail, 'PNG') else: # Model with preprocessing if 'w0' in v: image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg) image.save(logdir+'q_w0'+tail, 'PNG') image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg) image.save(logdir+'out_w'+tail, 'PNG') _x, _, _z_confab = model.gen_xz({}, {}, n_batch=144) x_samples = f_dec(_z_confab['x']) x_samples = np.minimum(np.maximum(x_samples, 0), 1) image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg) image.save(logdir+'samples'+tail, 'PNG')
def hook(epoch, t, ll): if epoch==-1: ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat) ll_test, _ = model.est_loglik(x_test, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat) #print 'Likelihood of the pre-trained model: ', ll_valid, ll_test if epoch%10 != 0: return ll_valid, _ = model.est_loglik(x_valid, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat) ll_test = ll_valid #if not dataset == 'mnist_binarized': if not dataset == 'svhn': ll_test, _ = model.est_loglik(x_test, n_samples=L_valid, n_batch=n_batch, byteToFloat=byteToFloat) # Log ndict.savez(ndict.get_value(model.v), logdir+'v') ndict.savez(ndict.get_value(model.w), logdir+'w') if True: ndict.get_value if ll_valid > ll_valid_stats[0]: ll_valid_stats[0] = ll_valid ll_valid_stats[1] = 0 ll_valid_stats[2] = epoch ndict.savez(ndict.get_value(model.v), logdir+'v_best') ndict.savez(ndict.get_value(model.w), logdir+'w_best') else: ll_valid_stats[1] += 1 # Stop when not improving validation set performance in 100 iterations if ll_valid_stats[1] > 1000: print "Finished" with open(logdir+'hook.txt', 'a') as f: print >>f, "Finished" exit() # Graphics if gfx and epoch%gfx_freq == 0: #tail = '.png' tail = '-'+str(epoch)+'.png' v = {i: model.v[i].get_value() for i in model.v} w = {i: model.w[i].get_value() for i in model.w} if True: def infer(data, n_batch=1000): size = data['x'].shape[1] #res = np.zeros((sum(n_hidden), size)) #res3 = np.zeros((n_z,size)) #res1 = np.zeros((n_z,size)) predy = [] for i in range(0, size, n_batch): idx_to = min(size, i+n_batch) x_batch = ndict.getCols(data, i, idx_to) # may have bugs nn_batch = idx_to - i _x, _z, _z_confab = model.gen_xz(x_batch, {}, nn_batch) x_samples = _z_confab['x'] predy += list(_z_confab['predy']) #for (hi, hidden) in enumerate(_z_confab['hidden']): # res[sum(n_hidden[:hi]):sum(n_hidden[:hi+1]),i:i+nn_batch] = hidden #res3[:,i:i+nn_batch] = _z_confab['logvar'] #res1[:,i:i+nn_batch] = _z_confab['mean'] stats = dict() #if epoch == -1: # print 'features: ', res.shape return predy #(res, predy, _z, res1, res3) def evaluate(data, predy): y = np.argmax(data['y'], axis=0) return sum([int(yi != py) for (yi, py) in zip(y, predy)]) / float(len(predy)) #if not dataset == 'mnist_binarized': #(z_test, pred_test,_z_test,z_test1,vv_test) = infer(x_test) pred_test = infer(x_test) pred_train = infer(x_train) if not dataset == 'svhn': n_used = 1000 if n_valid < 2 * n_used: n_used = n_valid / 2 pred_valid = infer(x_valid,n_used) #predy_valid = predy_test #(z_train, pred_train, _z_train,z_train1,vv_train) = infer(x_train) #l_t, px_t, pz_t, qz_t = model.test(x_train, n_samples=1, n_batch=n_batch, byteToFloat=byteToFloat) ''' print 'Elogpx', px_t, 'Elogpz', pz_t, '-Elogqz', qz_t #sigma_square = float(os.environ['sigma_square']) print 'var', np.mean(np.exp(vv_train)), 'q', np.mean(np.abs(z_train1)), 'p', np.mean(np.abs(train_mean_prior)), 'd', np.mean(np.abs(z_train1-train_mean_prior)) ''' #with open(logdir+'hook.txt', 'a') as f: # print >>f, 'Elogpx', px_t, 'Elogpz', pz_t, '-Elogqz', qz_t # print >>f, 'var', np.mean(np.exp(vv_train)), 'q', np.mean(np.abs(z_train1)), 'p', np.mean(np.abs(train_mean_prior)), 'd', np.mean(np.abs(z_train1-train_mean_prior)) # pre_train = evaluate(x_train, pred_train) pre_test = evaluate(x_test, pred_test) pre_valid = pre_test if not dataset == 'svhn': pre_valid = evaluate(x_valid, pred_valid) if pre_valid < predy_valid_stats[0]: predy_valid_stats[0] = pre_valid predy_valid_stats[1] = pre_test predy_valid_stats[2] = epoch predy_valid_stats[3] = 0 ndict.savez(ndict.get_value(model.v), logdir+'v_best_predy') ndict.savez(ndict.get_value(model.w), logdir+'w_best_predy') else: predy_valid_stats[3] += 1 # Stop when not improving validation set performance in 100 iterations if predy_valid_stats[3] > 10000 and model.param_c.get_value() > 0: print "Finished" with open(logdir+'hook.txt', 'a') as f: print >>f, "Finished" exit() if pre_test < predy_test_stats[1]: predy_test_stats[0] = pre_valid predy_test_stats[1] = pre_test predy_test_stats[2] = epoch #print 'c = ', model.param_c.get_value() print 'epoch', epoch, 't', t, 'll', ll, 'll_valid', ll_valid, 'll_test', ll_test#, 'valid_stats', ll_valid_stats print 'train_err = ', pre_train, 'valid_err = ', pre_valid, 'test_err = ', pre_test print '--best: predy_valid_stats', predy_valid_stats#, 'predy_test_stats', predy_test_stats with open(logdir+'hook.txt', 'a') as f: print >>f, 'epoch', epoch, 't', t, 'll', ll, 'll_valid', ll_valid,'ll_test', ll_test, ll_valid_stats print >>f, 'train_err = ', pre_train, 'valid_err = ', pre_valid, 'test_err = ', pre_test print >>f, '--best: predy_valid_stats', predy_valid_stats#, 'predy_test_stats', predy_test_stats #if not dataset == 'mnist_binarized': #sio.savemat(logdir+'latent.mat', {'z_test': z_test, 'z_train': z_train}) if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset: #print 'lallaaa' if 'w0' in v: image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T), dim_input, True, colorImg=colorImg) image.save(logdir+'q_w0'+tail, 'PNG') image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]), dim_input, True, colorImg=colorImg) image.save(logdir+'out_w'+tail, 'PNG') if 'out_unif' in w: image = paramgraphics.mat_to_img(f_dec(w['out_unif'].reshape((-1,1))), dim_input, True, colorImg=colorImg) image.save(logdir+'out_unif'+tail, 'PNG') if n_z == 2: n_width = 10 import scipy.stats z = {'z':np.zeros((2,n_width**2))} for i in range(0,n_width): for j in range(0,n_width): z['z'][0,n_width*i+j] = scipy.stats.norm.ppf(float(i)/n_width+0.5/n_width) z['z'][1,n_width*i+j] = scipy.stats.norm.ppf(float(j)/n_width+0.5/n_width) x, _, _z = model.gen_xz({}, z, n_width**2) if dataset == 'mnist': x = 1 - _z['x'] image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input) image.save(logdir+'2dmanifold'+tail, 'PNG') else: if 'norb' in dataset or dataset == 'svhn': nn_batch_nn = 64 else: nn_batch_nn = 144 if not(os.environ.has_key('train_residual') and bool(int(os.environ['train_residual'])) == True) and (os.environ.has_key('prior') and bool(int(os.environ['prior'])) == True): mp_in = np.random.randint(0,x_train['mean_prior'].shape[1],nn_batch_nn) m_p = x_train['mean_prior'][:,mp_in] s_s = 1 if os.environ.has_key('sigma_square'): s_s = float(os.environ['sigma_square']) x_samples = model.gen_xz_prior({}, {}, m_p, s_s, n_batch=nn_batch_nn) x_samples = x_samples['x'] m_p1 = (np.ones((n_z, nn_batch_nn)).T * np.mean(x_train['mean_prior'], axis = 1)).T x_samples1 = model.gen_xz_prior({}, {}, m_p1.astype(np.float32), s_s, n_batch=nn_batch_nn) image = paramgraphics.mat_to_img(f_dec(x_samples1['x']), dim_input, colorImg=colorImg) image.save(logdir+'mean_samples-prior'+tail, 'PNG') x_samples11 = model.gen_xz_prior11({}, {}, m_p, s_s, n_batch=nn_batch_nn) image = paramgraphics.mat_to_img(f_dec(x_samples11['x']), dim_input, colorImg=colorImg) image.save(logdir+'prior-image'+tail, 'PNG') else: _x, _, _z_confab = model.gen_xz({}, {}, n_batch=nn_batch_nn) x_samples = _z_confab['x'] image = paramgraphics.mat_to_img(f_dec(x_samples), dim_input, colorImg=colorImg) image.save(logdir+'samples-prior'+tail, 'PNG') else: # Model with preprocessing if 'w0' in v: tmp = f_dec(v['w0'][:].T) if 'zca' in dataset: tmp = zca_dec(zca_mean, zca_winv, tmp) image = paramgraphics.mat_to_img(tmp, dim_input, True, colorImg=colorImg) image.save(logdir+'q_w0'+tail, 'PNG') tmp = f_dec(w['out_w'][:]) if 'zca' in dataset: tmp = zca_dec(zca_mean, zca_winv, tmp) image = paramgraphics.mat_to_img(tmp, dim_input, True, colorImg=colorImg) image.save(logdir+'out_w'+tail, 'PNG') if dataset == 'svhn': nn_batch_nn = 64 else: nn_batch_nn = 144 if not(os.environ.has_key('train_residual') and bool(int(os.environ['train_residual'])) == True) and (os.environ.has_key('prior') and bool(int(os.environ['prior'])) == True): mp_in = np.random.randint(0,x_train['mean_prior'].shape[1],nn_batch_nn) m_p = x_train['mean_prior'][:,mp_in] s_s = 1 if os.environ.has_key('sigma_square'): s_s = float(os.environ['sigma_square']) x_samples = model.gen_xz_prior({}, {}, m_p, s_s, n_batch=nn_batch_nn) x_samples = zca_dec(zca_mean, zca_winv,x_samples['x']) x_samples = np.minimum(np.maximum(x_samples, 0), 1) x_samples11 = model.gen_xz_prior11({}, {}, m_p, s_s, n_batch=nn_batch_nn) x_samples11 = zca_dec(zca_mean,zca_winv,x_samples11['x']) x_samples11 = np.minimum(np.maximum(x_samples11, 0), 1) image = paramgraphics.mat_to_img(x_samples11, dim_input, colorImg=colorImg) image.save(logdir+'prior-image'+tail, 'PNG') else: _x, _z, _z_confab = model.gen_xz({}, {}, n_batch=nn_batch_nn) x_samples = f_dec(_z_confab['x']) x_samples = np.minimum(np.maximum(x_samples, 0), 1) image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg) image.save(logdir+'samples'+tail, 'PNG')