def test_generate_data():
x_len = 20
inputs, shifts, targets = ml.datasets.shift_gpu.generate_data(x_len, x_len, 10000)
inputs = gather(inputs)
shifts = gather(shifts)
targets = gather(targets)
assert verify_shift_dataset(inputs, shifts, targets), "data set verification failed"
def f_trn_dloss(p):
global trn_inputs, trn_shifts, trn_targets
if check_nans:
assert np.all(np.isfinite(gather(p))), "NaN in p given to f_trn_dloss"
dloss = f_dloss(p, trn_inputs, trn_shifts, trn_targets)
if check_nans:
if not np.all(np.isfinite(gather(dloss))):
print "NaN in calcuated gradient"
import pdb; pdb.set_trace()
return dloss
def plot_all_weights(ps):
plt.subplot(3,2,1)
plot_complex_weights(gather(ps['yhat_to_y_re']), gather(ps['yhat_to_y_im']))
plt.title('yhat_to_y')
plt.subplot(3,2,3)
plot_complex_weights(gather(ps['Xhat_to_Yhat_re']), gather(ps['Xhat_to_Yhat_im']))
plt.title('Xhat_to_Yhat')
plt.subplot(3,2,4)
plot_complex_weights(gather(ps['Shat_to_Yhat_re']), gather(ps['Shat_to_Yhat_im']))
plt.title('Shat_to_Yhat')
plt.subplot(3,2,5)
plot_complex_weights(gather(ps['x_to_xhat_re']), gather(ps['x_to_xhat_im']))
plt.title('x_to_xhat')
plt.subplot(3,2,6)
plot_complex_weights(gather(ps['s_to_shat_re']), gather(ps['s_to_shat_im']))
plt.title('s_to_shat')
plt.tight_layout()
def format_sample(smpl, width, height, threshold=0.5):
smpl = np.reshape(gather(smpl), (height, width))
if threshold is not None:
out = ""
for y in range(height):
for x in range(width):
if smpl[y, x] >= threshold:
out += "#"
else:
out += "."
out += "\n"
else:
out = str(smpl)
return out
def plot_all_weights(ps):
plt.subplot(3,1,1)
plot_complex_weights(gather(ps['yhat_to_y_re']), gather(ps['yhat_to_y_im']))
plt.title('yhat_to_y')
plt.subplot(3,1,2)
plot_complex_weights(gather(ps['Xhat_to_Yhat_re']), gather(ps['Xhat_to_Yhat_im']))
plt.title('Xhat_to_Yhat')
plt.subplot(3,1,3)
plot_complex_weights(gather(ps['x_to_xhat_re']), gather(ps['x_to_xhat_im']))
plt.title('x_to_xhat')
plt.tight_layout()
import os
import numpy as np
import ml.datasets.shift_gpu
import ml.apps.shiftnet.batchcfg as batchcfg
from ml.common.gpu import gather
try:
os.mkdir("testsets")
except:
pass
os.chdir("testsets")
for x_len in batchcfg.x_lens:
print "Generating x_len=%d" % x_len
inputs, shifts, targets = ml.datasets.shift_gpu.generate_data(x_len, x_len, batchcfg.test_samples)
np.savez_compressed("%d.npz" % x_len,
inputs=gather(inputs),
shifts=gather(shifts),
targets=gather(targets))
# i = np.random.randint(len(ps.data))
# ps.data[i] = 1
print "initial loss: ", f_loss(ps.data, trn_inputs, trn_shifts, trn_targets)
# optimize
his = ml.common.util.ParameterHistory(max_missed_val_improvements=2000,
desired_loss=0.0001)
#his = ml.common.util.ParameterHistory(max_missed_val_improvements=None,
# max_iters=20000,
# desired_loss=0.0001)
for iter, sts in enumerate(opt):
if iter % 10 == 0:
#print "learning rate is %f for iter %d" % (cfg.steprate[iter], iter)
opt.steprate = cfg.steprate[iter]
trn_loss = gather(f_loss(ps.data, trn_inputs, trn_shifts, trn_targets))
val_loss = gather(f_loss(ps.data, val_inputs, val_shifts, val_targets))
tst_loss = gather(f_loss(ps.data, tst_inputs, tst_shifts, tst_targets))
his.add(iter, ps.data, trn_loss, val_loss, tst_loss)
if his.should_terminate:
break
# save results
ps.data[:] = his.best_pars
his.plot()
plt.savefig(plot_dir + "/loss.pdf")
# check with simple patterns
sim_inputs, sim_shifts, sim_targets = generate_data(cfg.x_len, cfg.s_len, 3, binary=True)
sim_results = gather(f_output(ps.data, post(sim_inputs), post(sim_shifts)))
def f_trn_loss(p):
global trn_inputs, trn_shifts, trn_targets
if check_nans:
assert np.all(np.isfinite(gather(p))), "NaN in p given to f_trn_loss"
return f_loss(p, trn_inputs, trn_shifts, trn_targets)
# parameter and loss history
his = ml.common.util.ParameterHistory(max_missed_val_improvements=1000,
desired_loss=0.0001,
max_iters=cfg.max_iters,
min_iters=cfg.min_iters)
print "initial validation loss: ", f_loss(ps.data, val_inputs, val_shifts, val_targets)
# optimize
for sts in enumerate(opt):
explicit_cp = iteration % 10000 == 0
if cp_handler.requested or explicit_cp:
cp_handler.save(ps_data=gather(ps.data), opt_state=opt.state, iteration=iteration,
trn_inputs=gather(trn_inputs), trn_shifts=gather(trn_shifts), trn_targets=gather(trn_targets),
val_inputs=gather(val_inputs), val_shifts=gather(val_shifts), val_targets=gather(val_targets),
tst_inputs=gather(tst_inputs), tst_shifts=gather(tst_shifts), tst_targets=gather(tst_targets),
his=his,
explicit=explicit_cp)
if check_nans:
assert np.all(np.isfinite(gather(sts['step']))), 'NaN in step'
assert np.all(np.isfinite(gather(sts['moving_mean_squared']))), 'NaN in moving_mean_squared'
assert np.all(np.isfinite(gather(ps.data))), 'NaN in ps.data'
if iteration % cfg.new_data_iters == 0:
generate_new_data()
#if iter % 1000 == 0:
# transfer base parameters so that nets are equivalent
def doubling_matrix(n):
d = [[1, 1]]
nd = [d for _ in range(n)]
return block_diag(*nd)
def shift_doubling_matrix(n):
d = [[1, 0]]
nd = [d for _ in range(n)]
return block_diag(*nd)
def double_weights(w):
return 0.5 * np.dot(np.dot(doubling_matrix(w.shape[0]).T, w), doubling_matrix(w.shape[1]))
for wn in ps.views.iterkeys():
ps[wn][:] = post(double_weights(gather(bps[wn])))
# test:
#ps[wn][:] = post(gather(bps[wn]))
ps['s_to_shat_re'][:] = 2.0 * ps['s_to_shat_re'][:]
ps['s_to_shat_im'][:] = 2.0 * ps['s_to_shat_im'][:]
# data generation
def generate_data(n_samples):
if use_base_data:
inputs, shifts, targets = generate_base_data(n_samples)
else:
inputs, shifts, targets = generate_data(cfg.x_len, cfg.s_len, n_samples)
return inputs, shifts, targets
def generate_base_data(n_samples):
# Proposal: RBF on sigmoid layer
# <codecell>
# check kernel
x=post(np.array([[11, 21, 31], [12, 22, 32]]))
y=post(np.array([[101, 201], [102, 202]]))
l=post(np.array([[100]]))
tx = T.matrix('x')
ty = T.matrix('y')
tl = T.matrix('l')
f_kernel_matrix = function([tx, ty, tl], StackedRBF.kernel_matrix(tx, ty, tl))
K = f_kernel_matrix(x, y, l)
print gather(K)
# <codecell>
# hyperparameters
n_targets = RZ.get_value().shape[0]
n_features = RX.get_value().shape[0]
n_samples = RX.get_value().shape[1]
n_hidden = 50
#n_pivots = int(n_samples / 2)
n_pivots = 200
# <codecell>
# Theano expressions