def train_model():
batch_size = 16
num_epochs = c.ch4_train_epochs
sz = c.fcn_img_size
version = 2
for i in xrange(5):
data = u.DataH5PyStreamer(os.path.join(c.data_intermediate,
'ch4_256.hdf5'),
batch_size=batch_size,
folds=(5, i))
input_var = T.tensor4('input')
label_var = T.tensor4('label')
net, output, output_det = m.build_fcn_segmenter(input_var,
(None, 1, sz, sz),
version=version)
params = nn.layers.get_all_params(net['output'], trainable=True)
lr = theano.shared(nn.utils.floatX(3e-3))
loss = du.sorenson_dice(output, label_var)
te_loss = du.sorenson_dice(output_det, label_var)
te_acc = nn.objectives.binary_accuracy(output_det, label_var).mean()
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([input_var, label_var],
loss,
updates=updates)
test_fn = theano.function([input_var, label_var], te_loss)
acc_fn = theano.function([input_var, label_var], te_acc)
pred_fn = theano.function([input_var], output_det)
hist = u.train_with_hdf5(data, num_epochs=num_epochs,
train_fn = train_fn, test_fn=test_fn,
max_per_epoch=-1, use_tqdm=False,
tr_transform=lambda x: du.segmenter_data_transform(x, rotate=(-180, 180)),
te_transform=lambda x: du.segmenter_data_transform(x, rotate=None),
last_layer = net['output'],
save_params_to=os.path.join(c.params_dir, 'ch4seg_v{}/test_ch4seg_f{}_v{}.npz'\
.format(version, i, version)))
def train_model():
batch_size = 16
num_epochs = c.ch4_train_epochs
sz = c.fcn_img_size
version=2
for i in xrange(5):
data = u.DataH5PyStreamer(os.path.join(c.data_intermediate, 'ch4_256.hdf5'),
batch_size=batch_size, folds=(5,i))
input_var = T.tensor4('input')
label_var = T.tensor4('label')
net, output, output_det = m.build_fcn_segmenter(input_var,
(None, 1, sz, sz), version=version)
params = nn.layers.get_all_params(net['output'], trainable=True)
lr = theano.shared(nn.utils.floatX(3e-3))
loss = du.sorenson_dice(output, label_var)
te_loss = du.sorenson_dice(output_det, label_var)
te_acc = nn.objectives.binary_accuracy(output_det, label_var).mean()
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([input_var, label_var], loss, updates=updates)
test_fn = theano.function([input_var, label_var], te_loss)
acc_fn = theano.function([input_var, label_var], te_acc)
pred_fn = theano.function([input_var], output_det)
hist = u.train_with_hdf5(data, num_epochs=num_epochs,
train_fn = train_fn, test_fn=test_fn,
max_per_epoch=-1, use_tqdm=False,
tr_transform=lambda x: du.segmenter_data_transform(x, rotate=(-180, 180)),
te_transform=lambda x: du.segmenter_data_transform(x, rotate=None),
last_layer = net['output'],
save_params_to=os.path.join(c.params_dir, 'ch4seg_v{}/test_ch4seg_f{}_v{}.npz'\
.format(version, i, version)))
def main(num_epochs=20):
print("Building model and compiling functions...")
input_var = T.tensor4('inputs')
fcae = build_fcae(input_var)
output = nn.layers.get_output(fcae['output'])
output_det = nn.layers.get_output(fcae['output'], deterministic=True)
loss = nn.objectives.binary_crossentropy(output, input_var).mean()
test_loss = nn.objectives.binary_crossentropy(output_det, input_var).mean()
# ADAM updates
params = nn.layers.get_all_params(fcae['output'], trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=1e-3)
train_fn = theano.function([input_var], loss, updates=updates)
val_fn = theano.function([input_var], test_loss)
ae_fn = theano.function([input_var], nn.layers.get_output(fcae['output']))
data = u.DataH5PyStreamer(os.path.join(c.external_data, 'mnist.hdf5'),
batch_size=128)
hist = u.train_with_hdf5(
data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=val_fn,
max_per_epoch=40,
tr_transform=lambda x: u.raw_to_floatX(x[0], pixel_shift=0.),
te_transform=lambda x: u.raw_to_floatX(x[0], pixel_shift=0.))
u.save_params(fcae['output'],
'fcae_params_{}.npz'.format(np.asarray(hist)[-1, -1]))
from PIL import Image
from matplotlib import pyplot as plt
streamer = data.streamer()
imb = next(streamer.get_epoch_iterator())
batch = u.raw_to_floatX(imb[0], pixel_shift=0.).transpose((0, 1, 3, 2))
orig_dim = 28
im = Image.new("RGB", (orig_dim * 20, orig_dim * 20))
for j in xrange(10):
dim = orig_dim
orig_im = Image.fromarray(
u.get_picture_array(batch,
np.random.randint(batch.shape[0]),
shift=0.0))
im.paste(orig_im.resize((2 * orig_dim, 2 * orig_dim), Image.ANTIALIAS),
box=(0, j * orig_dim * 2))
new_im = {}
for i in xrange(9):
new_im = orig_im.resize((dim, dim), Image.ANTIALIAS)
new_im = ae_fn(
u.arr_from_img(new_im, shift=0.).reshape(1, -1, dim, dim))
new_im = Image.fromarray(u.get_picture_array(new_im, 0, shift=0.))\
.resize((orig_dim*2, orig_dim*2), Image.ANTIALIAS)
im.paste(new_im, box=((i + 1) * orig_dim * 2, j * orig_dim * 2))
dim = int(dim * 1.2)
im.save('increasing_size_autoencoded.jpg')
def main(num_epochs=10, seq_len=100, batch_size=2):
dtensor5 = T.TensorType('float32', (False, ) * 5)
input_var = dtensor5('input')
target_var = dtensor5('target')
net, output, output_det = rnn_orig(input_var, seq_len=seq_len)
params = nn.layers.get_all_params(net['output'])
pred = theano.function([input_var], output)
lr = theano.shared(nn.utils.floatX(1e-3))
loss = weighted_bce(output, target_var).mean()
loss_det = weighted_bce(output_det, target_var).mean()
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([input_var, target_var], loss, updates=updates)
test_fn = theano.function([input_var, target_var], loss_det)
data = u.DataH5PyStreamer('data/deeptrack.hdf5',
folds=(10, 0),
batch_size=batch_size * seq_len)
def transform_data(imb):
tgt = imb[0].reshape((-1, seq_len) + imb[0].shape[1:]).astype(
theano.config.floatX)
inp = np.copy(tgt)
for i in xrange(inp.shape[1]):
if i % 20 >= 10:
inp[:, i] = 0
return inp, tgt
if not os.path.exists('params'):
os.makedirs('params')
u.train_with_hdf5(
data,
num_epochs,
train_fn,
test_fn,
tr_transform=transform_data,
te_transform=transform_data,
train_shuffle=False,
max_per_epoch=-1,
use_tqdm=True,
#grad_clip=10,
last_layer=net['output'],
save_params_to='params/{}_params.npz'.format(
datetime.strftime(datetime.now(), "%Y%m%d%H%M%S")))
def main(
num_epochs=10,
seq_len = 100,
batch_size=2
):
dtensor5 = T.TensorType('float32', (False,)*5)
input_var = dtensor5('input')
target_var = dtensor5('target')
net,output, output_det = rnn_orig(input_var, seq_len=seq_len)
params = nn.layers.get_all_params(net['output'])
pred = theano.function([input_var], output)
lr = theano.shared(nn.utils.floatX(1e-3))
loss = weighted_bce(output, target_var).mean()
loss_det = weighted_bce(output_det, target_var).mean()
updates = nn.updates.adam(loss, params, learning_rate = lr)
train_fn = theano.function([input_var, target_var], loss, updates=updates)
test_fn = theano.function([input_var, target_var], loss_det)
data = u.DataH5PyStreamer('data/deeptrack.hdf5', folds=(10,0), batch_size=batch_size*seq_len)
def transform_data(imb):
tgt = imb[0].reshape((-1,seq_len) + imb[0].shape[1:]).astype(theano.config.floatX)
inp = np.copy(tgt)
for i in xrange(inp.shape[1]):
if i % 20 >= 10:
inp[:,i] = 0
return inp,tgt
if not os.path.exists('params'):
os.makedirs('params')
u.train_with_hdf5(data, num_epochs, train_fn, test_fn,
tr_transform = transform_data,
te_transform = transform_data,
train_shuffle = False,
max_per_epoch=-1,
use_tqdm=True,
#grad_clip=10,
last_layer = net['output'],
save_params_to='params/{}_params.npz'.format(datetime.strftime(datetime.now(), "%Y%m%d%H%M%S"))
)
def train_model():
batch_size = 8
version = 2
total_epochs = c.fcn_train_epochs
for normpct in [(10, 90), None]:
stop_times = []
for i in [0, 1, 2, 3, 4, -1]:
num_epochs = int(np.mean(stop_times)) if i == -1 else total_epochs
data = u.DataH5PyStreamer(os.path.join(c.data_intermediate,
'scd_seg_256.hdf5'),
batch_size=batch_size,
folds=(5, i))
input_var = T.tensor4('input')
label_var = T.tensor4('label')
net, output, output_det = m.build_fcn_segmenter(
input_var, (None, 1, c.fcn_img_size, c.fcn_img_size), version)
params = nn.layers.get_all_params(net['output'], trainable=True)
lr = theano.shared(nn.utils.floatX(3e-3))
loss = du.sorenson_dice(output, label_var)
te_loss = du.sorenson_dice(output_det, label_var)
te_acc = nn.objectives.binary_accuracy(output_det,
label_var).mean()
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([input_var, label_var],
loss,
updates=updates)
test_fn = theano.function([input_var, label_var], te_loss)
pred_fn = theano.function([input_var], output_det)
normstr = (str(normpct[0]) + str(normpct[1])) if normpct else 'MS'
pfn = os.path.join(
c.params_dir, 'fcn_v{}_p{}/fcn_v{}_p{}_f{}_{}.npz'.format(
version, normstr, version, normstr, i,
np.random.randint(100000)))
hist = u.train_with_hdf5(
data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=test_fn,
max_per_epoch=-1,
tr_transform=lambda x: du.segmenter_data_transform(
x, rotate=(-10, 50), normalize_pctwise=normpct),
te_transform=lambda x: du.segmenter_data_transform(
x, rotate=None, normalize_pctwise=normpct),
use_tqdm=False,
last_layer=net['output'],
save_last_params=(i == -1),
save_params_to=pfn)
if i != -1:
stop_times.append(np.argmin(np.array(hist)[:, 1]) + 1)
print 'stop time {}'.format(stop_times[-1])
def main(batch_size = 128, num_epochs = 30, learning_rate = 1e-4, d_ratio=4):
print("Building model and compiling functions...")
X = T.tensor4('inputs')
Z = T.matrix('Z')
ldict = build_nets(input_var = X)
def build_loss(deterministic):
# this currently has the problem that these 3 expressions come from 3 different
# get_output calls, so they won't return the same mask if dropout or other
# noise is used. Currently not using dropout so not a problem.
ae = nn.layers.get_output(ldict['ae_out'], deterministic=deterministic)
disc_real = nn.layers.get_output(ldict['disc_out'], deterministic=deterministic)
disc_fake = nn.layers.get_output(ldict['disc_out'], { ldict['disc_in']:ae },
deterministic=deterministic)
d_cost_real=nn.objectives.binary_crossentropy(disc_real, T.ones_like(disc_real)).mean()
d_cost_fake=nn.objectives.binary_crossentropy(disc_fake, T.zeros_like(disc_fake)).mean()
g_cost=nn.objectives.binary_crossentropy(disc_fake, T.ones_like(disc_fake)).mean()
d_cost = d_cost_real + d_cost_fake
mse = nn.objectives.squared_error(ae, X).mean()
return g_cost, d_cost, mse
g_cost, d_cost, _ = build_loss(deterministic=False)
g_cost_det, d_cost_det, mse = build_loss(deterministic=True)
d_lr = theano.shared(nn.utils.floatX(learning_rate))
d_params = nn.layers.get_all_params(ldict['disc_out'], trainable=True)
d_updates = nn.updates.adam(d_cost, d_params, learning_rate=d_lr)
g_lr = theano.shared(nn.utils.floatX(learning_rate))
g_params = nn.layers.get_all_params(ldict['ae_out'], trainable=True)
g_updates = nn.updates.adam(g_cost, g_params, learning_rate=g_lr)
_train_g = theano.function([X], g_cost, updates=g_updates)
_train_d = theano.function([X], d_cost, updates=d_updates)
_test_g = theano.function([X], g_cost_det)
_test_d = theano.function([X], d_cost_det)
mse = theano.function([X], mse)
print("Starting training...")
data = u.DataH5PyStreamer(os.path.join(c.external_data, 'mnist.hdf5'),
batch_size=batch_size)
def train_fn(x, d_ratio=d_ratio):
cost = 0
for i in xrange(d_ratio):
cost += _train_d(x)
cost += _train_g(x)
return cost
transform_data = lambda x: u.raw_to_floatX(x[0], pixel_shift=0.)
hist = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn = train_fn, test_fn = mse,
tr_transform=transform_data, te_transform=transform_data)
def main(specstr=default_specstr, z_dim=256, num_epochs=10, ch=3, init_from='',
img_size=64, pxsh=0.5, data_file='', batch_size=8, save_to='params'):
# build expressions for the output, loss, gradient
input_var = T.tensor4('inputs')
print('building specstr {} - zdim {}'.format(specstr, z_dim))
cae = m.build_cae_nopoolinv(input_var, shape=(img_size,img_size), channels=ch,
specstr=specstr.format(z_dim))
l_list = nn.layers.get_all_layers(cae)
pred = nn.layers.get_output(cae)
loss = nn.objectives.squared_error(pred, input_var.flatten(2)).mean()
params = nn.layers.get_all_params(cae, trainable=True)
grads = nn.updates.total_norm_constraint(T.grad(loss, params), 10)
updates = nn.updates.adam(grads, params, learning_rate=1e-3)
te_pred = nn.layers.get_output(cae, deterministic=True)
te_loss = nn.objectives.squared_error(te_pred, input_var.flatten(2)).mean()
# training functions
print('compiling functions')
train_fn = theano.function([input_var], loss, updates=updates)
val_fn = theano.function([input_var], te_loss)
# compile functions for encode/decode to test later
enc_layer = l_list[next(i for i in xrange(len(l_list)) if l_list[i].name=='encode')]
enc_fn = theano.function([input_var], nn.layers.get_output(enc_layer, deterministic=True))
dec_fn = lambda z: nn.layers.get_output(cae, deterministic=True,
inputs={l_list[0]:np.zeros((z.shape[0],ch,img_size,img_size),dtype=theano.config.floatX),
enc_layer:z}).eval().reshape(-1,ch,img_size,img_size)
# load params if requested, run training
if len(init_from) > 0:
print('loading params from {}'.format(init_from))
load_params(cae, init_from)
data = u.DataH5PyStreamer(data_file, batch_size=batch_size)
print('training for {} epochs'.format(num_epochs))
hist = u.train_with_hdf5(data, num_epochs=num_epochs,
train_fn = train_fn,
test_fn = val_fn,
tr_transform=lambda x: u.raw_to_floatX(x[0], pixel_shift=pxsh, center=False),
te_transform=lambda x: u.raw_to_floatX(x[0], pixel_shift=pxsh, center=True))
# generate examples, save training history
te_stream = data.streamer(shuffled=True)
imb, = next(te_stream.get_epoch_iterator())
tg = u.raw_to_floatX(imb, pixel_shift=pxsh, square=True, center=True)
pr = dec_fn(enc_fn(tg))
for i in range(pr.shape[0]):
u.get_image_pair(tg, pr,index=i,shift=pxsh).save('output_{}.jpg'.format(i))
hist = np.asarray(hist)
np.savetxt('cae_train_hist.csv', np.asarray(hist), delimiter=',', fmt='%.5f')
u.save_params(cae, os.path.join(save_to, 'cae_{}.npz'.format(hist[-1,-1])))
def main(num_epochs = 20):
print("Building model and compiling functions...")
input_var = T.tensor4('inputs')
fcae = build_fcae(input_var)
output = nn.layers.get_output(fcae['output'])
output_det = nn.layers.get_output(fcae['output'], deterministic=True)
loss = nn.objectives.binary_crossentropy(output, input_var).mean()
test_loss = nn.objectives.binary_crossentropy(output_det, input_var).mean()
# ADAM updates
params = nn.layers.get_all_params(fcae['output'], trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=1e-3)
train_fn = theano.function([input_var], loss, updates=updates)
val_fn = theano.function([input_var], test_loss)
ae_fn = theano.function([input_var], nn.layers.get_output(fcae['output']))
data = u.DataH5PyStreamer(os.path.join(c.external_data, 'mnist.hdf5'), batch_size=128)
hist = u.train_with_hdf5(data, num_epochs=num_epochs,
train_fn = train_fn, test_fn = val_fn,
max_per_epoch=40,
tr_transform = lambda x: u.raw_to_floatX(x[0], pixel_shift=0.),
te_transform = lambda x: u.raw_to_floatX(x[0], pixel_shift=0.))
u.save_params(fcae['output'], 'fcae_params_{}.npz'.format(np.asarray(hist)[-1,-1]))
from PIL import Image
from matplotlib import pyplot as plt
streamer = data.streamer()
imb = next(streamer.get_epoch_iterator())
batch = u.raw_to_floatX(imb[0], pixel_shift=0.).transpose((0,1,3,2))
orig_dim = 28
im = Image.new("RGB", (orig_dim*20, orig_dim*20))
for j in xrange(10):
dim = orig_dim
orig_im = Image.fromarray(u.get_picture_array(batch,
np.random.randint(batch.shape[0]), shift=0.0))
im.paste(orig_im.resize((2*orig_dim, 2*orig_dim), Image.ANTIALIAS),
box=(0,j*orig_dim*2))
new_im = {}
for i in xrange(9):
new_im = orig_im.resize((dim, dim), Image.ANTIALIAS)
new_im = ae_fn(u.arr_from_img(new_im, shift=0.).reshape(1,-1,dim,dim))
new_im = Image.fromarray(u.get_picture_array(new_im, 0, shift=0.))\
.resize((orig_dim*2, orig_dim*2), Image.ANTIALIAS)
im.paste(new_im, box=((i+1)*orig_dim*2, j*orig_dim*2))
dim = int(dim * 1.2)
im.save('increasing_size_autoencoded.jpg')
def main(L=2, img_size=64, pxsh=0., z_dim=32, n_hid=1024, num_epochs=12, binary='True',
init_from='', data_file='', batch_size=128, save_to='params', max_per_epoch=-1):
binary = binary.lower()=='true'
# Create VAE model
input_var = T.tensor4('inputs')
print("Building model and compiling functions...")
print("L = {}, z_dim = {}, n_hid = {}, binary={}".format(L, z_dim, n_hid, binary))
l_tup = l_z_mu, l_z_ls, l_x_mu_list, l_x_ls_list, l_x_list, l_x = \
m.build_vcae(input_var, L=L, binary=binary, z_dim=z_dim, n_hid=n_hid)
if len(init_from) > 0:
print('loading from {}'.format(init_from))
load_params(l_x, init_from)
# compile functions
loss, _ = u.build_vae_loss(input_var, *l_tup, deterministic=False, binary=binary, L=L)
test_loss, test_prediction = u.build_vae_loss(input_var, *l_tup, deterministic=True,
binary=binary, L=L)
params = nn.layers.get_all_params(l_x, trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=3e-5)
train_fn = theano.function([input_var], loss, updates=updates)
val_fn = theano.function([input_var], test_loss)
ae_fn = theano.function([input_var], test_prediction)
# run training loop
print('training for {} epochs'.format(num_epochs))
data = u.DataH5PyStreamer(data_file, batch_size=batch_size)
hist = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn=train_fn, test_fn=val_fn,
max_per_epoch=max_per_epoch,
tr_transform=lambda x: u.raw_to_floatX(x[0], pixel_shift=pxsh, center=False),
te_transform=lambda x: u.raw_to_floatX(x[0], pixel_shift=pxsh, center=True))
# generate examples, save training history
te_stream = data.streamer(shuffled=True)
imb, = next(te_stream.get_epoch_iterator())
orig_images = u.raw_to_floatX(imb, pixel_shift=pxsh)
autoencoded_images = ae_fn(orig_images)
for i in range(autoencoded_images.shape[0]):
u.get_image_pair(orig_images, autoencoded_images, index=i, shift=pxsh) \
.save('output_{}.jpg'.format(i))
hist = np.asarray(hist)
np.savetxt('vcae_train_hist.csv', np.asarray(hist), delimiter=',', fmt='%.5f')
u.save_params(l_x, os.path.join(save_to, 'vcae_{}.npz'.format(hist[-1,-1])))
def main(n_hid=256,
lstm_layers=2,
num_epochs=100,
batch_size=32,
save_to='output',
max_per_epoch=-1):
# load current set of words used
words = open(c.words_used_file, 'r').readlines()
idx_to_words = dict((i + 1, w.strip()) for i, w in enumerate(words))
idx_to_words[0] = '<e>'
word_dim = len(words) + 1
# normalization expected by vgg-net
mean_values = np.array([104, 117, 123]).reshape(
(3, 1, 1)).astype(theano.config.floatX)
# build function for extraction convolutional features
img_var = T.tensor4('images')
net = m.build_vgg(shape=(c.img_size, c.img_size), input_var=img_var)
values = pickle.load(open(c.vgg_weights))['param values']
nn.layers.set_all_param_values(net['pool5'], values)
conv_feats = theano.function([img_var], nn.layers.get_output(net['pool5']))
conv_shape = nn.layers.get_output_shape(net['pool5'])
# helper function for converting word vector to one-hot
raw_word_var = T.matrix('seq_raw')
one_hot = theano.function([raw_word_var],
nn.utils.one_hot(raw_word_var, m=word_dim))
# build expressions for lstm
conv_feats_var = T.tensor4('conv')
seq_var = T.tensor3('seq')
lstm = m.build_rnn(conv_feats_var, seq_var, conv_shape, word_dim, n_hid,
lstm_layers)
output = nn.layers.get_output(lstm['output'])
output_det = nn.layers.get_output(lstm['output'], deterministic=True)
loss = m.categorical_crossentropy_logdomain(output, seq_var).mean()
te_loss = m.categorical_crossentropy_logdomain(output_det, seq_var).mean()
# compile training functions
params = nn.layers.get_all_params(lstm['output'], trainable=True)
lr = theano.shared(nn.utils.floatX(1e-3))
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([conv_feats_var, seq_var],
loss,
updates=updates)
test_fn = theano.function([conv_feats_var, seq_var], te_loss)
predict_fn = theano.function([conv_feats_var, seq_var],
T.exp(output_det[:, -1:]))
zeros = np.zeros((batch_size, 1, word_dim), dtype=theano.config.floatX)
def transform_data(imb):
y, x = imb
# data augmentation: flip = -1 if we do flip over y-axis, 1 if not
flip = -2 * np.random.binomial(1, p=0.5) + 1
# this vgg-net expects image values that are normalized by mean but not magnitude
x = (u.raw_to_floatX(x[:,:,::flip], pixel_shift=0.)\
.transpose(0,1,3,2)[:,::-1] * 255. - mean_values)
return conv_feats(x), np.concatenate([zeros, one_hot(y)], axis=1)
data = u.DataH5PyStreamer(c.twimg_hdf5_file, batch_size=batch_size)
hist = u.train_with_hdf5(data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=test_fn,
max_per_epoch=max_per_epoch,
tr_transform=transform_data,
te_transform=transform_data)
np.savetxt('lstm_train_hist.csv',
np.asarray(hist),
delimiter=',',
fmt='%.5f')
u.save_params(
lstm['output'],
os.path.join(save_to, 'lstm_{}.npz'.format(np.asarray(hist)[-1, -1])))
# generate some example captions for one batch of images
streamer = data.streamer(training=False, shuffled=True)
y_raw, x_raw = next(streamer.get_epoch_iterator())
x, _ = transform_data((y_raw, x_raw))
y = zeros
captions = []
for idx in xrange(y.shape[0]):
captions.append([])
idx_to_words[0] = '<e>'
for sample_num in xrange(c.max_caption_len):
pred = predict_fn(x, y)
new_y = []
for idx in xrange(pred.shape[0]):
# reduce size by a small factor to prevent numerical imprecision from
# making it sum to > 1.
# reverse it so that <e> gets the additional probability, not a word
sample = np.random.multinomial(1,
pred[idx, 0, ::-1] * .999999)[::-1]
captions[idx].append(idx_to_words[np.argmax(sample)])
new_y.append(sample)
new_y = np.vstack(new_y).reshape(-1, 1,
word_dim).astype(theano.config.floatX)
y = np.concatenate([y, new_y], axis=1)
captions = [
'{},{}\n'.format(i, ' '.join(cap)) for i, cap in enumerate(captions)
]
with open(os.path.join(save_to, 'captions_sample.csv'), 'w') as wr:
wr.writelines(captions)
for idx in xrange(x_raw.shape[0]):
Image.fromarray(x_raw[idx].transpose(2, 1, 0)).save(
os.path.join(save_to, 'ex_{}.jpg'.format(idx)))
def main(data_file='',
img_size=64,
num_epochs=10,
batch_size=128,
pxsh=0.5,
split_layer='conv7',
specstr=c.pf_cae_specstr,
cae_params=c.pf_cae_params,
save_to='params'):
# transform function to go from images -> conv feats
conv_feats, _ = m.encoder_decoder(cae_params,
specstr=specstr,
layersplit=split_layer,
shape=(img_size, img_size))
# build pretrained net for images -> convfeats in order to get the input shape
# for the reverse function
print('compiling functions')
conv_net = m.build_cae(input_var=None,
specstr=specstr,
shape=(img_size, img_size))
cae_layer_dict = dict(
(l.name, l) for l in nn.layers.get_all_layers(conv_net))
shape = nn.layers.get_output_shape(cae_layer_dict[split_layer])
# build net for convfeats -> images
imgs_var = T.tensor4('images')
convs_var = T.tensor4('conv_features')
deconv_net = m.build_deconv_net(input_var=convs_var,
shape=shape,
specstr=specstr)
loss = nn.objectives.squared_error(
imgs_var, nn.layers.get_output(deconv_net)).mean()
te_loss = nn.objectives.squared_error(
imgs_var, nn.layers.get_output(deconv_net, deterministic=True)).mean()
params = nn.layers.get_all_params(deconv_net, trainable=True)
lr = theano.shared(nn.utils.floatX(3e-3))
updates = nn.updates.adam(loss, params, learning_rate=lr)
# compile functions
train_fn = theano.function([convs_var, imgs_var], loss, updates=updates)
val_fn = theano.function([convs_var, imgs_var], te_loss)
deconv_fn = theano.function([convs_var],
nn.layers.get_output(deconv_net,
deterministic=True))
# run training loop
print("training for {} epochs".format(num_epochs))
def data_transform(x, do_center):
floatx_ims = u.raw_to_floatX(x,
pixel_shift=pxsh,
square=True,
center=do_center)
return (conv_feats(floatx_ims), floatx_ims)
data = u.DataH5PyStreamer(data_file, batch_size=batch_size)
hist = u.train_with_hdf5(
data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=val_fn,
tr_transform=lambda x: data_transform(x[0], do_center=False),
te_transform=lambda x: data_transform(x[0], do_center=True))
# generate examples, save training history and params
te_stream = data.streamer(shuffled=True)
imb, = next(te_stream.get_epoch_iterator())
imb = data_transform(imb, True)[0]
result = deconv_fn(imb)
for i in range(result.shape[0]):
Image.fromarray(u.get_picture_array(result, index=i, shift=pxsh)) \
.save('output_{}.jpg'.format(i))
hist = np.asarray(hist)
np.savetxt('deconv_train_hist.csv',
np.asarray(hist),
delimiter=',',
fmt='%.5f')
u.save_params(deconv_net,
os.path.join(save_to, 'deconv_{}.npz'.format(hist[-1, -1])))
def main(specstr=default_specstr,
z_dim=256,
num_epochs=10,
ch=3,
init_from='',
img_size=64,
pxsh=0.5,
data_file='',
batch_size=8,
save_to='params'):
# build expressions for the output, loss, gradient
input_var = T.tensor4('inputs')
print('building specstr {} - zdim {}'.format(specstr, z_dim))
cae = m.build_cae_nopoolinv(input_var,
shape=(img_size, img_size),
channels=ch,
specstr=specstr.format(z_dim))
l_list = nn.layers.get_all_layers(cae)
pred = nn.layers.get_output(cae)
loss = nn.objectives.squared_error(pred, input_var.flatten(2)).mean()
params = nn.layers.get_all_params(cae, trainable=True)
grads = nn.updates.total_norm_constraint(T.grad(loss, params), 10)
updates = nn.updates.adam(grads, params, learning_rate=1e-3)
te_pred = nn.layers.get_output(cae, deterministic=True)
te_loss = nn.objectives.squared_error(te_pred, input_var.flatten(2)).mean()
# training functions
print('compiling functions')
train_fn = theano.function([input_var], loss, updates=updates)
val_fn = theano.function([input_var], te_loss)
# compile functions for encode/decode to test later
enc_layer = l_list[next(i for i in xrange(len(l_list))
if l_list[i].name == 'encode')]
enc_fn = theano.function([input_var],
nn.layers.get_output(enc_layer,
deterministic=True))
dec_fn = lambda z: nn.layers.get_output(
cae,
deterministic=True,
inputs={
l_list[0]:
np.zeros((z.shape[0], ch, img_size, img_size),
dtype=theano.config.floatX),
enc_layer:
z
}).eval().reshape(-1, ch, img_size, img_size)
# load params if requested, run training
if len(init_from) > 0:
print('loading params from {}'.format(init_from))
load_params(cae, init_from)
data = u.DataH5PyStreamer(data_file, batch_size=batch_size)
print('training for {} epochs'.format(num_epochs))
hist = u.train_with_hdf5(data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=val_fn,
tr_transform=lambda x: u.raw_to_floatX(
x[0], pixel_shift=pxsh, center=False),
te_transform=lambda x: u.raw_to_floatX(
x[0], pixel_shift=pxsh, center=True))
# generate examples, save training history
te_stream = data.streamer(shuffled=True)
imb, = next(te_stream.get_epoch_iterator())
tg = u.raw_to_floatX(imb, pixel_shift=pxsh, square=True, center=True)
pr = dec_fn(enc_fn(tg))
for i in range(pr.shape[0]):
u.get_image_pair(tg, pr, index=i,
shift=pxsh).save('output_{}.jpg'.format(i))
hist = np.asarray(hist)
np.savetxt('cae_train_hist.csv',
np.asarray(hist),
delimiter=',',
fmt='%.5f')
u.save_params(cae, os.path.join(save_to, 'cae_{}.npz'.format(hist[-1,
-1])))
if i == 5:
num_epochs = max_epoch + 1
print("full train data use {:d} epochs".format(num_epochs))
nn.layers.set_all_param_values(net['output'], init0)
data = u.DataH5PyStreamer(os.path.join(c.data_sunnybrook, datafile),
batch_size=batch_size,
folds=(5, i))
hist, best_epoch = u.train_with_hdf5(
data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=test_fn,
max_per_epoch=-1,
tr_transform=lambda x: du.segmenter_data_transform(
x,
shift=shi,
rotate=rot,
scale=sca,
normalize_pctwise=pct_norm_tr),
te_transform=lambda x: du.segmenter_data_transform(
x, normalize_pctwise=pct_norm, istest=True),
last_layer=net['output'],
save_best_params_to=c.params_dir +
'/fcn_v{}_{}_f{}.npz'.format(version, vvv, i))
if i < 5 and best_epoch > max_epoch:
max_epoch = best_epoch
if CV:
for pfn in [
'fcn_v{}_{}_f{}.npz'.format(version, vvv, i)
for i in xrange(ntimes - 1)
]: #!!!!CHANGE
def main(save_to='params',
dataset = 'mm',
kl_loss='true', # use kl-div in z-space instead of mse
diffs = 'false',
seq_length = 30,
num_epochs=1,
lstm_n_hid=1024,
max_per_epoch=-1
):
kl_loss = kl_loss.lower() == 'true'
diffs = diffs.lower() == 'true'
# set up functions for data pre-processing and model training
input_var = T.tensor4('inputs')
# different experimental setup for moving mnist vs pulp fiction dataests
if dataset == 'pf':
img_size = 64
cae_weights = c.pf_cae_params
cae_specstr = c.pf_cae_specstr
split_layer = 'conv7'
inpvar = T.tensor4('input')
net = m.build_cae(inpvar, specstr=cae_specstr, shape=(img_size, img_size))
convs_from_img,_ = m.encoder_decoder(cae_weights, specstr=cae_specstr,
layersplit=split_layer, shape=(img_size, img_size), poolinv=True)
laydict = dict((l.name, l) for l in nn.layers.get_all_layers(net))
zdec_in_shape = nn.layers.get_output_shape(laydict[split_layer])
deconv_weights = c.pf_deconv_params
vae_weights = c.pf_vae_params
img_from_convs = m.deconvoluter(deconv_weights, specstr=cae_specstr, shape=zdec_in_shape)
L=2
vae_n_hid = 1500
binary = False
z_dim = 256
l_tup = l_z_mu, l_z_ls, l_x_mu_list, l_x_ls_list, l_x_list, l_x = \
m.build_vae(input_var, L=L, binary=binary, z_dim=z_dim, n_hid=vae_n_hid,
shape=(zdec_in_shape[2], zdec_in_shape[3]), channels=zdec_in_shape[1])
u.load_params(l_x, vae_weights)
datafile = 'data/pf.hdf5'
frame_skip=3 # every 3rd frame in sequence
z_decode_layer = l_x_mu_list[0]
pixel_shift = 0.5
samples_per_image = 4
tr_batch_size = 16 # must be a multiple of samples_per_image
elif dataset == 'mm':
img_size = 64
cvae_weights = c.mm_cvae_params
L=2
vae_n_hid = 1024
binary = True
z_dim = 32
zdec_in_shape = (None, 1, img_size, img_size)
l_tup = l_z_mu, l_z_ls, l_x_mu_list, l_x_ls_list, l_x_list, l_x = \
m.build_vcae(input_var, L=L, z_dim=z_dim, n_hid=vae_n_hid, binary=binary,
shape=(zdec_in_shape[2], zdec_in_shape[3]), channels=zdec_in_shape[1])
u.load_params(l_x, cvae_weights)
datafile = 'data/moving_mnist.hdf5'
frame_skip=1
w,h=img_size,img_size # of raw input image in the hdf5 file
z_decode_layer = l_x_list[0]
pixel_shift = 0
samples_per_image = 1
tr_batch_size = 128 # must be a multiple of samples_per_image
# functions for moving to/from image or conv-space, and z-space
z_mat = T.matrix('z')
zenc = theano.function([input_var], nn.layers.get_output(l_z_mu, deterministic=True))
zdec = theano.function([z_mat], nn.layers.get_output(z_decode_layer, {l_z_mu:z_mat},
deterministic=True).reshape((-1, zdec_in_shape[1]) + zdec_in_shape[2:]))
zenc_ls = theano.function([input_var], nn.layers.get_output(l_z_ls, deterministic=True))
# functions for encoding sequences of z's
print 'compiling functions'
z_var = T.tensor3('z_in')
z_ls_var = T.tensor3('z_ls_in')
tgt_mu_var = T.tensor3('z_tgt')
tgt_ls_var = T.tensor3('z_ls_tgt')
learning_rate = theano.shared(nn.utils.floatX(1e-4))
# separate function definitions if we are using MSE and predicting only z, or KL divergence
# and predicting both mean and sigma of z
if kl_loss:
def kl(p_mu, p_sigma, q_mu, q_sigma):
return 0.5 * T.sum(T.sqr(p_sigma)/T.sqr(q_sigma) + T.sqr(q_mu - p_mu)/T.sqr(q_sigma)
- 1 + 2*T.log(q_sigma) - 2*T.log(p_sigma))
lstm, _ = m.Z_VLSTM(z_var, z_ls_var, z_dim=z_dim, nhid=lstm_n_hid, training=True)
z_mu_expr, z_ls_expr = nn.layers.get_output([lstm['output_mu'], lstm['output_ls']])
z_mu_expr_det, z_ls_expr_det = nn.layers.get_output([lstm['output_mu'],
lstm['output_ls']], deterministic=True)
loss = kl(tgt_mu_var, T.exp(tgt_ls_var), z_mu_expr, T.exp(z_ls_expr))
te_loss = kl(tgt_mu_var, T.exp(tgt_ls_var), z_mu_expr_det, T.exp(z_ls_expr_det))
params = nn.layers.get_all_params(lstm['output'], trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=learning_rate)
train_fn = theano.function([z_var, z_ls_var, tgt_mu_var, tgt_ls_var], loss,
updates=updates)
test_fn = theano.function([z_var, z_ls_var, tgt_mu_var, tgt_ls_var], te_loss)
else:
lstm, _ = m.Z_LSTM(z_var, z_dim=z_dim, nhid=lstm_n_hid, training=True)
loss = nn.objectives.squared_error(nn.layers.get_output(lstm['output']),
tgt_mu_var).mean()
te_loss = nn.objectives.squared_error(nn.layers.get_output(lstm['output'],
deterministic=True), tgt_mu_var).mean()
params = nn.layers.get_all_params(lstm['output'], trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=learning_rate)
train_fn = theano.function([z_var, tgt_mu_var], loss, updates=updates)
test_fn = theano.function([z_var, tgt_mu_var], te_loss)
if dataset == 'pf':
z_from_img = lambda x: zenc(convs_from_img(x))
z_ls_from_img = lambda x: zenc_ls(convs_from_img(x))
img_from_z = lambda z: img_from_convs(zdec(z))
elif dataset == 'mm':
z_from_img = zenc
z_ls_from_img = zenc_ls
img_from_z = zdec
# training loop
print('training for {} epochs'.format(num_epochs))
nbatch = (seq_length+1) * tr_batch_size * frame_skip / samples_per_image
data = u.DataH5PyStreamer(datafile, batch_size=nbatch)
# for taking arrays of uint8 (non square) and converting them to batches of sequences
def transform_data(ims_batch, center=False):
imb = u.raw_to_floatX(ims_batch, pixel_shift=pixel_shift,
center=center)[np.random.randint(frame_skip)::frame_skip]
zbatch = np.zeros((tr_batch_size, seq_length+1, z_dim), dtype=theano.config.floatX)
zsigbatch = np.zeros((tr_batch_size, seq_length+1, z_dim), dtype=theano.config.floatX)
for i in xrange(samples_per_image):
chunk = tr_batch_size/samples_per_image
if diffs:
zf = z_from_img(imb).reshape((chunk, seq_length+1, -1))
zbatch[i*chunk:(i+1)*chunk, 1:] = zf[:,1:] - zf[:,:-1]
if kl_loss:
zls = z_ls_from_img(imb).reshape((chunk, seq_length+1, -1))
zsigbatch[i*chunk:(i+1)*chunk, 1:] = zls[:,1:] - zls[:,:-1]
else:
zbatch[i*chunk:(i+1)*chunk] = z_from_img(imb).reshape((chunk, seq_length+1, -1))
if kl_loss:
zsigbatch[i*chunk:(i+1)*chunk] = z_ls_from_img(imb).reshape((chunk,
seq_length+1, -1))
if kl_loss:
return zbatch[:,:-1,:], zsigbatch[:,:-1,:], zbatch[:,1:,:], zsigbatch[:,1:,:]
return zbatch[:,:-1,:], zbatch[:,1:,:]
# we need sequences of images, so we do not shuffle data during trainin
hist = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn=train_fn, test_fn=test_fn,
train_shuffle=False,
max_per_epoch=max_per_epoch,
tr_transform=lambda x: transform_data(x[0], center=False),
te_transform=lambda x: transform_data(x[0], center=True))
hist = np.asarray(hist)
u.save_params(lstm['output'], os.path.join(save_to, 'lstm_{}.npz'.format(hist[-1,-1])))
# build functions to sample from LSTM
# separate cell_init and hid_init from the other learned model parameters
all_param_values = nn.layers.get_all_param_values(lstm['output'])
init_indices = [i for i,p in enumerate(nn.layers.get_all_params(lstm['output']))
if 'init' in str(p)]
init_values = [all_param_values[i] for i in init_indices]
params_noinit = [p for i,p in enumerate(all_param_values) if i not in init_indices]
# build model without learnable init values, and load non-init parameters
if kl_loss:
lstm_sample, state_vars = m.Z_VLSTM(z_var, z_ls_var, z_dim=z_dim, nhid=lstm_n_hid,
training=False)
else:
lstm_sample, state_vars = m.Z_LSTM(z_var, z_dim=z_dim, nhid=lstm_n_hid, training=False)
nn.layers.set_all_param_values(lstm_sample['output'], params_noinit)
# extract layers representing thee hidden and cell states, and have sample_fn
# return their outputs
state_layers_keys = [k for k in lstm_sample.keys() if 'hidfinal' in k or 'cellfinal' in k]
state_layers_keys = sorted(state_layers_keys)
state_layers_keys = sorted(state_layers_keys, key = lambda x:int(x.split('_')[1]))
state_layers = [lstm_sample[s] for s in state_layers_keys]
if kl_loss:
sample_fn = theano.function([z_var, z_ls_var] + state_vars,
nn.layers.get_output([lstm['output_mu'], lstm['output_ls']] + state_layers,
deterministic=True))
else:
sample_fn = theano.function([z_var] + state_vars,
nn.layers.get_output([lstm['output']] + state_layers, deterministic=True))
from images2gif import writeGif
from PIL import Image
# sample approximately 30 different generated video sequences
te_stream = data.streamer(training=True, shuffled=False)
interval = data.ntrain / data.batch_size / 30
for idx,imb in enumerate(te_stream.get_epoch_iterator()):
if idx % interval != 0:
continue
z_tup = transform_data(imb[0], center=True)
seg_idx = np.random.randint(z_tup[0].shape[0])
if kl_loss:
z_in, z_ls_in = z_tup[0], z_tup[1]
z_last, z_ls_last = z_in[seg_idx:seg_idx+1], z_ls_in[seg_idx:seg_idx+1]
z_vars = [z_last, z_ls_last]
else:
z_in = z_tup[0]
z_last = z_in[seg_idx:seg_idx+1]
z_vars = [z_last]
images = []
state_values = [np.dot(np.ones((z_last.shape[0],1), dtype=theano.config.floatX), s)
for s in init_values]
output_list = sample_fn(*(z_vars + state_values))
# use whole sequence of predictions for output
z_pred = output_list[0]
state_values = output_list[2 if kl_loss else 1:]
rec = img_from_z(z_pred.reshape(-1, z_dim))
for k in xrange(rec.shape[0]):
images.append(Image.fromarray(u.get_picture_array(rec, index=k, shift=pixel_shift)))
k += 1
# slice prediction to feed into lstm
z_pred = z_pred[:,-1:,:]
if kl_loss:
z_ls_pred = output_list[1][:,-1:,:]
z_vars = [z_pred, z_ls_pred]
else:
z_vars = [z_pred]
for i in xrange(30): # predict 30 frames after the end of the priming video
output_list = sample_fn(*(z_vars + state_values))
z_pred = output_list[0]
state_values = output_list[2 if kl_loss else 1:]
rec = img_from_z(z_pred.reshape(-1, z_dim))
images.append(Image.fromarray(u.get_picture_array(rec, index=0, shift=pixel_shift)))
if kl_loss:
z_ls_pred = output_list[1]
z_vars = [z_pred, z_ls_pred]
else:
z_vars = [z_pred]
writeGif("sample_{}.gif".format(idx),images,duration=0.1,dither=0)
def main(num_epochs=150, orig_filts=128, intermed_dim=64, batch_size=128,
p_drop_conv=0.5, p_drop_hid=0.5, learning_rate=0.001,
max_per_epoch=-1):
X = T.ftensor4()
Y = T.ivector()
hist = {}
accuracies = []
nparams = []
for convs_mult in [1,2,4]:
num_filts= orig_filts / convs_mult
norm_init = nn.init.Normal(std=0.01, mean=0.0)
w = theano.shared(norm_init.sample((num_filts, 3, 3, 3)))
w2 = theano.shared(norm_init.sample((num_filts, num_filts*convs_mult, 3, 3)))
w3 = theano.shared(norm_init.sample((num_filts, num_filts*convs_mult, 3, 3)))
w4 = theano.shared(norm_init.sample((intermed_dim, num_filts*convs_mult, 1, 1)))
w_o = theano.shared(norm_init.sample((intermed_dim*3*3, 10)))
py_x = model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hid, convs_mult=convs_mult)
py_x_det = model(X, w, w2, w3, w4, w_o, 0., 0., convs_mult=convs_mult)
y_x = T.argmax(py_x_det, axis=1)
cost = nn.objectives.categorical_crossentropy(py_x, Y).mean()
cost_det = nn.objectives.categorical_crossentropy(py_x_det, Y).mean()
acc = nn.objectives.categorical_accuracy(py_x_det, Y).mean()
params = [w, w2, w3, w4, w_o]
updates = nn.updates.adam(cost, params, learning_rate=learning_rate)
train_fn = theano.function([X, Y], outputs=cost, updates=updates,
allow_input_downcast=True)
test_fn = theano.function([X, Y], outputs=cost, allow_input_downcast=True)
predict_fn = theano.function([X], outputs=y_x, allow_input_downcast=True)
acc_fn = theano.function([X, Y], acc, allow_input_downcast=True)
hist[convs_mult] = np.asarray(u.train_with_hdf5(data, num_epochs=num_epochs,
train_fn=train_fn, test_fn=test_fn,
max_per_epoch=max_per_epoch,
tr_transform=transform_data, te_transform=transform_data))
streamer = data.streamer(training=False)
accur = 0
batches = 0
for tup in streamer.get_epoch_iterator():
x,y = transform_data(tup)
accur += acc_fn(x,y)
batches += 1
accuracies.append(accur/batches)
nparams.append(sum([p.get_value().size for p in params]))
np.savetxt('rotconv_hist_{}.csv'.format(convs_mult), hist[convs_mult], delimiter=',',
fmt='%.5f')
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
c1, = plt.plot(hist[1][:,1], label='orig')
c2, = plt.plot(hist[2][:,1], label='2x')
c4, = plt.plot(hist[4][:,1], label='4x')
plt.ylabel('Validation Loss')
plt.yscale('log')
plt.xlabel('Epoch')
plt.legend(handles=[c1, c2, c4], loc=1)
plt.savefig('validation_acc.jpg')
np.savetxt('accuracies.csv', np.asarray(accuracies), delimiter=',', fmt='%.5f')
np.savetxt('num_params.csv', np.asarray(nparams), delimiter=',', fmt='%d')
def main(data_file = '', num_epochs=10, batch_size = 128, L=2, z_dim=256,
n_hid=1500, binary='false', img_size = 64, init_from = '', save_to='params',
split_layer='conv7', pxsh = 0.5, specstr = c.pf_cae_specstr,
cae_weights=c.pf_cae_params, deconv_weights = c.pf_deconv_params):
binary = binary.lower() == 'true'
# pre-trained function for extracting convolutional features from images
cae = m.build_cae(input_var=None, specstr=specstr, shape=(img_size,img_size))
laydict = dict((l.name, l) for l in nn.layers.get_all_layers(cae))
convshape = nn.layers.get_output_shape(laydict[split_layer])
convs_from_img, _ = m.encoder_decoder(cae_weights, specstr=specstr, layersplit=split_layer,
shape=(img_size, img_size))
# pre-trained function for returning to images from convolutional features
img_from_convs = m.deconvoluter(deconv_weights, specstr=specstr, shape=convshape)
# Create VAE model
print("Building model and compiling functions...")
print("L = {}, z_dim = {}, n_hid = {}, binary={}".format(L, z_dim, n_hid, binary))
input_var = T.tensor4('inputs')
c,w,h = convshape[1], convshape[2], convshape[3]
l_tup = l_z_mu, l_z_ls, l_x_mu_list, l_x_ls_list, l_x_list, l_x = \
m.build_vae(input_var, L=L, binary=binary, z_dim=z_dim, n_hid=n_hid,
shape=(w,h), channels=c)
if len(init_from) > 0:
print("loading from {}".format(init_from))
u.load_params(l_x, init_from)
# build loss, updates, training, prediction functions
loss,_ = u.build_vae_loss(input_var, *l_tup, deterministic=False, binary=binary, L=L)
test_loss, test_prediction = u.build_vae_loss(input_var, *l_tup, deterministic=True,
binary=binary, L=L)
lr = theano.shared(nn.utils.floatX(1e-5))
params = nn.layers.get_all_params(l_x, trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([input_var], loss, updates=updates)
val_fn = theano.function([input_var], test_loss)
ae_fn = theano.function([input_var], test_prediction)
# run training loop
def data_transform(x, do_center):
floatx_ims = u.raw_to_floatX(x, pixel_shift=pxsh, square=True, center=do_center)
return convs_from_img(floatx_ims)
print("training for {} epochs".format(num_epochs))
data = u.DataH5PyStreamer(data_file, batch_size=batch_size)
hist = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn=train_fn, test_fn=val_fn,
tr_transform=lambda x: data_transform(x[0], do_center=False),
te_transform=lambda x: data_transform(x[0], do_center=True))
# generate examples, save training history
te_stream = data.streamer(shuffled=True)
imb, = next(te_stream.get_epoch_iterator())
orig_feats = data_transform(imb, do_center=True)
reconstructed_feats = ae_fn(orig_feats).reshape(orig_feats.shape)
orig_feats_deconv = img_from_convs(orig_feats)
reconstructed_feats_deconv = img_from_convs(reconstructed_feats)
for i in range(reconstructed_feats_deconv.shape[0]):
u.get_image_pair(orig_feats_deconv, reconstructed_feats_deconv, index=i, shift=pxsh)\
.save('output_{}.jpg'.format(i))
hist = np.asarray(hist)
np.savetxt('vae_convs_train_hist.csv', np.asarray(hist), delimiter=',', fmt='%.5f')
u.save_params(l_x, os.path.join(save_to, 'vae_convs_{}.npz'.format(hist[-1,-1])))
def main(L=2,
img_size=64,
pxsh=0.,
z_dim=32,
n_hid=1024,
num_epochs=12,
binary='True',
init_from='',
data_file='',
batch_size=128,
save_to='params',
max_per_epoch=-1):
binary = binary.lower() == 'true'
# Create VAE model
input_var = T.tensor4('inputs')
print("Building model and compiling functions...")
print("L = {}, z_dim = {}, n_hid = {}, binary={}".format(
L, z_dim, n_hid, binary))
l_tup = l_z_mu, l_z_ls, l_x_mu_list, l_x_ls_list, l_x_list, l_x = \
m.build_vcae(input_var, L=L, binary=binary, z_dim=z_dim, n_hid=n_hid)
if len(init_from) > 0:
print('loading from {}'.format(init_from))
load_params(l_x, init_from)
# compile functions
loss, _ = u.build_vae_loss(input_var,
*l_tup,
deterministic=False,
binary=binary,
L=L)
test_loss, test_prediction = u.build_vae_loss(input_var,
*l_tup,
deterministic=True,
binary=binary,
L=L)
params = nn.layers.get_all_params(l_x, trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=3e-5)
train_fn = theano.function([input_var], loss, updates=updates)
val_fn = theano.function([input_var], test_loss)
ae_fn = theano.function([input_var], test_prediction)
# run training loop
print('training for {} epochs'.format(num_epochs))
data = u.DataH5PyStreamer(data_file, batch_size=batch_size)
hist = u.train_with_hdf5(data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=val_fn,
max_per_epoch=max_per_epoch,
tr_transform=lambda x: u.raw_to_floatX(
x[0], pixel_shift=pxsh, center=False),
te_transform=lambda x: u.raw_to_floatX(
x[0], pixel_shift=pxsh, center=True))
# generate examples, save training history
te_stream = data.streamer(shuffled=True)
imb, = next(te_stream.get_epoch_iterator())
orig_images = u.raw_to_floatX(imb, pixel_shift=pxsh)
autoencoded_images = ae_fn(orig_images)
for i in range(autoencoded_images.shape[0]):
u.get_image_pair(orig_images, autoencoded_images, index=i, shift=pxsh) \
.save('output_{}.jpg'.format(i))
hist = np.asarray(hist)
np.savetxt('vcae_train_hist.csv',
np.asarray(hist),
delimiter=',',
fmt='%.5f')
u.save_params(l_x, os.path.join(save_to, 'vcae_{}.npz'.format(hist[-1,
-1])))
def main(
data_file="",
img_size=64,
num_epochs=10,
batch_size=128,
pxsh=0.5,
split_layer="conv7",
specstr=c.pf_cae_specstr,
cae_params=c.pf_cae_params,
save_to="params",
):
# transform function to go from images -> conv feats
conv_feats, _ = m.encoder_decoder(cae_params, specstr=specstr, layersplit=split_layer, shape=(img_size, img_size))
# build pretrained net for images -> convfeats in order to get the input shape
# for the reverse function
print("compiling functions")
conv_net = m.build_cae(input_var=None, specstr=specstr, shape=(img_size, img_size))
cae_layer_dict = dict((l.name, l) for l in nn.layers.get_all_layers(conv_net))
shape = nn.layers.get_output_shape(cae_layer_dict[split_layer])
# build net for convfeats -> images
imgs_var = T.tensor4("images")
convs_var = T.tensor4("conv_features")
deconv_net = m.build_deconv_net(input_var=convs_var, shape=shape, specstr=specstr)
loss = nn.objectives.squared_error(imgs_var, nn.layers.get_output(deconv_net)).mean()
te_loss = nn.objectives.squared_error(imgs_var, nn.layers.get_output(deconv_net, deterministic=True)).mean()
params = nn.layers.get_all_params(deconv_net, trainable=True)
lr = theano.shared(nn.utils.floatX(3e-3))
updates = nn.updates.adam(loss, params, learning_rate=lr)
# compile functions
train_fn = theano.function([convs_var, imgs_var], loss, updates=updates)
val_fn = theano.function([convs_var, imgs_var], te_loss)
deconv_fn = theano.function([convs_var], nn.layers.get_output(deconv_net, deterministic=True))
# run training loop
print("training for {} epochs".format(num_epochs))
def data_transform(x, do_center):
floatx_ims = u.raw_to_floatX(x, pixel_shift=pxsh, square=True, center=do_center)
return (conv_feats(floatx_ims), floatx_ims)
data = u.DataH5PyStreamer(data_file, batch_size=batch_size)
hist = u.train_with_hdf5(
data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=val_fn,
tr_transform=lambda x: data_transform(x[0], do_center=False),
te_transform=lambda x: data_transform(x[0], do_center=True),
)
# generate examples, save training history and params
te_stream = data.streamer(shuffled=True)
imb, = next(te_stream.get_epoch_iterator())
imb = data_transform(imb, True)[0]
result = deconv_fn(imb)
for i in range(result.shape[0]):
Image.fromarray(u.get_picture_array(result, index=i, shift=pxsh)).save("output_{}.jpg".format(i))
hist = np.asarray(hist)
np.savetxt("deconv_train_hist.csv", np.asarray(hist), delimiter=",", fmt="%.5f")
u.save_params(deconv_net, os.path.join(save_to, "deconv_{}.npz".format(hist[-1, -1])))
def main(n_hid=256, lstm_layers=2, num_epochs=100,
batch_size=32, save_to='output', max_per_epoch=-1):
# load current set of words used
words = open(c.words_used_file, 'r').readlines()
idx_to_words = dict((i+1,w.strip()) for i,w in enumerate(words))
idx_to_words[0] = '<e>'
word_dim=len(words)+1
# normalization expected by vgg-net
mean_values = np.array([104, 117, 123]).reshape((3,1,1)).astype(theano.config.floatX)
# build function for extraction convolutional features
img_var = T.tensor4('images')
net = m.build_vgg(shape=(c.img_size, c.img_size), input_var=img_var)
values = pickle.load(open(c.vgg_weights))['param values']
nn.layers.set_all_param_values(net['pool5'], values)
conv_feats = theano.function([img_var], nn.layers.get_output(net['pool5']))
conv_shape = nn.layers.get_output_shape(net['pool5'])
# helper function for converting word vector to one-hot
raw_word_var = T.matrix('seq_raw')
one_hot = theano.function([raw_word_var], nn.utils.one_hot(raw_word_var, m=word_dim))
# build expressions for lstm
conv_feats_var = T.tensor4('conv')
seq_var = T.tensor3('seq')
lstm = m.build_rnn(conv_feats_var, seq_var, conv_shape, word_dim, n_hid, lstm_layers)
output = nn.layers.get_output(lstm['output'])
output_det = nn.layers.get_output(lstm['output'], deterministic=True)
loss = m.categorical_crossentropy_logdomain(output, seq_var).mean()
te_loss = m.categorical_crossentropy_logdomain(output_det, seq_var).mean()
# compile training functions
params = nn.layers.get_all_params(lstm['output'], trainable=True)
lr = theano.shared(nn.utils.floatX(1e-3))
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([conv_feats_var, seq_var], loss, updates=updates)
test_fn = theano.function([conv_feats_var, seq_var], te_loss)
predict_fn = theano.function([conv_feats_var, seq_var], T.exp(output_det[:,-1:]))
zeros = np.zeros((batch_size, 1, word_dim), dtype=theano.config.floatX)
def transform_data(imb):
y,x = imb
# data augmentation: flip = -1 if we do flip over y-axis, 1 if not
flip = -2*np.random.binomial(1, p=0.5) + 1
# this vgg-net expects image values that are normalized by mean but not magnitude
x = (u.raw_to_floatX(x[:,:,::flip], pixel_shift=0.)\
.transpose(0,1,3,2)[:,::-1] * 255. - mean_values)
return conv_feats(x), np.concatenate([zeros, one_hot(y)], axis=1)
data = u.DataH5PyStreamer(c.twimg_hdf5_file, batch_size=batch_size)
hist = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn=train_fn, test_fn=test_fn,
max_per_epoch=max_per_epoch,
tr_transform=transform_data,
te_transform=transform_data)
np.savetxt('lstm_train_hist.csv', np.asarray(hist), delimiter=',', fmt='%.5f')
u.save_params(lstm['output'], os.path.join(save_to,
'lstm_{}.npz'.format(np.asarray(hist)[-1, -1])))
# generate some example captions for one batch of images
streamer = data.streamer(training=False, shuffled=True)
y_raw, x_raw = next(streamer.get_epoch_iterator())
x, _ = transform_data((y_raw, x_raw))
y = zeros
captions = []
for idx in xrange(y.shape[0]):
captions.append([])
idx_to_words[0] = '<e>'
for sample_num in xrange(c.max_caption_len):
pred = predict_fn(x, y)
new_y = []
for idx in xrange(pred.shape[0]):
# reduce size by a small factor to prevent numerical imprecision from
# making it sum to > 1.
# reverse it so that <e> gets the additional probability, not a word
sample = np.random.multinomial(1, pred[idx,0,::-1]*.999999)[::-1]
captions[idx].append(idx_to_words[np.argmax(sample)])
new_y.append(sample)
new_y = np.vstack(new_y).reshape(-1,1,word_dim).astype(theano.config.floatX)
y = np.concatenate([y, new_y], axis=1)
captions = ['{},{}\n'.format(i, ' '.join(cap)) for i,cap in enumerate(captions)]
with open(os.path.join(save_to, 'captions_sample.csv'), 'w') as wr:
wr.writelines(captions)
for idx in xrange(x_raw.shape[0]):
Image.fromarray(x_raw[idx].transpose(2,1,0)).save(os.path.join(save_to,
'ex_{}.jpg'.format(idx)))
def main(
save_to='params',
dataset='mm',
kl_loss='true', # use kl-div in z-space instead of mse
diffs='false',
seq_length=30,
num_epochs=1,
lstm_n_hid=1024,
max_per_epoch=-1):
kl_loss = kl_loss.lower() == 'true'
diffs = diffs.lower() == 'true'
# set up functions for data pre-processing and model training
input_var = T.tensor4('inputs')
# different experimental setup for moving mnist vs pulp fiction dataests
if dataset == 'pf':
img_size = 64
cae_weights = c.pf_cae_params
cae_specstr = c.pf_cae_specstr
split_layer = 'conv7'
inpvar = T.tensor4('input')
net = m.build_cae(inpvar,
specstr=cae_specstr,
shape=(img_size, img_size))
convs_from_img, _ = m.encoder_decoder(cae_weights,
specstr=cae_specstr,
layersplit=split_layer,
shape=(img_size, img_size),
poolinv=True)
laydict = dict((l.name, l) for l in nn.layers.get_all_layers(net))
zdec_in_shape = nn.layers.get_output_shape(laydict[split_layer])
deconv_weights = c.pf_deconv_params
vae_weights = c.pf_vae_params
img_from_convs = m.deconvoluter(deconv_weights,
specstr=cae_specstr,
shape=zdec_in_shape)
L = 2
vae_n_hid = 1500
binary = False
z_dim = 256
l_tup = l_z_mu, l_z_ls, l_x_mu_list, l_x_ls_list, l_x_list, l_x = \
m.build_vae(input_var, L=L, binary=binary, z_dim=z_dim, n_hid=vae_n_hid,
shape=(zdec_in_shape[2], zdec_in_shape[3]), channels=zdec_in_shape[1])
u.load_params(l_x, vae_weights)
datafile = 'data/pf.hdf5'
frame_skip = 3 # every 3rd frame in sequence
z_decode_layer = l_x_mu_list[0]
pixel_shift = 0.5
samples_per_image = 4
tr_batch_size = 16 # must be a multiple of samples_per_image
elif dataset == 'mm':
img_size = 64
cvae_weights = c.mm_cvae_params
L = 2
vae_n_hid = 1024
binary = True
z_dim = 32
zdec_in_shape = (None, 1, img_size, img_size)
l_tup = l_z_mu, l_z_ls, l_x_mu_list, l_x_ls_list, l_x_list, l_x = \
m.build_vcae(input_var, L=L, z_dim=z_dim, n_hid=vae_n_hid, binary=binary,
shape=(zdec_in_shape[2], zdec_in_shape[3]), channels=zdec_in_shape[1])
u.load_params(l_x, cvae_weights)
datafile = 'data/moving_mnist.hdf5'
frame_skip = 1
w, h = img_size, img_size # of raw input image in the hdf5 file
z_decode_layer = l_x_list[0]
pixel_shift = 0
samples_per_image = 1
tr_batch_size = 128 # must be a multiple of samples_per_image
# functions for moving to/from image or conv-space, and z-space
z_mat = T.matrix('z')
zenc = theano.function([input_var],
nn.layers.get_output(l_z_mu, deterministic=True))
zdec = theano.function(
[z_mat],
nn.layers.get_output(
z_decode_layer, {
l_z_mu: z_mat
}, deterministic=True).reshape((-1, zdec_in_shape[1]) +
zdec_in_shape[2:]))
zenc_ls = theano.function([input_var],
nn.layers.get_output(l_z_ls, deterministic=True))
# functions for encoding sequences of z's
print 'compiling functions'
z_var = T.tensor3('z_in')
z_ls_var = T.tensor3('z_ls_in')
tgt_mu_var = T.tensor3('z_tgt')
tgt_ls_var = T.tensor3('z_ls_tgt')
learning_rate = theano.shared(nn.utils.floatX(1e-4))
# separate function definitions if we are using MSE and predicting only z, or KL divergence
# and predicting both mean and sigma of z
if kl_loss:
def kl(p_mu, p_sigma, q_mu, q_sigma):
return 0.5 * T.sum(
T.sqr(p_sigma) / T.sqr(q_sigma) + T.sqr(q_mu - p_mu) /
T.sqr(q_sigma) - 1 + 2 * T.log(q_sigma) - 2 * T.log(p_sigma))
lstm, _ = m.Z_VLSTM(z_var,
z_ls_var,
z_dim=z_dim,
nhid=lstm_n_hid,
training=True)
z_mu_expr, z_ls_expr = nn.layers.get_output(
[lstm['output_mu'], lstm['output_ls']])
z_mu_expr_det, z_ls_expr_det = nn.layers.get_output(
[lstm['output_mu'], lstm['output_ls']], deterministic=True)
loss = kl(tgt_mu_var, T.exp(tgt_ls_var), z_mu_expr, T.exp(z_ls_expr))
te_loss = kl(tgt_mu_var, T.exp(tgt_ls_var), z_mu_expr_det,
T.exp(z_ls_expr_det))
params = nn.layers.get_all_params(lstm['output'], trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=learning_rate)
train_fn = theano.function([z_var, z_ls_var, tgt_mu_var, tgt_ls_var],
loss,
updates=updates)
test_fn = theano.function([z_var, z_ls_var, tgt_mu_var, tgt_ls_var],
te_loss)
else:
lstm, _ = m.Z_LSTM(z_var, z_dim=z_dim, nhid=lstm_n_hid, training=True)
loss = nn.objectives.squared_error(
nn.layers.get_output(lstm['output']), tgt_mu_var).mean()
te_loss = nn.objectives.squared_error(
nn.layers.get_output(lstm['output'], deterministic=True),
tgt_mu_var).mean()
params = nn.layers.get_all_params(lstm['output'], trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=learning_rate)
train_fn = theano.function([z_var, tgt_mu_var], loss, updates=updates)
test_fn = theano.function([z_var, tgt_mu_var], te_loss)
if dataset == 'pf':
z_from_img = lambda x: zenc(convs_from_img(x))
z_ls_from_img = lambda x: zenc_ls(convs_from_img(x))
img_from_z = lambda z: img_from_convs(zdec(z))
elif dataset == 'mm':
z_from_img = zenc
z_ls_from_img = zenc_ls
img_from_z = zdec
# training loop
print('training for {} epochs'.format(num_epochs))
nbatch = (seq_length + 1) * tr_batch_size * frame_skip / samples_per_image
data = u.DataH5PyStreamer(datafile, batch_size=nbatch)
# for taking arrays of uint8 (non square) and converting them to batches of sequences
def transform_data(ims_batch, center=False):
imb = u.raw_to_floatX(
ims_batch, pixel_shift=pixel_shift,
center=center)[np.random.randint(frame_skip)::frame_skip]
zbatch = np.zeros((tr_batch_size, seq_length + 1, z_dim),
dtype=theano.config.floatX)
zsigbatch = np.zeros((tr_batch_size, seq_length + 1, z_dim),
dtype=theano.config.floatX)
for i in xrange(samples_per_image):
chunk = tr_batch_size / samples_per_image
if diffs:
zf = z_from_img(imb).reshape((chunk, seq_length + 1, -1))
zbatch[i * chunk:(i + 1) * chunk, 1:] = zf[:, 1:] - zf[:, :-1]
if kl_loss:
zls = z_ls_from_img(imb).reshape(
(chunk, seq_length + 1, -1))
zsigbatch[i * chunk:(i + 1) * chunk,
1:] = zls[:, 1:] - zls[:, :-1]
else:
zbatch[i * chunk:(i + 1) * chunk] = z_from_img(imb).reshape(
(chunk, seq_length + 1, -1))
if kl_loss:
zsigbatch[i * chunk:(i + 1) *
chunk] = z_ls_from_img(imb).reshape(
(chunk, seq_length + 1, -1))
if kl_loss:
return zbatch[:, :
-1, :], zsigbatch[:, :
-1, :], zbatch[:,
1:, :], zsigbatch[:,
1:, :]
return zbatch[:, :-1, :], zbatch[:, 1:, :]
# we need sequences of images, so we do not shuffle data during trainin
hist = u.train_with_hdf5(
data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=test_fn,
train_shuffle=False,
max_per_epoch=max_per_epoch,
tr_transform=lambda x: transform_data(x[0], center=False),
te_transform=lambda x: transform_data(x[0], center=True))
hist = np.asarray(hist)
u.save_params(lstm['output'],
os.path.join(save_to, 'lstm_{}.npz'.format(hist[-1, -1])))
# build functions to sample from LSTM
# separate cell_init and hid_init from the other learned model parameters
all_param_values = nn.layers.get_all_param_values(lstm['output'])
init_indices = [
i for i, p in enumerate(nn.layers.get_all_params(lstm['output']))
if 'init' in str(p)
]
init_values = [all_param_values[i] for i in init_indices]
params_noinit = [
p for i, p in enumerate(all_param_values) if i not in init_indices
]
# build model without learnable init values, and load non-init parameters
if kl_loss:
lstm_sample, state_vars = m.Z_VLSTM(z_var,
z_ls_var,
z_dim=z_dim,
nhid=lstm_n_hid,
training=False)
else:
lstm_sample, state_vars = m.Z_LSTM(z_var,
z_dim=z_dim,
nhid=lstm_n_hid,
training=False)
nn.layers.set_all_param_values(lstm_sample['output'], params_noinit)
# extract layers representing thee hidden and cell states, and have sample_fn
# return their outputs
state_layers_keys = [
k for k in lstm_sample.keys() if 'hidfinal' in k or 'cellfinal' in k
]
state_layers_keys = sorted(state_layers_keys)
state_layers_keys = sorted(state_layers_keys,
key=lambda x: int(x.split('_')[1]))
state_layers = [lstm_sample[s] for s in state_layers_keys]
if kl_loss:
sample_fn = theano.function(
[z_var, z_ls_var] + state_vars,
nn.layers.get_output([lstm['output_mu'], lstm['output_ls']] +
state_layers,
deterministic=True))
else:
sample_fn = theano.function([z_var] + state_vars,
nn.layers.get_output([lstm['output']] +
state_layers,
deterministic=True))
from images2gif import writeGif
from PIL import Image
# sample approximately 30 different generated video sequences
te_stream = data.streamer(training=True, shuffled=False)
interval = data.ntrain / data.batch_size / 30
for idx, imb in enumerate(te_stream.get_epoch_iterator()):
if idx % interval != 0:
continue
z_tup = transform_data(imb[0], center=True)
seg_idx = np.random.randint(z_tup[0].shape[0])
if kl_loss:
z_in, z_ls_in = z_tup[0], z_tup[1]
z_last, z_ls_last = z_in[seg_idx:seg_idx +
1], z_ls_in[seg_idx:seg_idx + 1]
z_vars = [z_last, z_ls_last]
else:
z_in = z_tup[0]
z_last = z_in[seg_idx:seg_idx + 1]
z_vars = [z_last]
images = []
state_values = [
np.dot(np.ones((z_last.shape[0], 1), dtype=theano.config.floatX),
s) for s in init_values
]
output_list = sample_fn(*(z_vars + state_values))
# use whole sequence of predictions for output
z_pred = output_list[0]
state_values = output_list[2 if kl_loss else 1:]
rec = img_from_z(z_pred.reshape(-1, z_dim))
for k in xrange(rec.shape[0]):
images.append(
Image.fromarray(
u.get_picture_array(rec, index=k, shift=pixel_shift)))
k += 1
# slice prediction to feed into lstm
z_pred = z_pred[:, -1:, :]
if kl_loss:
z_ls_pred = output_list[1][:, -1:, :]
z_vars = [z_pred, z_ls_pred]
else:
z_vars = [z_pred]
for i in xrange(
30): # predict 30 frames after the end of the priming video
output_list = sample_fn(*(z_vars + state_values))
z_pred = output_list[0]
state_values = output_list[2 if kl_loss else 1:]
rec = img_from_z(z_pred.reshape(-1, z_dim))
images.append(
Image.fromarray(
u.get_picture_array(rec, index=0, shift=pixel_shift)))
if kl_loss:
z_ls_pred = output_list[1]
z_vars = [z_pred, z_ls_pred]
else:
z_vars = [z_pred]
writeGif("sample_{}.gif".format(idx), images, duration=0.1, dither=0)
def main(data_file='',
num_epochs=10,
batch_size=128,
L=2,
z_dim=256,
n_hid=1500,
binary='false',
img_size=64,
init_from='',
save_to='params',
split_layer='conv7',
pxsh=0.5,
specstr=c.pf_cae_specstr,
cae_weights=c.pf_cae_params,
deconv_weights=c.pf_deconv_params):
binary = binary.lower() == 'true'
# pre-trained function for extracting convolutional features from images
cae = m.build_cae(input_var=None,
specstr=specstr,
shape=(img_size, img_size))
laydict = dict((l.name, l) for l in nn.layers.get_all_layers(cae))
convshape = nn.layers.get_output_shape(laydict[split_layer])
convs_from_img, _ = m.encoder_decoder(cae_weights,
specstr=specstr,
layersplit=split_layer,
shape=(img_size, img_size))
# pre-trained function for returning to images from convolutional features
img_from_convs = m.deconvoluter(deconv_weights,
specstr=specstr,
shape=convshape)
# Create VAE model
print("Building model and compiling functions...")
print("L = {}, z_dim = {}, n_hid = {}, binary={}".format(
L, z_dim, n_hid, binary))
input_var = T.tensor4('inputs')
c, w, h = convshape[1], convshape[2], convshape[3]
l_tup = l_z_mu, l_z_ls, l_x_mu_list, l_x_ls_list, l_x_list, l_x = \
m.build_vae(input_var, L=L, binary=binary, z_dim=z_dim, n_hid=n_hid,
shape=(w,h), channels=c)
if len(init_from) > 0:
print("loading from {}".format(init_from))
u.load_params(l_x, init_from)
# build loss, updates, training, prediction functions
loss, _ = u.build_vae_loss(input_var,
*l_tup,
deterministic=False,
binary=binary,
L=L)
test_loss, test_prediction = u.build_vae_loss(input_var,
*l_tup,
deterministic=True,
binary=binary,
L=L)
lr = theano.shared(nn.utils.floatX(1e-5))
params = nn.layers.get_all_params(l_x, trainable=True)
updates = nn.updates.adam(loss, params, learning_rate=lr)
train_fn = theano.function([input_var], loss, updates=updates)
val_fn = theano.function([input_var], test_loss)
ae_fn = theano.function([input_var], test_prediction)
# run training loop
def data_transform(x, do_center):
floatx_ims = u.raw_to_floatX(x,
pixel_shift=pxsh,
square=True,
center=do_center)
return convs_from_img(floatx_ims)
print("training for {} epochs".format(num_epochs))
data = u.DataH5PyStreamer(data_file, batch_size=batch_size)
hist = u.train_with_hdf5(
data,
num_epochs=num_epochs,
train_fn=train_fn,
test_fn=val_fn,
tr_transform=lambda x: data_transform(x[0], do_center=False),
te_transform=lambda x: data_transform(x[0], do_center=True))
# generate examples, save training history
te_stream = data.streamer(shuffled=True)
imb, = next(te_stream.get_epoch_iterator())
orig_feats = data_transform(imb, do_center=True)
reconstructed_feats = ae_fn(orig_feats).reshape(orig_feats.shape)
orig_feats_deconv = img_from_convs(orig_feats)
reconstructed_feats_deconv = img_from_convs(reconstructed_feats)
for i in range(reconstructed_feats_deconv.shape[0]):
u.get_image_pair(orig_feats_deconv, reconstructed_feats_deconv, index=i, shift=pxsh)\
.save('output_{}.jpg'.format(i))
hist = np.asarray(hist)
np.savetxt('vae_convs_train_hist.csv',
np.asarray(hist),
delimiter=',',
fmt='%.5f')
u.save_params(
l_x, os.path.join(save_to, 'vae_convs_{}.npz'.format(hist[-1, -1])))
pred_fn = theano.function([input_var], output_det)
batch_size=16
max_epoch = (0 if CV else num_epochs);
for i in xrange(ntimes):
if not CV and i!=5:
continue
if i == 5:
num_epochs = max_epoch+1;
print("full train data use {:d} epochs".format(num_epochs))
nn.layers.set_all_param_values(net['output'],init0);
data = u.DataH5PyStreamer(os.path.join(c.data_sunnybrook, datafile), batch_size=batch_size, folds=(5,i))
hist,best_epoch = u.train_with_hdf5(data, num_epochs=num_epochs, train_fn = train_fn, test_fn=test_fn,
max_per_epoch=-1,
tr_transform=lambda x: du.segmenter_data_transform(x, shift=shi, rotate=rot, scale = sca, normalize_pctwise=pct_norm_tr),
te_transform=lambda x: du.segmenter_data_transform(x, normalize_pctwise=pct_norm,istest=True),
last_layer = net['output'],
save_best_params_to=c.params_dir + '/fcn_v{}_{}_f{}.npz'.format(version, vvv,i))
if i < 5 and best_epoch>max_epoch:
max_epoch = best_epoch;
if CV:
for pfn in ['fcn_v{}_{}_f{}.npz'.format(version, vvv, i) for i in xrange(ntimes-1)]:#!!!!CHANGE
u.load_params(net['output'], os.path.join(c.params_dir, pfn))
testfold = int(pfn.split('_')[-1][1])
data = u.DataH5PyStreamer(os.path.join(c.data_sunnybrook, datafile),
batch_size=16, folds=(5,testfold))
streamer = data.streamer(training=False, shuffled=True)
accs = []
for imb in streamer.get_epoch_iterator():
x,y = du.segmenter_data_transform(imb,normalize_pctwise=pct_norm)
