def def_invert(self,
model,
batch_size=1,
beta=0.5,
lr=0.1,
b1=0.9,
nz=100,
use_bin=True):
beta_r = sharedX(beta)
x_c = T.tensor4()
m_c = T.tensor4()
x_e = T.tensor4()
m_e = T.tensor4()
z0 = T.matrix()
z = sharedX(floatX(np_rng.uniform(-1., 1., size=(batch_size, nz))))
gx = model.model_G(z)
mm_c = T.tile(m_c, (1, gx.shape[1], 1, 1))
color_all = T.mean(T.sqr(gx - x_c) * mm_c, axis=(1, 2, 3)) / (
T.mean(m_c, axis=(1, 2, 3)) + sharedX(1e-5))
gx_edge = HOGNet.get_hog(gx, use_bin)
x_edge = HOGNet.get_hog(x_e, use_bin)
mm_e = T.tile(m_e, (1, gx_edge.shape[1], 1, 1))
sum_e = T.sum(T.abs_(mm_e))
sum_x_edge = T.sum(T.abs_(x_edge))
edge_all = T.mean(T.sqr(x_edge - gx_edge) * mm_e, axis=(1, 2, 3)) / (
T.mean(m_e, axis=(1, 2, 3)) + sharedX(1e-5))
rec_all = color_all + edge_all * sharedX(0.2)
z_const = sharedX(10.0)
init_all = T.mean(T.sqr(z0 - z)) * z_const
if beta > 0:
print('using D')
p_gen = model.model_D(gx)
real_all = T.nnet.binary_crossentropy(p_gen, T.ones(
p_gen.shape)).T # costs.bce(p_gen, T.ones(p_gen.shape))
cost_all = rec_all + beta_r * real_all[0] + init_all
else:
print('without D')
cost_all = rec_all + init_all
real_all = T.zeros(cost_all.shape)
cost = T.sum(cost_all)
d_updater = updates.Adam(
lr=sharedX(lr),
b1=sharedX(b1)) # ,regularizer=updates.Regularizer(l2=l2))
output = [
gx, cost, cost_all, rec_all, real_all, init_all, sum_e, sum_x_edge
]
print 'COMPILING...'
t = time()
z_updates = d_updater([z], cost)
_invert = theano.function(inputs=[x_c, m_c, x_e, m_e, z0],
outputs=output,
updates=z_updates)
print '%.2f seconds to compile _invert function' % (time() - t)
return [_invert, z_updates, z, beta_r, z_const]
def def_invert(self, model, batch_size=1, d_weight=0.5, nc=1, lr=0.1, b1=0.9, nz=100, use_bin=True):
d_weight_r = sharedX(d_weight)
x_c = T.tensor4()
m_c = T.tensor4()
x_e = T.tensor4()
m_e = T.tensor4()
z0 = T.matrix()
z = sharedX(floatX(np_rng.uniform(-1., 1., size=(batch_size, nz))))
gx = model.model_G(z)
# input: im_c: 255: no edge; 0: edge; transform=> 1: no edge, 0: edge
if nc == 1: # gx, range [0, 1] => edge, 1
gx3 = 1.0 - gx # T.tile(gx, (1, 3, 1, 1))
else:
gx3 = gx
mm_c = T.tile(m_c, (1, gx3.shape[1], 1, 1))
color_all = T.mean(T.sqr(gx3 - x_c) * mm_c, axis=(1, 2, 3)) / (T.mean(m_c, axis=(1, 2, 3)) + sharedX(1e-5))
gx_edge = self.hog.get_hog(gx3)
x_edge = self.hog.get_hog(x_e)
mm_e = T.tile(m_e, (1, gx_edge.shape[1], 1, 1))
sum_e = T.sum(T.abs_(mm_e))
sum_x_edge = T.sum(T.abs_(x_edge))
edge_all = T.mean(T.sqr(x_edge - gx_edge) * mm_e, axis=(1, 2, 3)) / (T.mean(m_e, axis=(1, 2, 3)) + sharedX(1e-5))
rec_all = color_all + edge_all * sharedX(0.2)
z_const = sharedX(5.0)
init_all = T.mean(T.sqr(z0 - z)) * z_const
if d_weight > 0:
print('using D')
p_gen = model.model_D(gx)
real_all = T.nnet.binary_crossentropy(p_gen, T.ones(p_gen.shape)).T
cost_all = rec_all + d_weight_r * real_all[0] + init_all
else:
print('without D')
cost_all = rec_all + init_all
real_all = T.zeros(cost_all.shape)
cost = T.sum(cost_all)
d_updater = updates.Adam(lr=sharedX(lr), b1=sharedX(b1))
output = [gx, cost, cost_all, rec_all, real_all, init_all, sum_e, sum_x_edge]
print('COMPILING...')
t = time()
z_updates = d_updater([z], cost)
_invert = theano.function(inputs=[x_c, m_c, x_e, m_e, z0], outputs=output, updates=z_updates)
print('%.2f seconds to compile _invert function' % (time() - t))
return [_invert, z_updates, z, d_weight_r, z_const]
d_cost_real = sum([bce(p, T.ones(p.shape)).mean() for p in p_real])
d_cost_gen = sum([bce(p, T.zeros(p.shape)).mean() for p in p_gen])
g_cost_d = sum([bce(p, T.ones(p.shape)).mean() for p in p_gen])
#d_cost_real = bce(p_real[-1], T.ones(p_real[-1].shape)).mean()
#d_cost_gen = bce(p_gen[-1], T.zeros(p_gen[-1].shape)).mean()
#g_cost_d = bce(p_gen[-1], T.ones(p_gen[-1].shape)).mean()
d_cost = d_cost_real + d_cost_gen + (
1e-5 * sum([T.sum(p**2.0) for p in discrim_params]))
g_cost = g_cost_d + (1e-5 * sum([T.sum(p**2.0) for p in gen_params]))
cost = [g_cost, d_cost, g_cost_d, d_cost_real, d_cost_gen]
lrt = sharedX(lr)
d_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2))
g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2))
d_updates = d_updater(discrim_params, d_cost)
g_updates = g_updater(gen_params, g_cost)
updates = d_updates + g_updates
print 'COMPILING'
t = time()
_train_g = theano.function([X, Z0], cost, updates=g_updates)
_train_d = theano.function([X, Z0], cost, updates=d_updates)
_gen = theano.function([Z0], gX)
print '%.2f seconds to compile theano functions' % (time() - t)
f_log = open("{}/{}.ndjson".format(log_dir, desc), 'wb')
log_fields = [
'n_epochs',
x_net = AlexNet.build_model(x_t, layer=args.layer, shape=(None, 3, npx, npx))
AlexNet.load_model(x_net, layer=args.layer)
x_f = lasagne.layers.get_output(x_net[args.layer], deterministic=True)
gx_t = AlexNet.transform_im(gx, npx=npx, nc=nc)
gx_net = AlexNet.build_model(gx_t, layer=args.layer, shape=(None, 3, npx, npx))
AlexNet.load_model(gx_net, layer=args.layer)
gx_f = lasagne.layers.get_output(gx_net[args.layer], deterministic=True)
ftr_loss = costs.L2Loss(gx_f, x_f)
# add two losses together
cost = pixel_loss + ftr_loss * sharedX(args.alpha)
output = [cost, z]
lrt = sharedX(args.lr)
b1t = sharedX(args.b1)
p_updater = updates.Adam(lr=lrt,
b1=b1t,
regularizer=updates.Regularizer(l2=args.weight_decay))
p_updates = p_updater(predict_params, cost)
print('COMPILING')
t = time()
_train_p = theano.function([x], cost, updates=p_updates)
_train_p_cost = theano.function([x], [cost, gx])
_predict_z = theano.function([x], z)
_gen = theano.function([z], gx)
print('%.2f seconds to compile theano functions' % (time() - t))
def rec_test(test_data, n_epochs=0, batch_size=128, output_dir=None):
print('computing reconstruction loss on test images')
# def gen_Z(dist):
# mu = dist[:Nz]
# sigma = dist[Nz:]
X = T.tensor5()
encodeZ = encoder(X, *encode_params)
decodeX = decoder(encodeZ, *decode_params)
cost = bce(T.flatten(decodeX, 2), T.flatten(X, 2)).mean()
lrt = sharedX(lrate)
AutoEnc_parameter = encode_params + decode_params
updater = updates.Adam(lr=lrt, b1=0.8, regularizer=updates.Regularizer(l2=l2))
updates = updater(AutoEnc_parameter, cost)
print 'COMPILING'
t = time()
_train_ = theano.function([X], cost, updates=updates)
print '%.2f seconds to compile theano functions' % (time() - t)
mat = scipy.io.loadmat('models_stats.mat')
mat = mat['models']
num = np.array(mat[0][0][1])
names = mat[0][0][0][0]
objname = []
for j in range(len(objectNumber)):
objname.append(names[objectNumber[j]][0])
gX = gen(Z, *gen_params) p_real = discrim(X, *discrim_params) p_gen = discrim(gX, *discrim_params) d_cost_real = bce(p_real, T.ones(p_real.shape)).mean() d_cost_gen = bce(p_gen, T.zeros(p_gen.shape)).mean() g_cost_d = bce(p_gen, T.ones(p_gen.shape)).mean() d_cost = d_cost_real + d_cost_gen g_cost = g_cost_d cost = [g_cost, d_cost, g_cost_d, d_cost_real, d_cost_gen] lrt = sharedX(lr) d_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=weightdecay)) g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=weightdecay)) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) updates = d_updates + g_updates print 'COMPILING' t = time.time() _train_g = theano.function([X, Z], cost, updates=g_updates) _train_d = theano.function([X, Z], cost, updates=d_updates) _gen = theano.function([Z], gX) print '%.2f seconds to compile theano functions'%(time.time()-t) ########
def load_model():
[e_params, g_params, d_params] = pickle.load(open("faces_dcgan.pkl", "rb"))
gwx = g_params[-1]
dwy = d_params[-1]
# inputs
X = T.tensor4()
## encode layer
e_layer_sizes = [128, 64, 32, 16, 8]
e_filter_sizes = [3, 256, 256, 512, 1024]
eX, e_params, e_layers = make_conv_set(X,
e_layer_sizes,
e_filter_sizes,
"e",
weights=e_params)
## generative layer
g_layer_sizes = [8, 16, 32, 64, 128]
g_num_filters = [1024, 512, 256, 256, 128]
g_out, g_params, g_layers = make_conv_set(eX,
g_layer_sizes,
g_num_filters,
"g",
weights=g_params)
g_params += [gwx]
gX = tanh(deconv(g_out, gwx, subsample=(1, 1), border_mode=(2, 2)))
## discrim layer(s)
df1 = 128
d_layer_sizes = [128, 64, 32, 16, 8]
d_filter_sizes = [3, df1, 2 * df1, 4 * df1, 8 * df1]
def discrim(input, name, weights=None):
d_out, disc_params, d_layers = make_conv_set(input,
d_layer_sizes,
d_filter_sizes,
name,
weights=weights)
d_flat = T.flatten(d_out, 2)
disc_params += [dwy]
y = sigmoid(T.dot(d_flat, dwy))
return y, disc_params, d_layers
# target outputs
target = T.tensor4()
p_real, d_params, d_layers = discrim(target, "d", weights=d_params)
# we need to make sure the p_gen params are the same as the p_real params
p_gen, d_params2, d_layers = discrim(gX, "d", weights=d_params)
## GAN costs
d_cost_real = bce(p_real, T.ones(p_real.shape)).mean()
d_cost_gen = bce(p_gen, T.zeros(p_gen.shape)).mean()
g_cost_d = bce(p_gen, T.ones(p_gen.shape)).mean()
## MSE encoding cost is done on an (averaged) downscaling of the image
target_pool = max_pool_2d(target, (4, 4),
mode="average_exc_pad",
ignore_border=True)
target_flat = T.flatten(target_pool, 2)
gX_pool = max_pool_2d(gX, (4, 4),
mode="average_exc_pad",
ignore_border=True)
gX_flat = T.flatten(gX_pool, 2)
enc_cost = mse(gX_flat, target_flat).mean()
## generator cost is a linear combination of the discrim cost plus the MSE enocding cost
d_cost = d_cost_real + d_cost_gen
g_cost = g_cost_d + enc_cost / 10 ## if the enc_cost is weighted too highly it will take a long time to train
## N.B. e_cost and e_updates will only try and minimise MSE loss on the autoencoder (for debugging)
e_cost = enc_cost
cost = [g_cost_d, d_cost_real, enc_cost]
elrt = sharedX(0.002)
lrt = sharedX(lr)
d_updater = updates.Adam(lr=lrt,
b1=b1,
regularizer=updates.Regularizer(l2=l2))
g_updater = updates.Adam(lr=lrt,
b1=b1,
regularizer=updates.Regularizer(l2=l2))
e_updater = updates.Adam(lr=elrt,
b1=b1,
regularizer=updates.Regularizer(l2=l2))
d_updates = d_updater(d_params, d_cost)
g_updates = g_updater(e_params + g_params, g_cost)
e_updates = e_updater(e_params, e_cost)
print 'COMPILING'
t = time()
_train_g = theano.function([X, target], cost, updates=g_updates)
_train_d = theano.function([X, target], cost, updates=d_updates)
_train_e = theano.function([X, target], cost, updates=e_updates)
_get_cost = theano.function([X, target], cost)
print('%.2f seconds to compile theano functions' % (time() - t))
img_dir = "gen_images/"
if not os.path.exists(img_dir):
os.makedirs(img_dir)
ae_encode = theano.function([X, target], [gX, target])
return ae_encode
d_cost_real = bce(p_real, T.ones(p_real.shape)).mean()
d_cost_gen = bce(p_gen, T.zeros(p_gen.shape)).mean()
g_cost_d = bce(p_gen, T.ones(p_gen.shape)).mean()
d_cost = d_cost_real + d_cost_gen
g_cost = g_cost_d
cost = [g_cost, d_cost, g_cost_d, d_cost_real, d_cost_gen]
Dlrt = sharedX(Discrimlrt)
Glrt = sharedX(Genlrt)
# Glrt2 = sharedX(Genlrt2)
d_updater = updates.Adam(lr=Dlrt,
b1=0.5,
regularizer=updates.Regularizer(l2=l2))
g_updater = updates.Adam(lr=Glrt,
b1=0.5,
regularizer=updates.Regularizer(l2=l2))
# g_updater2 = updates.Adam(lr=Glrt2, b1=0.5, regularizer=updates.Regularizer(l2=l2))
d_updates = d_updater(discrim_params, d_cost)
g_updates = g_updater(gen_params, g_cost)
# g_updates2 = g_updater2(gen_params, g_cost)
updates = d_updates + g_updates
print 'COMPILING'
t = time()
_train_g = theano.function([X, Z], cost, updates=g_updates)
# _train_g2 = theano.function([X, Z], cost, updates=g_updates2)
_train_d = theano.function([X, Z], cost, updates=d_updates)
word_K_list_flat_ZT[T.arange(As_word_list_flat.shape[0]) * n_word_dict +
As_word_list_flat] +
1e-7) #tensor.arange(x_flat.shape[0]) * probs.shape[1] + x_flat
cost_re_ZT = T.reshape(cost_ZT, [As_word_list.shape[1], As_word_list.shape[0]],
ndim=2) #T *batch
cost1_ZT = cost_re_ZT * As_mask.T #T *batch
cost2_ZT = cost1_ZT.sum(axis=0) #/Mask_captions.sum(axis=0)
cost3_ZT = cost2_ZT.mean()
cost4 = cost3 + (KL_cost0 +
KL_cost_t) * KL_weight + cost3_bow * alpha + beta * cost3_ZT
lrt = sharedX(learning_rate)
g_updater = updates.Adam(lr=lrt,
b1=b1,
regularizer=updates.Regularizer(l2=l2),
clipnorm=10)
g_updates = g_updater(total_params, cost4)
print 'COMPILING'
t = time()
_train = theano.function([
KL_weight, Qs_word_list, As_word_list, Qs_mask, As_mask, Qns_word_list,
Ans_word_list, Qns_mask, Ans_mask
], [cost4, cost3, KL_cost0, KL_cost_t, cost3_bow],
updates=g_updates) #, profile=True)
print '%.2f seconds to compile theano functions' % (time() - t)
print 'finish printing'
reconstructed_x, logpxz = conv_decoder(x_repeated, z, *dec_params)
z_vgd_grad = 0. - _vgd_gradient(z, num_z, logpxz)
# L operator
dHdPhi = T.Lop(
f=z.flatten() / T.cast(num_z * nbatch, 'float32'),
wrt=enc_params,
eval_points=z_vgd_grad.flatten())
en_updater = updates.GAdam(lr=sharedX(en_lrt), regularizer=updates.Regularizer(l2=l2))
en_updates = en_updater(enc_params, dHdPhi)
decost = 0 - logpxz.sum() / T.cast(num_z * nbatch, 'float32')
de_updater = updates.Adam(lr=sharedX(de_lrt), regularizer=updates.Regularizer(l2=l2))
de_updates = de_updater(dec_params, decost)
gupdates = en_updates + de_updates
X_train, X_valid, X_test = mnist()
ntrain, nvalid, ntest = len(X_train), len(X_valid), len(X_test)
print X_train.shape, X_valid.shape, X_test.shape
print 'COMPILING'
t = time()
_train = theano.function([X, num_z], decost, updates=gupdates)
_reconstruct = theano.function([X], func_res_x)
Xtsdf = T.tensor5() # input voxel grid
ylabels = T.matrix() # ground truth classification labels
predictions = classify(Xtsdf, model_params[0], model_params[1],
model_params[2], model_params[3], model_params[4],
model_params[5], model_params[6], model_params[7],
model_params[8], model_params[9], model_params[10],
model_params[11], model_params[12], model_params[13],
model_params[14])
cost_y = cce(predictions, ylabels).mean()
cost = [cost_y]
updater = updates.Adam()
updates = updater(model_params, cost_y)
print 'COMPILING'
t = time()
_train_ = theano.function([Xtsdf, ylabels], cost, updates=updates)
print '%.2f seconds to compile theano functions\n' % (time() - t)
print '\nNum training images:', nImages, '--- Number of class labels:', nb_classes, '\n'
nb_epochs = 50
nb_samples_per_epoch = 75000
nbatch = 6
Xmb = np.zeros((nbatch, 1, Nx, Ny, Nz), dtype=np.float32)
ymb = np.zeros((nbatch, nb_classes), dtype=np.float32)
gX = gen(Z, *gen_params) p_real = discrim(X, *discrim_params) p_gen = discrim(gX, *discrim_params) d_cost_real = bce(p_real, T.ones(p_real.shape)).mean() d_cost_gen = bce(p_gen, T.zeros(p_gen.shape)).mean() g_cost_d = bce(p_gen, T.ones(p_gen.shape)).mean() d_cost = d_cost_real + d_cost_gen g_cost = g_cost_d cost = [g_cost, d_cost, g_cost_d, d_cost_real, d_cost_gen] lrt = sharedX(lr) d_updater = updates.Adam(lr=lrt, b1=b1) g_updater = updates.Adam(lr=lrt, b1=b1) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) updates = d_updates + g_updates print 'COMPILING' t = time() _train_g = theano.function([X, Z], cost, updates=g_updates) _train_d = theano.function([X, Z], cost, updates=d_updates) _gen = theano.function([Z], gX) print '%.2f seconds to compile theano functions' % (time() - t) vis_idxs = py_rng.sample(np.arange(len(vaX)), nvis) vaX_vis = inverse_transform(vaX[vis_idxs]) color_grid_vis(vaX_vis, (20, 20), 'samples/%s_etl_test.png' % desc)
