def def_invert(self, model, batch_size=1, beta=0.5, lr=0.1, b1=0.9, nz=100, use_bin=True): beta_r = sharedX(beta) x_c = T.tensor4() m_c = T.tensor4() x_e = T.tensor4() m_e = T.tensor4() z0 = T.matrix() z = sharedX(floatX(np_rng.uniform(-1., 1., size=(batch_size, nz)))) gx = model.model_G(z) mm_c = T.tile(m_c, (1, gx.shape[1], 1, 1)) color_all = T.mean(T.sqr(gx - x_c) * mm_c, axis=(1, 2, 3)) / ( T.mean(m_c, axis=(1, 2, 3)) + sharedX(1e-5)) gx_edge = HOGNet.get_hog(gx, use_bin) x_edge = HOGNet.get_hog(x_e, use_bin) mm_e = T.tile(m_e, (1, gx_edge.shape[1], 1, 1)) sum_e = T.sum(T.abs_(mm_e)) sum_x_edge = T.sum(T.abs_(x_edge)) edge_all = T.mean(T.sqr(x_edge - gx_edge) * mm_e, axis=(1, 2, 3)) / ( T.mean(m_e, axis=(1, 2, 3)) + sharedX(1e-5)) rec_all = color_all + edge_all * sharedX(0.2) z_const = sharedX(10.0) init_all = T.mean(T.sqr(z0 - z)) * z_const if beta > 0: print('using D') p_gen = model.model_D(gx) real_all = T.nnet.binary_crossentropy(p_gen, T.ones( p_gen.shape)).T # costs.bce(p_gen, T.ones(p_gen.shape)) cost_all = rec_all + beta_r * real_all[0] + init_all else: print('without D') cost_all = rec_all + init_all real_all = T.zeros(cost_all.shape) cost = T.sum(cost_all) d_updater = updates.Adam( lr=sharedX(lr), b1=sharedX(b1)) # ,regularizer=updates.Regularizer(l2=l2)) output = [ gx, cost, cost_all, rec_all, real_all, init_all, sum_e, sum_x_edge ] print 'COMPILING...' t = time() z_updates = d_updater([z], cost) _invert = theano.function(inputs=[x_c, m_c, x_e, m_e, z0], outputs=output, updates=z_updates) print '%.2f seconds to compile _invert function' % (time() - t) return [_invert, z_updates, z, beta_r, z_const]
def def_invert(self, model, batch_size=1, d_weight=0.5, nc=1, lr=0.1, b1=0.9, nz=100, use_bin=True): d_weight_r = sharedX(d_weight) x_c = T.tensor4() m_c = T.tensor4() x_e = T.tensor4() m_e = T.tensor4() z0 = T.matrix() z = sharedX(floatX(np_rng.uniform(-1., 1., size=(batch_size, nz)))) gx = model.model_G(z) # input: im_c: 255: no edge; 0: edge; transform=> 1: no edge, 0: edge if nc == 1: # gx, range [0, 1] => edge, 1 gx3 = 1.0 - gx # T.tile(gx, (1, 3, 1, 1)) else: gx3 = gx mm_c = T.tile(m_c, (1, gx3.shape[1], 1, 1)) color_all = T.mean(T.sqr(gx3 - x_c) * mm_c, axis=(1, 2, 3)) / (T.mean(m_c, axis=(1, 2, 3)) + sharedX(1e-5)) gx_edge = self.hog.get_hog(gx3) x_edge = self.hog.get_hog(x_e) mm_e = T.tile(m_e, (1, gx_edge.shape[1], 1, 1)) sum_e = T.sum(T.abs_(mm_e)) sum_x_edge = T.sum(T.abs_(x_edge)) edge_all = T.mean(T.sqr(x_edge - gx_edge) * mm_e, axis=(1, 2, 3)) / (T.mean(m_e, axis=(1, 2, 3)) + sharedX(1e-5)) rec_all = color_all + edge_all * sharedX(0.2) z_const = sharedX(5.0) init_all = T.mean(T.sqr(z0 - z)) * z_const if d_weight > 0: print('using D') p_gen = model.model_D(gx) real_all = T.nnet.binary_crossentropy(p_gen, T.ones(p_gen.shape)).T cost_all = rec_all + d_weight_r * real_all[0] + init_all else: print('without D') cost_all = rec_all + init_all real_all = T.zeros(cost_all.shape) cost = T.sum(cost_all) d_updater = updates.Adam(lr=sharedX(lr), b1=sharedX(b1)) output = [gx, cost, cost_all, rec_all, real_all, init_all, sum_e, sum_x_edge] print('COMPILING...') t = time() z_updates = d_updater([z], cost) _invert = theano.function(inputs=[x_c, m_c, x_e, m_e, z0], outputs=output, updates=z_updates) print('%.2f seconds to compile _invert function' % (time() - t)) return [_invert, z_updates, z, d_weight_r, z_const]
d_cost_real = sum([bce(p, T.ones(p.shape)).mean() for p in p_real]) d_cost_gen = sum([bce(p, T.zeros(p.shape)).mean() for p in p_gen]) g_cost_d = sum([bce(p, T.ones(p.shape)).mean() for p in p_gen]) #d_cost_real = bce(p_real[-1], T.ones(p_real[-1].shape)).mean() #d_cost_gen = bce(p_gen[-1], T.zeros(p_gen[-1].shape)).mean() #g_cost_d = bce(p_gen[-1], T.ones(p_gen[-1].shape)).mean() d_cost = d_cost_real + d_cost_gen + ( 1e-5 * sum([T.sum(p**2.0) for p in discrim_params])) g_cost = g_cost_d + (1e-5 * sum([T.sum(p**2.0) for p in gen_params])) cost = [g_cost, d_cost, g_cost_d, d_cost_real, d_cost_gen] lrt = sharedX(lr) d_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) updates = d_updates + g_updates print 'COMPILING' t = time() _train_g = theano.function([X, Z0], cost, updates=g_updates) _train_d = theano.function([X, Z0], cost, updates=d_updates) _gen = theano.function([Z0], gX) print '%.2f seconds to compile theano functions' % (time() - t) f_log = open("{}/{}.ndjson".format(log_dir, desc), 'wb') log_fields = [ 'n_epochs',
x_net = AlexNet.build_model(x_t, layer=args.layer, shape=(None, 3, npx, npx)) AlexNet.load_model(x_net, layer=args.layer) x_f = lasagne.layers.get_output(x_net[args.layer], deterministic=True) gx_t = AlexNet.transform_im(gx, npx=npx, nc=nc) gx_net = AlexNet.build_model(gx_t, layer=args.layer, shape=(None, 3, npx, npx)) AlexNet.load_model(gx_net, layer=args.layer) gx_f = lasagne.layers.get_output(gx_net[args.layer], deterministic=True) ftr_loss = costs.L2Loss(gx_f, x_f) # add two losses together cost = pixel_loss + ftr_loss * sharedX(args.alpha) output = [cost, z] lrt = sharedX(args.lr) b1t = sharedX(args.b1) p_updater = updates.Adam(lr=lrt, b1=b1t, regularizer=updates.Regularizer(l2=args.weight_decay)) p_updates = p_updater(predict_params, cost) print('COMPILING') t = time() _train_p = theano.function([x], cost, updates=p_updates) _train_p_cost = theano.function([x], [cost, gx]) _predict_z = theano.function([x], z) _gen = theano.function([z], gx) print('%.2f seconds to compile theano functions' % (time() - t)) def rec_test(test_data, n_epochs=0, batch_size=128, output_dir=None): print('computing reconstruction loss on test images')
# def gen_Z(dist): # mu = dist[:Nz] # sigma = dist[Nz:] X = T.tensor5() encodeZ = encoder(X, *encode_params) decodeX = decoder(encodeZ, *decode_params) cost = bce(T.flatten(decodeX, 2), T.flatten(X, 2)).mean() lrt = sharedX(lrate) AutoEnc_parameter = encode_params + decode_params updater = updates.Adam(lr=lrt, b1=0.8, regularizer=updates.Regularizer(l2=l2)) updates = updater(AutoEnc_parameter, cost) print 'COMPILING' t = time() _train_ = theano.function([X], cost, updates=updates) print '%.2f seconds to compile theano functions' % (time() - t) mat = scipy.io.loadmat('models_stats.mat') mat = mat['models'] num = np.array(mat[0][0][1]) names = mat[0][0][0][0] objname = [] for j in range(len(objectNumber)): objname.append(names[objectNumber[j]][0])
gX = gen(Z, *gen_params) p_real = discrim(X, *discrim_params) p_gen = discrim(gX, *discrim_params) d_cost_real = bce(p_real, T.ones(p_real.shape)).mean() d_cost_gen = bce(p_gen, T.zeros(p_gen.shape)).mean() g_cost_d = bce(p_gen, T.ones(p_gen.shape)).mean() d_cost = d_cost_real + d_cost_gen g_cost = g_cost_d cost = [g_cost, d_cost, g_cost_d, d_cost_real, d_cost_gen] lrt = sharedX(lr) d_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=weightdecay)) g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=weightdecay)) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) updates = d_updates + g_updates print 'COMPILING' t = time.time() _train_g = theano.function([X, Z], cost, updates=g_updates) _train_d = theano.function([X, Z], cost, updates=d_updates) _gen = theano.function([Z], gX) print '%.2f seconds to compile theano functions'%(time.time()-t) ########
def load_model(): [e_params, g_params, d_params] = pickle.load(open("faces_dcgan.pkl", "rb")) gwx = g_params[-1] dwy = d_params[-1] # inputs X = T.tensor4() ## encode layer e_layer_sizes = [128, 64, 32, 16, 8] e_filter_sizes = [3, 256, 256, 512, 1024] eX, e_params, e_layers = make_conv_set(X, e_layer_sizes, e_filter_sizes, "e", weights=e_params) ## generative layer g_layer_sizes = [8, 16, 32, 64, 128] g_num_filters = [1024, 512, 256, 256, 128] g_out, g_params, g_layers = make_conv_set(eX, g_layer_sizes, g_num_filters, "g", weights=g_params) g_params += [gwx] gX = tanh(deconv(g_out, gwx, subsample=(1, 1), border_mode=(2, 2))) ## discrim layer(s) df1 = 128 d_layer_sizes = [128, 64, 32, 16, 8] d_filter_sizes = [3, df1, 2 * df1, 4 * df1, 8 * df1] def discrim(input, name, weights=None): d_out, disc_params, d_layers = make_conv_set(input, d_layer_sizes, d_filter_sizes, name, weights=weights) d_flat = T.flatten(d_out, 2) disc_params += [dwy] y = sigmoid(T.dot(d_flat, dwy)) return y, disc_params, d_layers # target outputs target = T.tensor4() p_real, d_params, d_layers = discrim(target, "d", weights=d_params) # we need to make sure the p_gen params are the same as the p_real params p_gen, d_params2, d_layers = discrim(gX, "d", weights=d_params) ## GAN costs d_cost_real = bce(p_real, T.ones(p_real.shape)).mean() d_cost_gen = bce(p_gen, T.zeros(p_gen.shape)).mean() g_cost_d = bce(p_gen, T.ones(p_gen.shape)).mean() ## MSE encoding cost is done on an (averaged) downscaling of the image target_pool = max_pool_2d(target, (4, 4), mode="average_exc_pad", ignore_border=True) target_flat = T.flatten(target_pool, 2) gX_pool = max_pool_2d(gX, (4, 4), mode="average_exc_pad", ignore_border=True) gX_flat = T.flatten(gX_pool, 2) enc_cost = mse(gX_flat, target_flat).mean() ## generator cost is a linear combination of the discrim cost plus the MSE enocding cost d_cost = d_cost_real + d_cost_gen g_cost = g_cost_d + enc_cost / 10 ## if the enc_cost is weighted too highly it will take a long time to train ## N.B. e_cost and e_updates will only try and minimise MSE loss on the autoencoder (for debugging) e_cost = enc_cost cost = [g_cost_d, d_cost_real, enc_cost] elrt = sharedX(0.002) lrt = sharedX(lr) d_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) e_updater = updates.Adam(lr=elrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) d_updates = d_updater(d_params, d_cost) g_updates = g_updater(e_params + g_params, g_cost) e_updates = e_updater(e_params, e_cost) print 'COMPILING' t = time() _train_g = theano.function([X, target], cost, updates=g_updates) _train_d = theano.function([X, target], cost, updates=d_updates) _train_e = theano.function([X, target], cost, updates=e_updates) _get_cost = theano.function([X, target], cost) print('%.2f seconds to compile theano functions' % (time() - t)) img_dir = "gen_images/" if not os.path.exists(img_dir): os.makedirs(img_dir) ae_encode = theano.function([X, target], [gX, target]) return ae_encode
d_cost_real = bce(p_real, T.ones(p_real.shape)).mean() d_cost_gen = bce(p_gen, T.zeros(p_gen.shape)).mean() g_cost_d = bce(p_gen, T.ones(p_gen.shape)).mean() d_cost = d_cost_real + d_cost_gen g_cost = g_cost_d cost = [g_cost, d_cost, g_cost_d, d_cost_real, d_cost_gen] Dlrt = sharedX(Discrimlrt) Glrt = sharedX(Genlrt) # Glrt2 = sharedX(Genlrt2) d_updater = updates.Adam(lr=Dlrt, b1=0.5, regularizer=updates.Regularizer(l2=l2)) g_updater = updates.Adam(lr=Glrt, b1=0.5, regularizer=updates.Regularizer(l2=l2)) # g_updater2 = updates.Adam(lr=Glrt2, b1=0.5, regularizer=updates.Regularizer(l2=l2)) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) # g_updates2 = g_updater2(gen_params, g_cost) updates = d_updates + g_updates print 'COMPILING' t = time() _train_g = theano.function([X, Z], cost, updates=g_updates) # _train_g2 = theano.function([X, Z], cost, updates=g_updates2) _train_d = theano.function([X, Z], cost, updates=d_updates)
word_K_list_flat_ZT[T.arange(As_word_list_flat.shape[0]) * n_word_dict + As_word_list_flat] + 1e-7) #tensor.arange(x_flat.shape[0]) * probs.shape[1] + x_flat cost_re_ZT = T.reshape(cost_ZT, [As_word_list.shape[1], As_word_list.shape[0]], ndim=2) #T *batch cost1_ZT = cost_re_ZT * As_mask.T #T *batch cost2_ZT = cost1_ZT.sum(axis=0) #/Mask_captions.sum(axis=0) cost3_ZT = cost2_ZT.mean() cost4 = cost3 + (KL_cost0 + KL_cost_t) * KL_weight + cost3_bow * alpha + beta * cost3_ZT lrt = sharedX(learning_rate) g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2), clipnorm=10) g_updates = g_updater(total_params, cost4) print 'COMPILING' t = time() _train = theano.function([ KL_weight, Qs_word_list, As_word_list, Qs_mask, As_mask, Qns_word_list, Ans_word_list, Qns_mask, Ans_mask ], [cost4, cost3, KL_cost0, KL_cost_t, cost3_bow], updates=g_updates) #, profile=True) print '%.2f seconds to compile theano functions' % (time() - t) print 'finish printing'
reconstructed_x, logpxz = conv_decoder(x_repeated, z, *dec_params) z_vgd_grad = 0. - _vgd_gradient(z, num_z, logpxz) # L operator dHdPhi = T.Lop( f=z.flatten() / T.cast(num_z * nbatch, 'float32'), wrt=enc_params, eval_points=z_vgd_grad.flatten()) en_updater = updates.GAdam(lr=sharedX(en_lrt), regularizer=updates.Regularizer(l2=l2)) en_updates = en_updater(enc_params, dHdPhi) decost = 0 - logpxz.sum() / T.cast(num_z * nbatch, 'float32') de_updater = updates.Adam(lr=sharedX(de_lrt), regularizer=updates.Regularizer(l2=l2)) de_updates = de_updater(dec_params, decost) gupdates = en_updates + de_updates X_train, X_valid, X_test = mnist() ntrain, nvalid, ntest = len(X_train), len(X_valid), len(X_test) print X_train.shape, X_valid.shape, X_test.shape print 'COMPILING' t = time() _train = theano.function([X, num_z], decost, updates=gupdates) _reconstruct = theano.function([X], func_res_x)
Xtsdf = T.tensor5() # input voxel grid ylabels = T.matrix() # ground truth classification labels predictions = classify(Xtsdf, model_params[0], model_params[1], model_params[2], model_params[3], model_params[4], model_params[5], model_params[6], model_params[7], model_params[8], model_params[9], model_params[10], model_params[11], model_params[12], model_params[13], model_params[14]) cost_y = cce(predictions, ylabels).mean() cost = [cost_y] updater = updates.Adam() updates = updater(model_params, cost_y) print 'COMPILING' t = time() _train_ = theano.function([Xtsdf, ylabels], cost, updates=updates) print '%.2f seconds to compile theano functions\n' % (time() - t) print '\nNum training images:', nImages, '--- Number of class labels:', nb_classes, '\n' nb_epochs = 50 nb_samples_per_epoch = 75000 nbatch = 6 Xmb = np.zeros((nbatch, 1, Nx, Ny, Nz), dtype=np.float32) ymb = np.zeros((nbatch, nb_classes), dtype=np.float32)
gX = gen(Z, *gen_params) p_real = discrim(X, *discrim_params) p_gen = discrim(gX, *discrim_params) d_cost_real = bce(p_real, T.ones(p_real.shape)).mean() d_cost_gen = bce(p_gen, T.zeros(p_gen.shape)).mean() g_cost_d = bce(p_gen, T.ones(p_gen.shape)).mean() d_cost = d_cost_real + d_cost_gen g_cost = g_cost_d cost = [g_cost, d_cost, g_cost_d, d_cost_real, d_cost_gen] lrt = sharedX(lr) d_updater = updates.Adam(lr=lrt, b1=b1) g_updater = updates.Adam(lr=lrt, b1=b1) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) updates = d_updates + g_updates print 'COMPILING' t = time() _train_g = theano.function([X, Z], cost, updates=g_updates) _train_d = theano.function([X, Z], cost, updates=d_updates) _gen = theano.function([Z], gX) print '%.2f seconds to compile theano functions' % (time() - t) vis_idxs = py_rng.sample(np.arange(len(vaX)), nvis) vaX_vis = inverse_transform(vaX[vis_idxs]) color_grid_vis(vaX_vis, (20, 20), 'samples/%s_etl_test.png' % desc)