log_fields = [ 'n_epochs', 'n_updates', 'n_examples', 'n_seconds', '1k_va_nnd', '10k_va_nnd', '100k_va_nnd', 'g_cost', 'd_cost', ] tr_data, te_data, tr_stream, val_stream, te_stream = faces(ntrain=ntrain) # Only tr_data/tr_stream are used. tr_handle = tr_data.open() vaX, = tr_data.get_data(tr_handle, slice(0, 10000)) vaX = transform(vaX) vis_idxs = py_rng.sample(np.arange(len(vaX)), nvis) vaX_vis = inverse_transform(vaX[vis_idxs]) color_grid_vis(vaX_vis, (14, 14), 'samples/%s_etl_test.png'%desc) sample_zmb = floatX(np_rng.uniform(-1., 1., size=(nvis, nz))) vaX = vaX.reshape(len(vaX), -1) # DEFINE NETWORKS. relu = activations.Rectify() sigmoid = activations.Sigmoid() lrelu = activations.LeakyRectify() tanh = activations.Tanh() bce = T.nnet.binary_crossentropy gifn = inits.Normal(scale=0.02) difn = inits.Normal(scale=0.02) gain_ifn = inits.Normal(loc=1., scale=0.02) bias_ifn = inits.Constant(c=0.)
fmbs = _features(floatX(xmb)) for i, fmb in enumerate(fmbs): Xfs.append(fmb) return np.concatenate(Xfs, axis=0) cs = [0.0001, 0.0002, 0.0005, 0.001, 0.002, 0.005, 0.01] vaXt = features(vaX) mean_va_accs = [] for c in cs: tr_accs = [] va_accs = [] te_accs = [] for _ in tqdm(range(10), leave=False, ncols=80): idxs = np.arange(len(trX)) classes_idxs = [idxs[trY==y] for y in range(10)] sampled_idxs = [py_rng.sample(class_idxs, 100) for class_idxs in classes_idxs] sampled_idxs = np.asarray(sampled_idxs).flatten() trXt = features(trX[sampled_idxs]) model = LSVC(C=c) model.fit(trXt[:1000], trY[sampled_idxs]) tr_pred = model.predict(trXt) va_pred = model.predict(vaXt) tr_acc = metrics.accuracy_score(trY[sampled_idxs], tr_pred[:1000]) va_acc = metrics.accuracy_score(vaY, va_pred) tr_accs.append(100*(1-tr_acc)) va_accs.append(100*(1-va_acc)) mean_va_accs.append(np.mean(va_accs)) print 'c: %.4f train: %.4f %.4f valid: %.4f %.4f'%(c, np.mean(tr_accs), np.std(tr_accs)*1.96, np.mean(va_accs), np.std(va_accs)*1.96) best_va_idx = np.argmin(mean_va_accs)
do_initial_valid = True n_epochs = j['n_epochs'] n_updates = j['n_updates'] n_examples = j['n_examples'] lrg.set_value(floatX(j['lrg'])) lrd.set_value(floatX(j['lrd'])) l2t.set_value(floatX(j['l2d'])) log_lines.append(l) f_log.close() f_log = open('logs/%s.ndjson' % desc, 'wb') for l in log_lines: f_log.write(l) vis_idxs = py_rng.sample(np.arange(len(vaX)), nvis) vaX_vis = inverse_transform(vaX[vis_idxs]) color_grid_vis(vaX_vis, (args.nvis2, args.nvis2), 'samples/%s_etl_test.png' % desc) sample_zmb = gen_z(nvis) vaX = vaX.reshape(len(vaX), -1) print desc.upper() t = time() costs = [] label_sums = np.zeros(2) def validate():
lrt = sharedX(lr) d_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) updates = d_updates + g_updates print "COMPILING" t = time() _train_g = theano.function([X, Z, Y], cost, updates=g_updates) _train_d = theano.function([X, Z, Y], cost, updates=d_updates) _gen = theano.function([Z, Y], gX) print "%.2f seconds to compile theano functions" % (time() - t) tr_idxs = np.arange(len(trX)) trX_vis = np.asarray([[trX[i] for i in py_rng.sample(tr_idxs[trY == y], 20)] for y in range(10)]).reshape(200, -1) trX_vis = inverse_transform(transform(trX_vis)) grayscale_grid_vis(trX_vis, (10, 20), "samples/%s_etl_test.png" % desc) sample_zmb = floatX(np_rng.uniform(-1.0, 1.0, size=(200, nz))) sample_ymb = floatX(OneHot(np.asarray([[i for _ in range(20)] for i in range(10)]).flatten(), ny)) def gen_samples(n, nbatch=128): samples = [] labels = [] n_gen = 0 for i in range(n / nbatch): ymb = floatX(OneHot(np_rng.randint(0, 10, nbatch), ny)) zmb = floatX(np_rng.uniform(-1.0, 1.0, size=(nbatch, nz))) xmb = _gen(zmb, ymb)
web_dir = os.path.join(args.cache_dir, 'web_dcgan') html = image_save.ImageSave(web_dir, expr_name, append=True) utils.mkdirs([sample_dir, model_dir, log_dir, web_dir]) # load data from hdf5 file tr_data, te_data, tr_stream, te_stream, ntrain, ntest = load.load_imgs( ntrain=None, ntest=None, batch_size=args.batch_size, data_file=args.data_file) te_handle = te_data.open() test_x, = te_data.get_data(te_handle, slice(0, ntest)) # generate real samples and test transform/inverse_transform test_x = train_dcgan_utils.transform(test_x, nc=nc) vis_idxs = py_rng.sample(np.arange(len(test_x)), n_vis) vaX_vis = train_dcgan_utils.inverse_transform(test_x[vis_idxs], npx=npx, nc=nc) # st() n_grid = int(np.sqrt(n_vis)) grid_real = utils.grid_vis((vaX_vis * 255.0).astype(np.uint8), n_grid, n_grid) train_dcgan_utils.save_image(grid_real, os.path.join(sample_dir, 'real_samples.png')) # define DCGAN model disc_params = train_dcgan_utils.init_disc_params(n_f=n_f, n_layers=n_layers, nc=nc) gen_params = train_dcgan_utils.init_gen_params(nz=nz, n_f=n_f, n_layers=n_layers, nc=nc)
g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) updates = d_updates + g_updates print 'COMPILING' t = time() _train_g = theano.function([X, Z, Y], cost, updates=g_updates) _train_d = theano.function([X, Z, Y], cost, updates=d_updates) ### _gen = theano.function([Z, Y], [gX, yb, yb2, d, h3, h5]) print '%.2f seconds to compile theano functions' % (time() - t) tr_idxs = np.arange(len(trX)) trX_vis = np.asarray([[trX[i] for i in py_rng.sample(tr_idxs[trY == y], 20)] for y in range(10)]).reshape(200, -1) trX_vis = inverse_transform(transform(trX_vis)) grayscale_grid_vis(trX_vis, (10, 20), 'samples/%s_etl_test.png' % desc) sample_zmb = floatX(np_rng.uniform(-1., 1., size=(200, nz))) sample_ymb = floatX( OneHot( np.asarray([[i for _ in range(20)] for i in range(10)]).flatten(), ny)) def gen_samples(n, nbatch=128): samples = [] labels = [] n_gen = 0 for i in range(n / nbatch):
d_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) updates = d_updates + g_updates print 'COMPILING' t = time() _train_g = theano.function([X, Z, Y], cost, updates=g_updates) _train_d = theano.function([X, Z, Y], cost, updates=d_updates) _gen = theano.function([Z, Y], gX) print '%.2f seconds to compile theano functions'%(time()-t) cols = 10 tr_idxs = np.arange(len(trX)) trX_vis = np.asarray([[trX[i] for i in py_rng.sample(tr_idxs[trY==y], cols)] for y in range(ny)]).reshape(ny * cols, -1) trX_vis = inverse_transform(transform(trX_vis)) grayscale_grid_vis(trX_vis, (ny, cols), 'samples/test.png') ############ # set up targets normally steps = 6 numtargets = 9 #This is how many letter you will count start = 1 targets = np.asarray([[i+start for _ in range(steps)] for i in range(numtargets)]) sample_ymb = floatX(OneHot(targets.flatten(), ny)) # set up random z sample_zmb = floatX(np_rng.uniform(-1., 1., size=(numtargets * steps, nz)))
sys.stdout.flush() print('COMPILING...') t = time() _train_g = theano.function([X, Z, Y], cost, updates=g_updates) _train_d = theano.function([X, Z, Y], cost, updates=d_updates) _gen = theano.function( [Z, Y], [gX, out_lYS, out_G3_1, out_G3_2, out_G10, out_G11, out_G12]) print('COMPILING...DONE') print('%.2f seconds to compile theano functions' % (time() - t)) # # Saving the training images in grey-scaled. # tr_idxs = np.arange(len(trX0)) trX0_vis = np.asarray( [[trX0[i] for i in py_rng.sample(tr_idxs[trY0 == y], 20)] for y in range(10)]).reshape(200, -1) trX0_vis = inverse_transform(transform(trX0_vis)) grayscale_grid_vis(trX0_vis, (10, 20), 'samples/%s_etl_test.png' % desc) f_log = open('logs/%s.ndjson' % desc, 'wb') log_fields = [ 'n_epochs', 'n_updates', 'n_examples', 'n_seconds', '1k_va_nnc_acc', '10k_va_nnc_acc', '100k_va_nnc_acc', '1k_va_nnd',
# create directories sample_dir = os.path.join(args.cache_dir, 'samples') model_dir = os.path.join(args.cache_dir, 'models') log_dir = os.path.join(args.cache_dir, 'log') web_dir = os.path.join(args.cache_dir, 'web_dcgan') html = image_save.ImageSave(web_dir, expr_name, append=True) utils.mkdirs([sample_dir, model_dir, log_dir, web_dir]) # load data from hdf5 file tr_data, te_data, tr_stream, te_stream, ntrain, ntest = load.load_imgs(ntrain=None, ntest=None, batch_size=args.batch_size,data_file=args.data_file) te_handle = te_data.open() test_x, = te_data.get_data(te_handle, slice(0, ntest)) # generate real samples and test transform/inverse_transform test_x = train_dcgan_utils.transform(test_x, nc=nc) vis_idxs = py_rng.sample(np.arange(len(test_x)), n_vis) vaX_vis = train_dcgan_utils.inverse_transform(test_x[vis_idxs], npx=npx, nc=nc) # st() n_grid = int(np.sqrt(n_vis)) grid_real = utils.grid_vis((vaX_vis*255.0).astype(np.uint8), n_grid, n_grid) train_dcgan_utils.save_image(grid_real, os.path.join(sample_dir, 'real_samples.png')) # define DCGAN model disc_params = train_dcgan_utils.init_disc_params(n_f=n_f, n_layers=n_layers, nc=nc) gen_params = train_dcgan_utils.init_gen_params(nz=nz, n_f=n_f, n_layers=n_layers, nc=nc) x = T.tensor4() z = T.matrix() gx = train_dcgan_utils.gen(z, gen_params, n_layers=n_layers, n_f=n_f, nc=nc) p_real = train_dcgan_utils.discrim(x, disc_params, n_layers=n_layers)
Xfs.append(fmb) return np.concatenate(Xfs, axis=0) cs = [0.0001, 0.0002, 0.0005, 0.001, 0.002, 0.005, 0.01] vaXt = features(vaX) mean_va_accs = [] for c in cs: tr_accs = [] va_accs = [] te_accs = [] for _ in tqdm(range(10), leave=False, ncols=80): idxs = np.arange(len(trX)) classes_idxs = [idxs[trY == y] for y in range(10)] sampled_idxs = [ py_rng.sample(class_idxs, 100) for class_idxs in classes_idxs ] sampled_idxs = np.asarray(sampled_idxs).flatten() trXt = features(trX[sampled_idxs]) model = LSVC(C=c) model.fit(trXt[:1000], trY[sampled_idxs]) tr_pred = model.predict(trXt) va_pred = model.predict(vaXt) tr_acc = metrics.accuracy_score(trY[sampled_idxs], tr_pred[:1000]) va_acc = metrics.accuracy_score(vaY, va_pred) tr_accs.append(100 * (1 - tr_acc)) va_accs.append(100 * (1 - va_acc)) mean_va_accs.append(np.mean(va_accs)) print 'c: %.4f train: %.4f %.4f valid: %.4f %.4f' % (
lrt = sharedX(lr) d_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) g_updater = updates.Adam(lr=lrt, b1=b1, regularizer=updates.Regularizer(l2=l2)) d_updates = d_updater(discrim_params, d_cost) g_updates = g_updater(gen_params, g_cost) updates = d_updates + g_updates print 'COMPILING' t = time() _train_g = theano.function([X, Z, Y], cost, updates=g_updates) _train_d = theano.function([X, Z, Y], cost, updates=d_updates) _gen = theano.function([Z, Y], gX) print '%.2f seconds to compile theano functions'%(time()-t) tr_idxs = np.arange(len(trX)) trX_vis = np.asarray([[trX[i] for i in py_rng.sample(tr_idxs[trY==y], 20)] for y in range(10)]).reshape(200, -1) trX_vis = inverse_transform(transform(trX_vis)) grayscale_grid_vis(trX_vis, (10, 20), 'samples/%s_etl_test.png'%desc) sample_zmb = floatX(np_rng.uniform(-1., 1., size=(200, nz))) sample_ymb = floatX(OneHot(np.asarray([[i for _ in range(20)] for i in range(10)]).flatten(), ny)) def gen_samples(n, nbatch=128): samples = [] labels = [] n_gen = 0 for i in range(n/nbatch): ymb = floatX(OneHot(np_rng.randint(0, 10, nbatch), ny)) zmb = floatX(np_rng.uniform(-1., 1., size=(nbatch, nz))) xmb = _gen(zmb, ymb) samples.append(xmb)