add_resume_counter=True, other_keys=[lowest_valid_str, corresp_test_str, test_nll_str]) # At this point we saved the pkl file. last_print_iters = total_iters print "### RESUMING JOB FROM EPOCH {}, ITER {}".format(epoch, total_iters) # Consumes this much iters to get to the last point in training data. consume_time = time.time() for i in xrange(iters_to_consume): tr_feeder.next() consume_time = time.time() - consume_time print "Train data ready in {:.2f}secs after consuming {} minibatches.".\ format(consume_time, iters_to_consume) lib.load_params(res_path) print "Parameters from last available checkpoint loaded from path {}".format(res_path) test_time = 0.0 while True: # THIS IS ONE ITERATION if total_iters % 500 == 0: print total_iters, total_iters += 1 try: # Take as many mini-batches as possible from train set mini_batch = tr_feeder.next() except StopIteration:
# ('reconst', reconst_cost), # ('reg', reg_cost), # ('likelihood', loglikelihood) # ], # optimizer=functools.partial(lasagne.updates.adam, learning_rate=LR), # train_data=train_data, # test_data=dev_data, # callback=generate_and_save_samples, # times=TIMES # ) k_ = 1 print("Evaluating loglikelihood with k={}".format(k_)) SAVE_FILE = "file_name" path = OUT_DIR lib.load_params(os.path.join(path, 'iters100000_time14396.23296_params.pkl')) print "Generating" generate_and_save_samples("yet_initial_Samples") vae_bound = reconst_cost + reg_cost log_lik_latent_prior = log_lik(latents, 0., 0.) log_lik_latent_posterior = log_lik(latents, mu, log_sigma) loglikelihood_normal = log_lik_latent_prior - reconst_cost - log_lik_latent_posterior loglikelihood = -log_mean_exp(loglikelihood_normal) lik_fn = theano.function( [images], [loglikelihood, vae_bound, reconst_cost, reg_cost, log_lik_latent_prior, log_lik_latent_posterior, loglikelihood_normal] )
add_resume_counter=True, other_keys=[lowest_valid_str, corresp_test_str, test_nll_str]) # At this point we saved the pkl file. last_print_iters = total_iters print "### RESUMING JOB FROM EPOCH {}, ITER {}".format(epoch, total_iters) # Consumes this much iters to get to the last point in training data. consume_time = time() for i in xrange(iters_to_consume): tr_feeder.next() consume_time = time() - consume_time print "Train data ready in {:.2f}secs after consuming {} minibatches.".\ format(consume_time, iters_to_consume) lib.load_params(res_path) print "Parameters from last available checkpoint loaded." while True: # THIS IS ONE ITERATION if total_iters % 500 == 0: print total_iters, total_iters += 1 try: # Take as many mini-batches as possible from train set mini_batch = tr_feeder.next() except StopIteration: # Mini-batches are finished. Load it again. # Basically, one epoch.
costs = [] lowest_valid_cost = numpy.finfo(numpy.float32).max corresponding_test_cost = numpy.finfo(numpy.float32).max new_lowest_cost = False end_of_batch = False epoch = 0 h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * DIM), dtype='float32') big_h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * BIG_DIM), dtype='float32') # Initial load train dataset tr_feeder = load_data(train_feeder) if GEN_FLAG: print "Loading checkpoint ", args.checkpoint lib.load_params(args.checkpoint) print "Wall clock time spent before sampling started: {:.2f}mins"\ .format((time()-exp_start)/60.) print "Sampling!", # Load conditioning print "Loading conditioning file " + CONDITIONING_FILES conditionings = [ numpy.loadtxt(e, dtype='int8', delimiter=";") for e in CONDITIONING_FILES ] # Generate samples for conditioning in conditionings: # For each conditioning, we generate N_SEQS sequences conditioning_repeated = numpy.repeat(conditioning, N_SEQS, axis=1) generate_and_save_samples('staticgeneration', conditioning_repeated)
corresponding_test_cost = numpy.finfo(numpy.float32).max new_lowest_cost = False end_of_batch = False epoch = 0 cost_log_list = [] h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * DIM), dtype='float32') big_h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * BIG_DIM), dtype='float32') # Initial load train dataset tr_feeder = load_data(train_feeder) if FLAG_GEN: print('---loading gen_para.pkl---') lib.load_params(GEN_DIRFILE) print('---loading complete---') print('sampling') tmp = GEN_DIRFILE.split('/')[-1] generate_and_save_samples(tmp) print('ok') sys.exit() ### start from uncon if UCINIT_DIRFILE == 'flat_start': FLAG_UCINIT = False else: FLAG_UCINIT = True if (FLAG_UCINIT and not RESUME): print('---loading uncon_para_expand_3t.pkl---') uncon_para_expand_path = UCINIT_DIRFILE lib.load_params(uncon_para_expand_path) print('---loading complete---')
samp = mu2linear(samp) elif Q_TYPE == 'a-law': raise NotImplementedError('a-law is not implemented') write_audio_file("sample_{}_{}".format(tag, i), samp) def save_audio_file(path, data): data = data.astype('float32') data -= data.min() data /= data.max() data -= 0.5 data *= 0.95 scipy.io.wavfile.write(path, BITRATE, data) lib.load_params(os.path.join(BEST_PATH, 'params_best.pkl')) print "Parameters from last available checkpoint loaded." from datasets.dataset import \ find_dataset, __test, __onom_file, __batch_quantize data_path = find_dataset(__test(__onom_file)) files = numpy.load(data_path) for seed_ind in [0, 32, 64, 96, 128, 160, 192, 224, 240, 255][::-1]: print "\nIND: ", seed_ind selected_seed = numpy.array(files[seed_ind])[None, :] print "Len of selected test file: ", len(selected_seed[0]) selected_seed = __batch_quantize(selected_seed, Q_LEVELS, Q_TYPE) save_audio_file('seed{}.wav'.format(seed_ind), selected_seed[0]) for div in [1, 2, 4]: print "\tdiv: ", div # Just seed
samples[:, i, j, k] = samples_p_value[:, i, j, k] t1 = time.time() print("Time taken for generation normally {:.4f}".format(t1 - t0)) save_images(samples_p_value, 'samples_pval_repeated_') # samples = np.zeros( # (100, N_CHANNELS, HEIGHT, WIDTH), # dtype=theano.config.floatX # ) # samples = generate_with_only_receptive_field(sample_fn, samples, latents) # save_images(samples, 'samples_pval_') # generate_and_save_samples("initial_Samples") lib.load_params(os.path.join(OUT_DIR, 'iters70500_time10614.8930545_params.pkl')) generate_and_save_samples("initial_Samples") exit() # exit() print("Training") lib.train_loop.train_loop( inputs=[total_iters, images], inject_total_iters=True, cost=cost.mean(), prints=[ ('alpha', alpha), ('reconst', reconst_cost.mean()), ('reg', reg_cost.mean())
new_tag = tag images = images.transpose(0, 2, 3, 1) images = images.reshape((10, 10, 64, 64, 3)) images = images.transpose(0, 2, 1, 3, 4) images = images.reshape((10 * 64, 10 * 64, 3)) image = scipy.misc.toimage(images, channel_axis=2) image.save('{}/{}_{}.jpg'.format(OUT_DIR_SAMPLES, filename, new_tag)) save_images(images, "samples") if args.resume: lib.load_params(os.path.join(OUT_DIR_PARAMS, "params_last.pkl")) print "Creating data loader.." train_data, dev_data, test_data = lib.celeba.load(BATCH_SIZE, TEST_BATCH_SIZE) generate_and_save_samples("initial") print "Running Train loop.." try: lib.train_loop.train_loop(inputs=[total_iters, images], inject_total_iters=True, cost=cost, prints=[('KL', kl_cost), ('reconst', reconst_cost), ('alpha/beta', alpha)],
print "Wall clock time spent before training started: {:.2f}h"\ .format((time()-exp_start)/3600.) print "Training!" total_iters = 0 total_time = 0. last_print_time = 0. last_print_iters = 0 costs = [] lowest_valid_cost = numpy.finfo(numpy.float32).max corresponding_test_cost = numpy.finfo(numpy.float32).max new_lowest_cost = False end_of_batch = False epoch = 0 h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * DIM), dtype='float32') pdb.set_trace() ### load para and generate print('---loading para pkl---') paraFile = 'params_e12_i76801_t13.87_tr1.4649_v1.2918_best.pkl' paraPath = 'results_2t/models-two_tier-two_tier_con.py-expCON_ZLINIT_FULLLAB-n_frames40-frame_size20-emb_size256-skip_connF-dim1024-n_rnn3-rnn_typeGRU-q_levels256-q_typelinear-batch_size20-weight_normT-learn_h0T-which_setSPEECH-lr0.001/params/' + paraFile lib.load_params(paraPath) print('---loading complete---') tag = 'T4generation_' + paraFile print('debug: sampling') generate_and_save_samples(tag) print('debug: ok') sys.exit()
np.asarray([np.random.choice(N_SPEAKERS),]).reshape((1,)).astype('int32'), ctx ) generate.generate_wav(pred_X[0],base='generated_sample_16',do_post_filtering=False) def char2wav(text,PATH='/data/lisa/exp/kumarrit/vctk'): lib.load_params('vctk_nmt_best.pkl') model = nltk.corpus.cmudict.dict() char_to_idx = pickle.load(open(PATH+'/char2code.pkl')) result = list(text) ctx = np.asarray([36]+[char_to_idx[x] for x in result]+[0]).reshape((1,-1)).astype('int32') phons = nmt_fn(ctx,np.ones_like(ctx).astype('float32')).flatten() try: end_idx = np.where(phons==0)[0][0] phons = phons[:end_idx].tolist() except: phons = phons.tolist() pred_X = predict_fn( np.asarray([np.random.choice(N_SPEAKERS),]).reshape((1,)).astype('int32'), np.asarray(phons,dtype=np.int32).reshape((1,-1)) ) generate.generate_wav(pred_X[0],base='generated_sample_16',do_post_filtering=False) # new_sentence("GET A GLIMPSE OF WHAT THE LYRE BIRD A P I WILL OFFER") print "Compiled Function!" lib.load_params('vctk_synthesizer_GRU_l1.pkl') lib.load_params('vctk_duration_predictor_best.pkl') lib.load_params('vctk_nmt_best.pkl')
# for x,y in lib._updates.iteritems(): # if 'Generator' in x.name: # updates[x] = y # print "Adding update: ",x.name train_fn = theano.function([X, ctx, mask, learn_rate], cost, updates=updates, on_unused_input='warn') test_fn = theano.function([X, ctx, mask], test_cost, on_unused_input='warn') lib.load_params( '/data/lisa/exp/kumarrit/ppgn-speech/blizzard_tanh_recognizer_best.pkl' ) print "Compiled Function!" i = 0 train_stream = parrot_stream('blizzard', False, ('train', ), BATCH_SIZE, noise_level=0., labels_type='phonemes', seq_size=100000) i = 0 best_score = 1000. for i in xrange(i, NB_EPOCHS):
loglikelihood = -log_mean_exp(loglikelihood_normal) lik_fn = theano.function([images], [ loglikelihood, vae_bound, reconst_cost, reg_cost, log_lik_latent_prior, log_lik_latent_posterior, loglikelihood_normal ]) def compute_importance_weighted_likelihood(): i = 0 total_lik = [] total_lik_bound = [] for (images, ) in test_data(): for im in images: batch_ = np.tile(im, [k_, 1, 1, 1]) res = lik_fn(batch_) total_lik_bound.append(res[1].mean()) total_lik.append(res[0]) i += 1 print "Importance weighted likelihood", np.mean(total_lik) print "normal likelihood", np.mean(total_lik_bound) print("Loading parameters...") lib.load_params(args.pre_trained_weights) print("Computing Log-likelihood..") compute_importance_weighted_likelihood()
costs = [] lowest_valid_cost = numpy.finfo(numpy.float32).max corresponding_test_cost = numpy.finfo(numpy.float32).max new_lowest_cost = False end_of_batch = False epoch = 0 h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * DIM), dtype='float32') # Initial load train dataset tr_feeder = load_data(train_feeder) ### start from uncon print('---loading uncon_para_expand.pkl---') uncon_para_expand_path = 'uncon_para_expand.pkl' lib.load_params(uncon_para_expand_path) print('---loading complete---') ### Handling the resume option: if RESUME: # Check if checkpoint from previous run is not corrupted. # Then overwrite some of the variables above. iters_to_consume, res_path, epoch, total_iters,\ [lowest_valid_cost, corresponding_test_cost, test_cost] = \ lib.resumable(path=FOLDER_PREFIX, iter_key=iter_str, epoch_key=epoch_str, add_resume_counter=True, other_keys=[lowest_valid_str, corresp_test_str, test_nll_str])
total_time = 0. last_print_time = 0. last_print_iters = 0 costs = [] lowest_valid_cost = numpy.finfo(numpy.float32).max corresponding_test_cost = numpy.finfo(numpy.float32).max new_lowest_cost = False end_of_batch = False h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT*DIM), dtype='float32') # Initial load train dataset tr_feeder = load_data(train_feeder) print "Loading checkpoint ", args.checkpoint lib.load_params(args.checkpoint) print epoch print "Wall clock time spent before sampling started: {:.2f}mins"\ .format((time()-exp_start)/60.) print "Sampling!", # Generate samples generate_and_save_samples('staticgeneration_e' + epoch) print "Done!" print "Wall clock time spent after sampling ended: {:.2f}mins"\ .format((time()-exp_start)/60.)