add_resume_counter=True,
other_keys=[lowest_valid_str,
corresp_test_str,
test_nll_str])
# At this point we saved the pkl file.
last_print_iters = total_iters
print "### RESUMING JOB FROM EPOCH {}, ITER {}".format(epoch, total_iters)
# Consumes this much iters to get to the last point in training data.
consume_time = time.time()
for i in xrange(iters_to_consume):
tr_feeder.next()
consume_time = time.time() - consume_time
print "Train data ready in {:.2f}secs after consuming {} minibatches.".\
format(consume_time, iters_to_consume)
lib.load_params(res_path)
print "Parameters from last available checkpoint loaded from path {}".format(res_path)
test_time = 0.0
while True:
# THIS IS ONE ITERATION
if total_iters % 500 == 0:
print total_iters,
total_iters += 1
try:
# Take as many mini-batches as possible from train set
mini_batch = tr_feeder.next()
except StopIteration:
# ('reconst', reconst_cost),
# ('reg', reg_cost),
# ('likelihood', loglikelihood)
# ],
# optimizer=functools.partial(lasagne.updates.adam, learning_rate=LR),
# train_data=train_data,
# test_data=dev_data,
# callback=generate_and_save_samples,
# times=TIMES
# )
k_ = 1
print("Evaluating loglikelihood with k={}".format(k_))
SAVE_FILE = "file_name"
path = OUT_DIR
lib.load_params(os.path.join(path, 'iters100000_time14396.23296_params.pkl'))
print "Generating"
generate_and_save_samples("yet_initial_Samples")
vae_bound = reconst_cost + reg_cost
log_lik_latent_prior = log_lik(latents, 0., 0.)
log_lik_latent_posterior = log_lik(latents, mu, log_sigma)
loglikelihood_normal = log_lik_latent_prior - reconst_cost - log_lik_latent_posterior
loglikelihood = -log_mean_exp(loglikelihood_normal)
lik_fn = theano.function(
[images],
[loglikelihood, vae_bound, reconst_cost, reg_cost, log_lik_latent_prior, log_lik_latent_posterior, loglikelihood_normal]
)
add_resume_counter=True,
other_keys=[lowest_valid_str,
corresp_test_str,
test_nll_str])
# At this point we saved the pkl file.
last_print_iters = total_iters
print "### RESUMING JOB FROM EPOCH {}, ITER {}".format(epoch, total_iters)
# Consumes this much iters to get to the last point in training data.
consume_time = time()
for i in xrange(iters_to_consume):
tr_feeder.next()
consume_time = time() - consume_time
print "Train data ready in {:.2f}secs after consuming {} minibatches.".\
format(consume_time, iters_to_consume)
lib.load_params(res_path)
print "Parameters from last available checkpoint loaded."
while True:
# THIS IS ONE ITERATION
if total_iters % 500 == 0:
print total_iters,
total_iters += 1
try:
# Take as many mini-batches as possible from train set
mini_batch = tr_feeder.next()
except StopIteration:
# Mini-batches are finished. Load it again.
# Basically, one epoch.
costs = []
lowest_valid_cost = numpy.finfo(numpy.float32).max
corresponding_test_cost = numpy.finfo(numpy.float32).max
new_lowest_cost = False
end_of_batch = False
epoch = 0
h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * DIM), dtype='float32')
big_h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * BIG_DIM), dtype='float32')
# Initial load train dataset
tr_feeder = load_data(train_feeder)
if GEN_FLAG:
print "Loading checkpoint ", args.checkpoint
lib.load_params(args.checkpoint)
print "Wall clock time spent before sampling started: {:.2f}mins"\
.format((time()-exp_start)/60.)
print "Sampling!",
# Load conditioning
print "Loading conditioning file " + CONDITIONING_FILES
conditionings = [
numpy.loadtxt(e, dtype='int8', delimiter=";")
for e in CONDITIONING_FILES
]
# Generate samples
for conditioning in conditionings:
# For each conditioning, we generate N_SEQS sequences
conditioning_repeated = numpy.repeat(conditioning, N_SEQS, axis=1)
generate_and_save_samples('staticgeneration', conditioning_repeated)
corresponding_test_cost = numpy.finfo(numpy.float32).max
new_lowest_cost = False
end_of_batch = False
epoch = 0
cost_log_list = []
h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * DIM), dtype='float32')
big_h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * BIG_DIM), dtype='float32')
# Initial load train dataset
tr_feeder = load_data(train_feeder)
if FLAG_GEN:
print('---loading gen_para.pkl---')
lib.load_params(GEN_DIRFILE)
print('---loading complete---')
print('sampling')
tmp = GEN_DIRFILE.split('/')[-1]
generate_and_save_samples(tmp)
print('ok')
sys.exit()
### start from uncon
if UCINIT_DIRFILE == 'flat_start': FLAG_UCINIT = False
else: FLAG_UCINIT = True
if (FLAG_UCINIT and not RESUME):
print('---loading uncon_para_expand_3t.pkl---')
uncon_para_expand_path = UCINIT_DIRFILE
lib.load_params(uncon_para_expand_path)
print('---loading complete---')
samp = mu2linear(samp)
elif Q_TYPE == 'a-law':
raise NotImplementedError('a-law is not implemented')
write_audio_file("sample_{}_{}".format(tag, i), samp)
def save_audio_file(path, data):
data = data.astype('float32')
data -= data.min()
data /= data.max()
data -= 0.5
data *= 0.95
scipy.io.wavfile.write(path, BITRATE, data)
lib.load_params(os.path.join(BEST_PATH, 'params_best.pkl'))
print "Parameters from last available checkpoint loaded."
from datasets.dataset import \
find_dataset, __test, __onom_file, __batch_quantize
data_path = find_dataset(__test(__onom_file))
files = numpy.load(data_path)
for seed_ind in [0, 32, 64, 96, 128, 160, 192, 224, 240, 255][::-1]:
print "\nIND: ", seed_ind
selected_seed = numpy.array(files[seed_ind])[None, :]
print "Len of selected test file: ", len(selected_seed[0])
selected_seed = __batch_quantize(selected_seed, Q_LEVELS, Q_TYPE)
save_audio_file('seed{}.wav'.format(seed_ind), selected_seed[0])
for div in [1, 2, 4]:
print "\tdiv: ", div
# Just seed
samples[:, i, j, k] = samples_p_value[:, i, j, k]
t1 = time.time()
print("Time taken for generation normally {:.4f}".format(t1 - t0))
save_images(samples_p_value, 'samples_pval_repeated_')
# samples = np.zeros(
# (100, N_CHANNELS, HEIGHT, WIDTH),
# dtype=theano.config.floatX
# )
# samples = generate_with_only_receptive_field(sample_fn, samples, latents)
# save_images(samples, 'samples_pval_')
# generate_and_save_samples("initial_Samples")
lib.load_params(os.path.join(OUT_DIR, 'iters70500_time10614.8930545_params.pkl'))
generate_and_save_samples("initial_Samples")
exit()
# exit()
print("Training")
lib.train_loop.train_loop(
inputs=[total_iters, images],
inject_total_iters=True,
cost=cost.mean(),
prints=[
('alpha', alpha),
('reconst', reconst_cost.mean()),
('reg', reg_cost.mean())
new_tag = tag
images = images.transpose(0, 2, 3, 1)
images = images.reshape((10, 10, 64, 64, 3))
images = images.transpose(0, 2, 1, 3, 4)
images = images.reshape((10 * 64, 10 * 64, 3))
image = scipy.misc.toimage(images, channel_axis=2)
image.save('{}/{}_{}.jpg'.format(OUT_DIR_SAMPLES, filename, new_tag))
save_images(images, "samples")
if args.resume:
lib.load_params(os.path.join(OUT_DIR_PARAMS, "params_last.pkl"))
print "Creating data loader.."
train_data, dev_data, test_data = lib.celeba.load(BATCH_SIZE, TEST_BATCH_SIZE)
generate_and_save_samples("initial")
print "Running Train loop.."
try:
lib.train_loop.train_loop(inputs=[total_iters, images],
inject_total_iters=True,
cost=cost,
prints=[('KL', kl_cost),
('reconst', reconst_cost),
('alpha/beta', alpha)],
print "Wall clock time spent before training started: {:.2f}h"\
.format((time()-exp_start)/3600.)
print "Training!"
total_iters = 0
total_time = 0.
last_print_time = 0.
last_print_iters = 0
costs = []
lowest_valid_cost = numpy.finfo(numpy.float32).max
corresponding_test_cost = numpy.finfo(numpy.float32).max
new_lowest_cost = False
end_of_batch = False
epoch = 0
h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * DIM), dtype='float32')
pdb.set_trace()
### load para and generate
print('---loading para pkl---')
paraFile = 'params_e12_i76801_t13.87_tr1.4649_v1.2918_best.pkl'
paraPath = 'results_2t/models-two_tier-two_tier_con.py-expCON_ZLINIT_FULLLAB-n_frames40-frame_size20-emb_size256-skip_connF-dim1024-n_rnn3-rnn_typeGRU-q_levels256-q_typelinear-batch_size20-weight_normT-learn_h0T-which_setSPEECH-lr0.001/params/' + paraFile
lib.load_params(paraPath)
print('---loading complete---')
tag = 'T4generation_' + paraFile
print('debug: sampling')
generate_and_save_samples(tag)
print('debug: ok')
sys.exit()
np.asarray([np.random.choice(N_SPEAKERS),]).reshape((1,)).astype('int32'),
ctx
)
generate.generate_wav(pred_X[0],base='generated_sample_16',do_post_filtering=False)
def char2wav(text,PATH='/data/lisa/exp/kumarrit/vctk'):
lib.load_params('vctk_nmt_best.pkl')
model = nltk.corpus.cmudict.dict()
char_to_idx = pickle.load(open(PATH+'/char2code.pkl'))
result = list(text)
ctx = np.asarray([36]+[char_to_idx[x] for x in result]+[0]).reshape((1,-1)).astype('int32')
phons = nmt_fn(ctx,np.ones_like(ctx).astype('float32')).flatten()
try:
end_idx = np.where(phons==0)[0][0]
phons = phons[:end_idx].tolist()
except:
phons = phons.tolist()
pred_X = predict_fn(
np.asarray([np.random.choice(N_SPEAKERS),]).reshape((1,)).astype('int32'),
np.asarray(phons,dtype=np.int32).reshape((1,-1))
)
generate.generate_wav(pred_X[0],base='generated_sample_16',do_post_filtering=False)
# new_sentence("GET A GLIMPSE OF WHAT THE LYRE BIRD A P I WILL OFFER")
print "Compiled Function!"
lib.load_params('vctk_synthesizer_GRU_l1.pkl')
lib.load_params('vctk_duration_predictor_best.pkl')
lib.load_params('vctk_nmt_best.pkl')
# for x,y in lib._updates.iteritems():
# if 'Generator' in x.name:
# updates[x] = y
# print "Adding update: ",x.name
train_fn = theano.function([X, ctx, mask, learn_rate],
cost,
updates=updates,
on_unused_input='warn')
test_fn = theano.function([X, ctx, mask],
test_cost,
on_unused_input='warn')
lib.load_params(
'/data/lisa/exp/kumarrit/ppgn-speech/blizzard_tanh_recognizer_best.pkl'
)
print "Compiled Function!"
i = 0
train_stream = parrot_stream('blizzard',
False, ('train', ),
BATCH_SIZE,
noise_level=0.,
labels_type='phonemes',
seq_size=100000)
i = 0
best_score = 1000.
for i in xrange(i, NB_EPOCHS):
loglikelihood = -log_mean_exp(loglikelihood_normal)
lik_fn = theano.function([images], [
loglikelihood, vae_bound, reconst_cost, reg_cost, log_lik_latent_prior,
log_lik_latent_posterior, loglikelihood_normal
])
def compute_importance_weighted_likelihood():
i = 0
total_lik = []
total_lik_bound = []
for (images, ) in test_data():
for im in images:
batch_ = np.tile(im, [k_, 1, 1, 1])
res = lik_fn(batch_)
total_lik_bound.append(res[1].mean())
total_lik.append(res[0])
i += 1
print "Importance weighted likelihood", np.mean(total_lik)
print "normal likelihood", np.mean(total_lik_bound)
print("Loading parameters...")
lib.load_params(args.pre_trained_weights)
print("Computing Log-likelihood..")
compute_importance_weighted_likelihood()
costs = []
lowest_valid_cost = numpy.finfo(numpy.float32).max
corresponding_test_cost = numpy.finfo(numpy.float32).max
new_lowest_cost = False
end_of_batch = False
epoch = 0
h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT * DIM), dtype='float32')
# Initial load train dataset
tr_feeder = load_data(train_feeder)
### start from uncon
print('---loading uncon_para_expand.pkl---')
uncon_para_expand_path = 'uncon_para_expand.pkl'
lib.load_params(uncon_para_expand_path)
print('---loading complete---')
### Handling the resume option:
if RESUME:
# Check if checkpoint from previous run is not corrupted.
# Then overwrite some of the variables above.
iters_to_consume, res_path, epoch, total_iters,\
[lowest_valid_cost, corresponding_test_cost, test_cost] = \
lib.resumable(path=FOLDER_PREFIX,
iter_key=iter_str,
epoch_key=epoch_str,
add_resume_counter=True,
other_keys=[lowest_valid_str,
corresp_test_str,
test_nll_str])
total_time = 0.
last_print_time = 0.
last_print_iters = 0
costs = []
lowest_valid_cost = numpy.finfo(numpy.float32).max
corresponding_test_cost = numpy.finfo(numpy.float32).max
new_lowest_cost = False
end_of_batch = False
h0 = numpy.zeros((BATCH_SIZE, N_RNN, H0_MULT*DIM), dtype='float32')
# Initial load train dataset
tr_feeder = load_data(train_feeder)
print "Loading checkpoint ", args.checkpoint
lib.load_params(args.checkpoint)
print epoch
print "Wall clock time spent before sampling started: {:.2f}mins"\
.format((time()-exp_start)/60.)
print "Sampling!",
# Generate samples
generate_and_save_samples('staticgeneration_e' + epoch)
print "Done!"
print "Wall clock time spent after sampling ended: {:.2f}mins"\
.format((time()-exp_start)/60.)
