def train(
dim_word=100, # word vector dimensionality
dim=1000, # the number of LSTM units
encoder='gru',
decoder='gru_cond',
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
diag_c=0.,
clip_c=-1.,
lrate=0.01,
n_words_src=100000,
n_words=100000,
maxlen=100, # maximum length of the description
optimizer='rmsprop',
batch_size=16,
valid_batch_size=16,
saveto='model.npz',
validFreq=1000,
saveFreq=1000, # save the parameters after every saveFreq updates
sampleFreq=100, # generate some samples after every sampleFreq updates
datasets=[
'/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.en.tok',
'/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.fr.tok'
],
valid_datasets=[
'../data/dev/newstest2011.en.tok',
'../data/dev/newstest2011.fr.tok'
],
dictionaries=[
'/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.en.tok.pkl',
'/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.fr.tok.pkl'
],
use_dropout=False,
reload_=False):
# Model options
model_options = locals().copy()
worddicts = [None] * len(dictionaries)
worddicts_r = [None] * len(dictionaries)
for ii, dd in enumerate(dictionaries):
with open(dd, 'rb') as f:
worddicts[ii] = pkl.load(f)
worddicts_r[ii] = dict()
for kk, vv in worddicts[ii].iteritems():
worddicts_r[ii][vv] = kk
# reload options
if reload_ and os.path.exists(saveto):
with open('%s.pkl' % saveto, 'rb') as f:
models_options = pkl.load(f)
print 'Loading data'
train = TextIterator(datasets[0],
datasets[1],
dictionaries[0],
dictionaries[1],
n_words_source=n_words_src,
n_words_target=n_words,
batch_size=batch_size,
maxlen=maxlen)
valid = TextIterator(valid_datasets[0],
valid_datasets[1],
dictionaries[0],
dictionaries[1],
n_words_source=n_words_src,
n_words_target=n_words,
batch_size=valid_batch_size,
maxlen=maxlen)
print 'Building model'
params = init_params(model_options)
# reload parameters
if reload_ and os.path.exists(saveto):
params = load_params(saveto, params)
tparams = init_tparams(params)
trng, use_noise, \
x, x_mask, y, y_mask, \
opt_ret, \
cost = \
build_model(tparams, model_options)
inps = [x, x_mask, y, y_mask]
print 'Buliding sampler'
f_init, f_next = build_sampler(tparams, model_options, trng)
# before any regularizer
print 'Building f_log_probs...',
f_log_probs = theano.function(inps, cost, profile=profile)
print 'Done'
cost = cost.mean()
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
if alpha_c > 0. and not model_options['decoder'].endswith('simple'):
alpha_c = theano.shared(numpy.float32(alpha_c), name='alpha_c')
alpha_reg = alpha_c * (
(tensor.cast(y_mask.sum(0) // x_mask.sum(0), 'float32')[:, None] -
opt_ret['dec_alphas'].sum(0))**2).sum(1).mean()
cost += alpha_reg
# after any regularizer
print 'Building f_cost...',
f_cost = theano.function(inps, cost, profile=profile)
print 'Done'
print 'Computing gradient...',
grads = tensor.grad(cost, wrt=itemlist(tparams))
print 'Done'
print 'Building f_grad...',
f_grad = theano.function(inps, grads, profile=profile)
print 'Done'
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(
tensor.switch(g2 > (clip_c**2), g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, inps, cost)
print 'Done'
print 'Optimization'
history_errs = []
# reload history
if reload_ and os.path.exists(saveto):
history_errs = list(numpy.load(saveto)['history_errs'])
best_p = None
bad_count = 0
if validFreq == -1:
validFreq = len(train[0]) / batch_size
if saveFreq == -1:
saveFreq = len(train[0]) / batch_size
if sampleFreq == -1:
sampleFreq = len(train[0]) / batch_size
uidx = 0
estop = False
for eidx in xrange(max_epochs):
n_samples = 0
for x, y in train:
n_samples += len(x)
uidx += 1
use_noise.set_value(1.)
x, x_mask, y, y_mask = prepare_data(x,
y,
maxlen=maxlen,
n_words_src=n_words_src,
n_words=n_words)
if x is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
cost = f_grad_shared(x, x_mask, y, y_mask)
f_update(lrate)
ud = time.time() - ud_start
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
return 1., 1., 1.
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud
if numpy.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None:
params = best_p
else:
params = unzip(tparams)
numpy.savez(saveto, history_errs=history_errs, **params)
pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'))
print 'Done'
if numpy.mod(uidx, sampleFreq) == 0:
# FIXME: random selection?
for jj in xrange(numpy.minimum(5, x.shape[1])):
stochastic = True
sample, score = gen_sample(tparams,
f_init,
f_next,
x[:, jj][:, None],
model_options,
trng=trng,
k=1,
maxlen=30,
stochastic=stochastic,
argmax=False)
print 'Source ', jj, ': ',
for vv in x[:, jj]:
if vv == 0:
break
if vv in worddicts_r[0]:
print worddicts_r[0][vv],
else:
print 'UNK',
print
print 'Truth ', jj, ' : ',
for vv in y[:, jj]:
if vv == 0:
break
if vv in worddicts_r[1]:
print worddicts_r[1][vv],
else:
print 'UNK',
print
print 'Sample ', jj, ': ',
if stochastic:
ss = sample
else:
score = score / numpy.array([len(s) for s in sample])
ss = sample[score.argmin()]
for vv in ss:
if vv == 0:
break
if vv in worddicts_r[1]:
print worddicts_r[1][vv],
else:
print 'UNK',
print
if numpy.mod(uidx, validFreq) == 0:
use_noise.set_value(0.)
valid_errs = pred_probs(f_log_probs, prepare_data,
model_options, valid)
valid_err = valid_errs.mean()
history_errs.append(valid_err)
if uidx == 0 or valid_err <= numpy.array(history_errs).min():
best_p = unzip(tparams)
bad_counter = 0
if len(history_errs) > patience and valid_err >= numpy.array(
history_errs)[:-patience].min():
bad_counter += 1
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if numpy.isnan(valid_err):
import ipdb
ipdb.set_trace()
print 'Valid ', valid_err
print 'Seen %d samples' % n_samples
if estop:
break
if best_p is not None:
zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = pred_probs(f_log_probs, prepare_data, model_options,
valid).mean()
print 'Valid ', valid_err
params = copy.copy(best_p)
numpy.savez(saveto,
zipped_params=best_p,
history_errs=history_errs,
**params)
return valid_err
def train(dim_word=100, # word vector dimensionality
dim=1000, # the number of GRU units
encoder='gru',
patience=10, # early stopping patience
max_epochs=5000,
finish_after=10000000, # finish after this many updates
dispFreq=100,
decay_c=0., # L2 weight decay penalty
lrate=0.01,
n_words=100000, # vocabulary size
maxlen=100, # maximum length of the description
optimizer='rmsprop',
batch_size=16,
valid_batch_size=16,
saveto='model.npz',
validFreq=1000,
saveFreq=1000, # save the parameters after every saveFreq updates
sampleFreq=100, # generate some samples after every sampleFreq
dataset='/data/lisatmp4/anirudhg/wiki.tok.txt.gz',
valid_dataset='/data/lisatmp4/anirudhg/newstest2011.en.tok',
dictionary='/data/lisatmp4/anirudhg/wiki.tok.txt.gz.pkl',
use_dropout=False,
reload_=False):
# Model options
model_options = locals().copy()
# load dictionary
with open(dictionary, 'rb') as f:
worddicts = pkl.load(f)
# invert dictionary
worddicts_r = dict()
for kk, vv in worddicts.iteritems():
worddicts_r[vv] = kk
# reload options
if reload_ and os.path.exists(saveto):
with open('%s.pkl' % saveto, 'rb') as f:
model_options = pkl.load(f)
print 'Loading data'
train = TextIterator(dataset,
dictionary,
n_words_source=n_words,
batch_size=batch_size,
maxlen=maxlen)
valid = TextIterator(valid_dataset,
dictionary,
n_words_source=n_words,
batch_size=valid_batch_size,
maxlen=maxlen)
print 'Building model'
params = init_params(model_options)
# reload parameters
if reload_ and os.path.exists(saveto):
params = load_params(saveto, params)
# create shared variables for parameters
tparams = init_tparams(params)
# build the symbolic computational graph
trng, use_noise, \
x, x_mask, \
opt_ret, \
cost = \
build_model(tparams, model_options)
inps = [x, x_mask]
print 'Buliding sampler'
f_next = build_sampler(tparams, model_options, trng)
# before any regularizer
print 'Building f_log_probs...',
f_log_probs = theano.function(inps, cost, profile=profile)
print 'Done'
cost = cost.mean()
# apply L2 regularization on weights
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
# after any regularizer - compile the computational graph for cost
print 'Building f_cost...',
f_cost = theano.function(inps, cost, profile=profile)
print 'Done'
print 'Computing gradient...',
grads = tensor.grad(cost, wrt=itemlist(tparams))
print 'Done'
# compile the optimizer, the actual computational graph is compiled here
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
f_grad_shared, f_update = getattr(optimizers, optimizer)(lr, tparams,
grads, inps, cost)
print 'Done'
print 'Optimization'
history_errs = []
# reload history
if reload_ and os.path.exists(saveto):
history_errs = list(numpy.load(saveto)['history_errs'])
best_p = None
bad_count = 0
if validFreq == -1:
validFreq = len(train[0])/batch_size
if saveFreq == -1:
saveFreq = len(train[0])/batch_size
if sampleFreq == -1:
sampleFreq = len(train[0])/batch_size
# Training loop
uidx = 0
estop = False
bad_counter = 0
for eidx in xrange(max_epochs):
n_samples = 0
for x in train:
n_samples += len(x)
uidx += 1
use_noise.set_value(1.)
# pad batch and create mask
x, x_mask = prepare_data(x, maxlen=maxlen, n_words=n_words)
if x is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
# compute cost, grads and copy grads to shared variables
cost = f_grad_shared(x, x_mask)
# do the update on parameters
f_update(lrate)
ud = time.time() - ud_start
# check for bad numbers
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
return 1.
# verbose
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud
# save the best model so far
if numpy.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None:
params = best_p
else:
params = unzip(tparams)
numpy.savez(saveto, history_errs=history_errs, **params)
pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'))
print 'Done'
# generate some samples with the model and display them
if numpy.mod(uidx, sampleFreq) == 0:
# FIXME: random selection?
for jj in xrange(5):
sample, score = gen_sample(tparams, f_next,
model_options, trng=trng,
maxlen=30, argmax=False)
print 'Sample ', jj, ': ',
ss = sample
for vv in ss:
if vv == 0:
break
if vv in worddicts_r:
print worddicts_r[vv],
else:
print 'UNK',
print
# validate model on validation set and early stop if necessary
if numpy.mod(uidx, validFreq) == 0:
use_noise.set_value(0.)
valid_errs = pred_probs(f_log_probs, prepare_data,
model_options, valid)
valid_err = valid_errs.mean()
history_errs.append(valid_err)
if uidx == 0 or valid_err <= numpy.array(history_errs).min():
best_p = unzip(tparams)
bad_counter = 0
if len(history_errs) > patience and valid_err >= \
numpy.array(history_errs)[:-patience].min():
bad_counter += 1
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if numpy.isnan(valid_err):
ipdb.set_trace()
print 'Valid ', valid_err
# finish after this many updates
if uidx >= finish_after:
print 'Finishing after %d iterations!' % uidx
estop = True
break
print 'Seen %d samples' % n_samples
if estop:
break
if best_p is not None:
zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = pred_probs(f_log_probs, prepare_data,
model_options, valid).mean()
print 'Valid ', valid_err
params = copy.copy(best_p)
numpy.savez(saveto, zipped_params=best_p,
history_errs=history_errs,
**params)
return valid_err
def train(
random_seed=1234,
dim_word=256, # word vector dimensionality
ctx_dim=-1, # context vector dimensionality, auto set
dim=1000, # the number of LSTM units
n_layers_out=1,
n_layers_init=1,
encoder='none',
encoder_dim=100,
prev2out=False,
ctx2out=False,
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
alpha_entropy_r=0.,
lrate=0.01,
selector=False,
n_words=100000,
maxlen=100, # maximum length of the description
optimizer='adadelta',
clip_c=2.,
batch_size=64,
valid_batch_size=64,
save_model_dir='/data/lisatmp3/yaoli/exp/capgen_vid/attention/test/',
validFreq=10,
saveFreq=10, # save the parameters after every saveFreq updates
sampleFreq=10, # generate some samples after every sampleFreq updates
metric='blue',
dataset='youtube2text',
video_feature='googlenet',
use_dropout=False,
reload_=False,
from_dir=None,
K=10,
OutOf=240,
verbose=True,
debug=True):
rng_numpy, rng_theano = utils.get_two_rngs()
model_options = locals().copy()
if 'self' in model_options:
del model_options['self']
with open('%smodel_options.pkl' % save_model_dir, 'wb') as f:
pkl.dump(model_options, f)
# instance model
layers = Layers()
model = Model()
print 'Loading data'
engine = data_engine.Movie2Caption('attention', dataset, video_feature,
batch_size, valid_batch_size, maxlen,
n_words, K, OutOf)
model_options['ctx_dim'] = engine.ctx_dim
model_options['n_words'] = engine.n_words
print 'n_words:', model_options['n_words']
# set test values, for debugging
idx = engine.kf_train[0]
[x_tv, mask_tv, ctx_tv, ctx_mask_tv
] = data_engine.prepare_data(engine,
[engine.train[index] for index in idx])
print 'init params'
t0 = time.time()
params = model.init_params(model_options)
# reloading
if reload_:
model_saved = from_dir + '/model_best_so_far.npz'
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = utils.load_params(model_saved, params)
tparams = utils.init_tparams(params)
trng, use_noise, \
x, mask, ctx, mask_ctx, \
cost, extra = \
model.build_model(tparams, model_options)
alphas = extra[1]
betas = extra[2]
print 'buliding sampler'
f_init, f_next = model.build_sampler(tparams, model_options, use_noise,
trng)
# before any regularizer
print 'building f_log_probs'
f_log_probs = theano.function([x, mask, ctx, mask_ctx],
-cost,
profile=False,
on_unused_input='ignore')
cost = cost.mean()
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
if alpha_c > 0.:
alpha_c = theano.shared(numpy.float32(alpha_c), name='alpha_c')
alpha_reg = alpha_c * ((1. - alphas.sum(0))**2).sum(-1).mean()
cost += alpha_reg
if alpha_entropy_r > 0:
alpha_entropy_r = theano.shared(numpy.float32(alpha_entropy_r),
name='alpha_entropy_r')
alpha_reg_2 = alpha_entropy_r * (-tensor.sum(
alphas * tensor.log(alphas + 1e-8), axis=-1)).sum(-1).mean()
cost += alpha_reg_2
else:
alpha_reg_2 = tensor.zeros_like(cost)
print 'building f_alpha'
f_alpha = theano.function([x, mask, ctx, mask_ctx], [alphas, betas],
name='f_alpha',
on_unused_input='ignore')
print 'compute grad'
grads = tensor.grad(cost, wrt=utils.itemlist(tparams))
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(
tensor.switch(g2 > (clip_c**2), g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
lr = tensor.scalar(name='lr')
print 'build train fns'
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads,
[x, mask, ctx, mask_ctx], cost,
extra + grads)
print 'compilation took %.4f sec' % (time.time() - t0)
print 'Optimization'
history_errs = []
# reload history
if reload_:
print 'loading history error...'
history_errs = numpy.load(
from_dir + 'model_best_so_far.npz')['history_errs'].tolist()
bad_counter = 0
processes = None
queue = None
rqueue = None
shared_params = None
uidx = 0
uidx_best_blue = 0
uidx_best_valid_err = 0
estop = False
best_p = utils.unzip(tparams)
best_blue_valid = 0
best_valid_err = 999
alphas_ratio = []
for eidx in xrange(max_epochs):
n_samples = 0
train_costs = []
grads_record = []
print 'Epoch ', eidx
for idx in engine.kf_train:
tags = [engine.train[index] for index in idx]
n_samples += len(tags)
uidx += 1
use_noise.set_value(1.)
pd_start = time.time()
x, mask, ctx, ctx_mask = data_engine.prepare_data(engine, tags)
pd_duration = time.time() - pd_start
if x is None:
print 'Minibatch with zero sample under length ', maxlen
continue
ud_start = time.time()
rvals = f_grad_shared(x, mask, ctx, ctx_mask)
cost = rvals[0]
probs = rvals[1]
alphas = rvals[2]
betas = rvals[3]
grads = rvals[4:]
grads, NaN_keys = utils.grad_nan_report(grads, tparams)
if len(grads_record) >= 5:
del grads_record[0]
grads_record.append(grads)
if NaN_keys != []:
print 'grads contain NaN'
import pdb
pdb.set_trace()
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected in cost'
import pdb
pdb.set_trace()
# update params
f_update(lrate)
ud_duration = time.time() - ud_start
if eidx == 0:
train_error = cost
else:
train_error = train_error * 0.95 + cost * 0.05
train_costs.append(cost)
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Train cost mean so far', \
train_error, 'fetching data time spent (sec)', pd_duration, \
'update time spent (sec)', ud_duration, 'save_dir', save_model_dir
alphas, betas = f_alpha(x, mask, ctx, ctx_mask)
counts = mask.sum(0)
betas_mean = (betas * mask).sum(0) / counts
betas_mean = betas_mean.mean()
print 'alpha ratio %.3f, betas mean %.3f' % (
alphas.min(-1).mean() /
(alphas.max(-1)).mean(), betas_mean)
l = 0
for vv in x[:, 0]:
if vv == 0:
break
if vv in engine.word_idict:
print '(', numpy.round(betas[l, 0],
3), ')', engine.word_idict[vv],
else:
print '(', numpy.round(betas[l, 0], 3), ')', 'UNK',
l += 1
print '(', numpy.round(betas[l, 0], 3), ')'
if numpy.mod(uidx, saveFreq) == 0:
pass
if numpy.mod(uidx, sampleFreq) == 0:
use_noise.set_value(0.)
print '------------- sampling from train ----------'
x_s = x
mask_s = mask
ctx_s = ctx
ctx_mask_s = ctx_mask
model.sample_execute(engine, model_options, tparams, f_init,
f_next, x_s, ctx_s, ctx_mask_s, trng)
print '------------- sampling from valid ----------'
idx = engine.kf_valid[numpy.random.randint(
1,
len(engine.kf_valid) - 1)]
tags = [engine.valid[index] for index in idx]
x_s, mask_s, ctx_s, mask_ctx_s = data_engine.prepare_data(
engine, tags)
model.sample_execute(engine, model_options, tparams, f_init,
f_next, x_s, ctx_s, mask_ctx_s, trng)
if validFreq != -1 and numpy.mod(uidx, validFreq) == 0:
t0_valid = time.time()
alphas, _ = f_alpha(x, mask, ctx, ctx_mask)
ratio = alphas.min(-1).mean() / (alphas.max(-1)).mean()
alphas_ratio.append(ratio)
numpy.savetxt(save_model_dir + 'alpha_ratio.txt', alphas_ratio)
current_params = utils.unzip(tparams)
numpy.savez(save_model_dir + 'model_current.npz',
history_errs=history_errs,
**current_params)
use_noise.set_value(0.)
train_err = -1
train_perp = -1
valid_err = -1
valid_perp = -1
test_err = -1
test_perp = -1
if not debug:
# first compute train cost
if 0:
print 'computing cost on trainset'
train_err, train_perp = model.pred_probs(
engine,
'train',
f_log_probs,
verbose=model_options['verbose'])
else:
train_err = 0.
train_perp = 0.
if 1:
print 'validating...'
valid_err, valid_perp = model.pred_probs(
engine,
'valid',
f_log_probs,
verbose=model_options['verbose'],
)
else:
valid_err = 0.
valid_perp = 0.
if 1:
print 'testing...'
test_err, test_perp = model.pred_probs(
engine,
'test',
f_log_probs,
verbose=model_options['verbose'])
else:
test_err = 0.
test_perp = 0.
mean_ranking = 0
blue_t0 = time.time()
scores, processes, queue, rqueue, shared_params = \
metrics.compute_score(
model_type='attention',
model_archive=current_params,
options=model_options,
engine=engine,
save_dir=save_model_dir,
beam=5, n_process=5,
whichset='both',
on_cpu=False,
processes=processes, queue=queue, rqueue=rqueue,
shared_params=shared_params, metric=metric,
one_time=False,
f_init=f_init, f_next=f_next, model=model
)
'''
{'blue': {'test': [-1], 'valid': [77.7, 60.5, 48.7, 38.5, 38.3]},
'alternative_valid': {'Bleu_3': 0.40702270203174923,
'Bleu_4': 0.29276570520368456,
'CIDEr': 0.25247168210607884,
'Bleu_2': 0.529069629270047,
'Bleu_1': 0.6804308797115253,
'ROUGE_L': 0.51083584331688392},
'meteor': {'test': [-1], 'valid': [0.282787550236724]}}
'''
valid_B1 = scores['valid']['Bleu_1']
valid_B2 = scores['valid']['Bleu_2']
valid_B3 = scores['valid']['Bleu_3']
valid_B4 = scores['valid']['Bleu_4']
valid_Rouge = scores['valid']['ROUGE_L']
valid_Cider = scores['valid']['CIDEr']
valid_meteor = scores['valid']['METEOR']
test_B1 = scores['test']['Bleu_1']
test_B2 = scores['test']['Bleu_2']
test_B3 = scores['test']['Bleu_3']
test_B4 = scores['test']['Bleu_4']
test_Rouge = scores['test']['ROUGE_L']
test_Cider = scores['test']['CIDEr']
test_meteor = scores['test']['METEOR']
print 'computing meteor/blue score used %.4f sec, '\
'blue score: %.1f, meteor score: %.1f'%(
time.time()-blue_t0, valid_B4, valid_meteor)
history_errs.append([
eidx, uidx, train_err, train_perp, valid_perp, test_perp,
valid_err, test_err, valid_B1, valid_B2, valid_B3,
valid_B4, valid_meteor, valid_Rouge, valid_Cider, test_B1,
test_B2, test_B3, test_B4, test_meteor, test_Rouge,
test_Cider
])
numpy.savetxt(save_model_dir + 'train_valid_test.txt',
history_errs,
fmt='%.3f')
print 'save validation results to %s' % save_model_dir
# save best model according to the best blue or meteor
if len(history_errs) > 1 and \
valid_B4 > numpy.array(history_errs)[:-1,11].max():
print 'Saving to %s...' % save_model_dir,
numpy.savez(save_model_dir +
'model_best_blue_or_meteor.npz',
history_errs=history_errs,
**best_p)
if len(history_errs) > 1 and \
valid_err < numpy.array(history_errs)[:-1,6].min():
best_p = utils.unzip(tparams)
bad_counter = 0
best_valid_err = valid_err
uidx_best_valid_err = uidx
print 'Saving to %s...' % save_model_dir,
numpy.savez(save_model_dir + 'model_best_so_far.npz',
history_errs=history_errs,
**best_p)
with open('%smodel_options.pkl' % save_model_dir,
'wb') as f:
pkl.dump(model_options, f)
print 'Done'
elif len(history_errs) > 1 and \
valid_err >= numpy.array(history_errs)[:-1,6].min():
bad_counter += 1
print 'history best ', numpy.array(history_errs)[:,
6].min()
print 'bad_counter ', bad_counter
print 'patience ', patience
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if test_B4 > 0.52 and test_meteor > 0.32:
print 'Saving to %s...' % save_model_dir,
numpy.savez(save_model_dir + 'model_' + str(uidx) + '.npz',
history_errs=history_errs,
**current_params)
print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err, \
'best valid err so far',best_valid_err
print 'valid took %.2f sec' % (time.time() - t0_valid)
# end of validatioin
if debug:
break
if estop:
break
if debug:
break
# end for loop over minibatches
print 'This epoch has seen %d samples, train cost %.2f' % (
n_samples, numpy.mean(train_costs))
# end for loop over epochs
print 'Optimization ended.'
if best_p is not None:
utils.zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = 0
test_err = 0
if not debug:
#if valid:
valid_err, valid_perp = model.pred_probs(
engine, 'valid', f_log_probs, verbose=model_options['verbose'])
#if test:
#test_err, test_perp = self.pred_probs(
# 'test', f_log_probs,
# verbose=model_options['verbose'])
print 'stopped at epoch %d, minibatch %d, '\
'curent Train %.2f, current Valid %.2f, current Test %.2f '%(
eidx,uidx,numpy.mean(train_err),numpy.mean(valid_err),numpy.mean(test_err))
params = copy.copy(best_p)
numpy.savez(save_model_dir + 'model_best.npz',
train_err=train_err,
valid_err=valid_err,
test_err=test_err,
history_errs=history_errs,
**params)
if history_errs != []:
history = numpy.asarray(history_errs)
best_valid_idx = history[:, 6].argmin()
numpy.savetxt(save_model_dir + 'train_valid_test.txt',
history,
fmt='%.4f')
print 'final best exp ', history[best_valid_idx]
return train_err, valid_err, test_err
def train(dim_word=100, # word vector dimensionality
dim=1000, # the number of GRU units
encoder='gru',
patience=10, # early stopping patience
max_epochs=5000,
finish_after=10000000, # finish after this many updates
dispFreq=100,
decay_c=0., # L2 weight decay penalty
lrate=0.01,
n_words=100000, # vocabulary size
vocab_dim=100000, # Size of M, C
memory_dim=1000, # Dimension of memory
memory_size=15, # n_back to attend
maxlen=100, # maximum length of the description
optimizer='rmsprop',
batch_size=16,
valid_batch_size=16,
saveto='model.npz',
validFreq=1000,
saveFreq=1000, # save the parameters after every saveFreq updates
sampleFreq=100, # generate some samples after every sampleFreq
dataset='/data/lisatmp3/chokyun/wikipedia/extracted/wiki.tok.txt.gz',
valid_dataset='../data/dev/newstest2011.en.tok',
dictionary='/data/lisatmp3/chokyun/wikipedia/extracted/'
'wiki.tok.txt.gz.pkl',
use_dropout=False,
reload_=False):
# Model options
model_options = locals().copy()
# Theano random stream
trng = RandomStreams(1234)
# load dictionary
with open(dictionary, 'rb') as f:
worddicts = pkl.load(f)
# invert dictionary
worddicts_r = dict()
for kk, vv in worddicts.iteritems():
worddicts_r[vv] = kk
# reload options
if reload_ and os.path.exists(saveto):
with open('%s.pkl' % saveto, 'rb') as f:
model_options = pkl.load(f)
print 'Loading data'
train = TextIterator(dataset,
dictionary,
n_words_source=n_words,
batch_size=batch_size,
maxlen=maxlen)
valid = TextIterator(valid_dataset,
dictionary,
n_words_source=n_words,
batch_size=valid_batch_size,
maxlen=maxlen)
# initialize RMN
rmn_ = RMN(model_options)
print 'Building model'
rmn_.init_params()
# reload parameters
if reload_ and os.path.exists(saveto):
rmn_.load_params(saveto)
# create shared variables for parameters
tparams = rmn_.tparams
# build the symbolic computational graph
use_noise, x, x_mask, opt_ret, cost = rmn_.build_model()
inps = [x, x_mask]
print 'Buliding sampler'
f_next = rmn_.build_sampler(trng)
# before any regularizer
print 'Building f_log_probs...',
f_log_probs = theano.function(inps, cost, profile=profile)
print 'Done'
cost = cost.mean()
# apply L2 regularization on weights
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
# after any regularizer - compile the computational graph for cost
print 'Building f_cost...',
f_cost = theano.function(inps, cost, profile=profile)
print 'Done'
print 'Computing gradient...',
grads = tensor.grad(cost, wrt=itemlist(tparams))
print 'Done'
# compile the optimizer, the actual computational graph is compiled here
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
optimizer = getattr(importlib.import_module('optimizer'), optimizer)
f_grad_shared, f_update = optimizer(lr, tparams, grads, inps, cost)
print 'Done'
print 'Optimization'
history_errs = []
uidx = 0
estop = False
bad_counter = 0
# reload history
if reload_ and os.path.exists(saveto):
history_errs = list(numpy.load(saveto)['history_errs'])
uidx = numpy.load(saveto)['uidx']
best_p = None
if validFreq == -1:
validFreq = len(train[0])/batch_size
if saveFreq == -1:
saveFreq = len(train[0])/batch_size
if sampleFreq == -1:
sampleFreq = len(train[0])/batch_size
# Training loop
for eidx in xrange(max_epochs):
n_samples = 0
for x in train:
n_samples += len(x)
uidx += 1
use_noise.set_value(1.)
# pad batch and create mask
x, x_mask = prepare_data(x, maxlen=maxlen, n_words=n_words)
if x is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
# compute cost, grads and copy grads to shared variables
cost = f_grad_shared(x, x_mask)
# do the update on parameters
f_update(lrate)
ud = time.time() - ud_start
# check for bad numbers
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
return 1.
# verbose
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud
# save the best model so far
if numpy.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None:
params = best_p
else:
params = unzip(tparams)
numpy.savez(saveto, history_errs=history_errs, uidx=uidx,
**params)
pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'))
print 'Done'
# generate some samples with the model and display them
if numpy.mod(uidx, sampleFreq) == 0:
# FIXME: random selection?
for jj in xrange(5):
sample, score = rmn_.gen_sample(tparams, f_next,
trng=trng, maxlen=30,
argmax=False)
print 'Sample ', jj, ': ',
ss = sample
for vv in ss:
if vv == 0:
break
if vv in worddicts_r:
print worddicts_r[vv],
else:
print 'UNK',
print
# validate model on validation set and early stop if necessary
if numpy.mod(uidx, validFreq) == 0:
use_noise.set_value(0.)
valid_errs = rmn_.pred_probs(valid, f_log_probs, prepare_data)
valid_err = valid_errs.mean()
history_errs.append(valid_err)
if uidx == 0 or valid_err <= numpy.array(history_errs).min():
best_p = unzip(tparams)
bad_counter = 0
if len(history_errs) > patience and valid_err >= \
numpy.array(history_errs)[:-patience].min():
bad_counter += 1
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if numpy.isnan(valid_err):
ipdb.set_trace()
print 'Valid ', valid_err
# finish after this many updates
if uidx >= finish_after:
print 'Finishing after %d iterations!' % uidx
estop = True
break
print 'Seen %d samples' % n_samples
if estop:
break
if best_p is not None:
zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = rmn_.pred_probs(f_log_probs, prepare_data,
model_options, valid).mean()
print 'Valid ', valid_err
params = copy.copy(best_p)
numpy.savez(saveto, zipped_params=best_p,
history_errs=history_errs,
uidx=uidx,
**params)
return valid_err
def train_model(
dim_proj=128, # word embeding dimension and LSTM number of hidden units.
patience=10, # Number of epoch to wait before early stop if no progress
max_epochs=100, # The maximum number of epoch to run
dispFreq=500, # Display to stdout the training progress every N updates
decay_c=0., # Weight decay for the classifier applied to the U weights.
lrate=0.0001, # Learning rate for sgd (not used for adadelta and rmsprop)
n_words=10000, # Vocabulary size
optimizer='adadelta',
encoder='lstm',
rnnshare=True,
bidir=False,
saveto=None, # The best model will be saved there
validFreq=370, # Compute the validation error after this number of update.
saveFreq=1110, # Save the parameters after every saveFreq updates
maxlen=100, # Sequence longer then this get ignored
batch_size=16, # The batch size during training.
valid_batch_size=64, # The batch size used for validation/test set.
dataset='imdb',
W=None, # embeddings
deep=0, # number of layers above
rnnlayer=0, # number of rnn layers
filter_hs=[3, 4, 5], #filter's width
feature_maps=100, #number of filters
pool_type='max', #pooling type
combine=False,
init='uniform',
salstm=False,
noise_std=0.,
dropout_penul=0.5,
reload_model=None, # Path to a saved model we want to start from.
data_loader=None,
fname='',
):
# Model options
optimizer = optims[optimizer]
model_options = locals().copy()
#print "model options", model_options
# Load data
(load_data, prepare_data) = data_loader
print 'Loading', dataset, 'data'
train, valid, test = load_data()
ydim = numpy.max(train[1]) + 1
model_options['ydim'] = ydim
print 'Building model'
# This create the initial parameters as numpy ndarrays.
# Dict name (string) -> numpy ndarray
params = init_params(model_options)
if reload_model:
load_params(reload_model, params)
# This create Theano Shared Variable from the parameters.
# Dict name (string) -> Theano Tensor Shared Variable
# params and tparams have different copy of the weights.
tparams = init_tparams(params)
# use_noise is for dropout
(use_noise, x, mask, y, f_pred_prob, f_pred,
cost) = build_model(tparams, model_options, SEED)
if decay_c > 0.:
decay_c = theano.shared(numpy_floatX(decay_c), name='decay_c')
weight_decay = 0.
weight_decay += (tparams['U']**2).sum()
if model_options['encoder'] == 'lstm':
for layer in range(model_options['deep']):
weight_decay += (tparams['U' + str(layer + 1)]**2).sum()
elif model_options['encoder'] == 'cnnlstm':
for filter_h in model_options['filter_hs']:
weight_decay += (tparams['cnn_f' + str(filter_h)]**2).sum()
weight_decay *= decay_c
cost += weight_decay
f_cost = theano.function([x, mask, y], cost, name='f_cost')
grads = tensor.grad(cost, wrt=tparams.values())
f_grad = theano.function([x, mask, y], grads, name='f_grad')
lr = tensor.scalar(name='lr')
f_grad_shared, f_update = optimizer(lr, tparams, grads, x, mask, y, cost)
print 'Optimization'
kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size)
kf_test = get_minibatches_idx(len(test[0]), valid_batch_size)
print "%d train examples" % len(train[0])
print "%d valid examples" % len(valid[0])
print "%d test examples" % len(test[0])
history_errs = []
best_p = None
bad_count = 0
if validFreq == -1:
validFreq = len(train[0]) / batch_size
if saveFreq == -1:
saveFreq = len(train[0]) / batch_size
uidx = 0 # the number of update done
estop = False # early stop
start_time = time.time()
try:
for eidx in xrange(max_epochs):
n_samples = 0
# Get new shuffled index for the training set.
kf = get_minibatches_idx(len(train[0]), batch_size, shuffle=True)
for _, train_index in kf:
uidx += 1
use_noise.set_value(1.)
# Select the random examples for this minibatch
y = [train[1][t] for t in train_index]
x = [train[0][t] for t in train_index]
# Get the data in numpy.ndarray format
# This swap the axis!
# Return something of shape (minibatch maxlen, n samples)
x, mask, y = prepare_data(x, y)
n_samples += x.shape[1]
cost = f_grad_shared(x, mask, y)
f_update(lrate)
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
return 1., 1., 1.
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost
if saveto and numpy.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None:
params = best_p
else:
params = unzip(tparams)
numpy.savez(saveto, history_errs=history_errs, **params)
pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'), -1)
print 'Done'
if numpy.mod(uidx, validFreq) == 0:
use_noise.set_value(0.)
train_err = pred_error(f_pred, prepare_data, train, kf)
valid_err = pred_error(f_pred, prepare_data, valid,
kf_valid)
test_err = pred_error(f_pred, prepare_data, test, kf_test)
history_errs.append([valid_err, test_err])
if (uidx == 0 or valid_err <=
numpy.array(history_errs)[:, 0].min()):
best_p = unzip(tparams)
bad_counter = 0
print('Train ', train_err, 'Valid ', valid_err, 'Test ',
test_err)
if ((len(history_errs) > patience and valid_err >=
numpy.array(history_errs)[:-patience, 0].min())):
bad_counter += 1
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
print 'Seen %d samples' % n_samples
if estop:
break
except KeyboardInterrupt:
print "Training interupted"
end_time = time.time()
if best_p is not None:
zipp(best_p, tparams)
else:
best_p = unzip(tparams)
use_noise.set_value(0.)
kf_train_sorted = get_minibatches_idx(len(train[0]), batch_size)
train_err = pred_error(f_pred, prepare_data, train, kf_train_sorted)
valid_err = pred_error(f_pred, prepare_data, valid, kf_valid)
test_err = pred_error(f_pred,
prepare_data,
test,
kf_test,
fname=fname,
verbose=True)
print 'Train Error', train_err, 'Valid Error', valid_err, 'Test Error', test_err
if saveto:
numpy.savez(saveto,
train_err=train_err,
valid_err=valid_err,
test_err=test_err,
history_errs=history_errs,
**best_p)
print 'The code run for %d epochs, with %f sec/epochs' % (
(eidx + 1), (end_time - start_time) / (1. * (eidx + 1)))
print >> sys.stderr, ('Training took %.1fs' % (end_time - start_time))
return train_err, valid_err, test_err
def train(
dim_word_desc=400, # word vector dimensionality
dim_word_q=400,
dim_word_ans=600,
dim_proj=300,
dim=400, # the number of LSTM units
encoder_desc='lstm',
encoder_desc_word='lstm',
encoder_desc_sent='lstm',
use_dq_sims=False,
eyem=None,
learn_h0=False,
use_desc_skip_c_g=False,
debug=False,
encoder_q='lstm',
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
clip_c=-1.,
lrate=0.01,
n_words_q=49145,
n_words_desc=115425,
n_words_ans=409,
pkl_train_files=None,
pkl_valid_files=None,
maxlen=2000, # maximum length of the description
optimizer='rmsprop',
batch_size=2,
vocab=None,
valid_batch_size=16,
use_elu_g=False,
saveto='model.npz',
model_dir=None,
ms_nlayers=3,
validFreq=1000,
saveFreq=1000, # save the parameters after every saveFreq updates
datasets=[None],
truncate=400,
momentum=0.9,
use_bidir=False,
cost_mask=None,
valid_datasets=[
'/u/yyu/stor/caglar/rc-data/cnn/cnn_test_data.h5',
'/u/yyu/stor/caglar/rc-data/cnn/cnn_valid_data.h5'
],
dropout_rate=0.5,
use_dropout=True,
reload_=True,
**opt_ds):
ensure_dir_exists(model_dir)
mpath = os.path.join(model_dir, saveto)
mpath_best = os.path.join(model_dir, prfx("best", saveto))
mpath_last = os.path.join(model_dir, prfx("last", saveto))
mpath_stats = os.path.join(model_dir, prfx("stats", saveto))
# Model options
model_options = locals().copy()
model_options['use_sent_reps'] = opt_ds['use_sent_reps']
stats = defaultdict(list)
del model_options['eyem']
del model_options['cost_mask']
if cost_mask is not None:
cost_mask = sharedX(cost_mask)
# reload options and parameters
if reload_:
print "Reloading the model."
if os.path.exists(mpath_best):
print "Reloading the best model from %s." % mpath_best
with open(os.path.join(mpath_best, '%s.pkl' % mpath_best),
'rb') as f:
models_options = pkl.load(f)
params = init_params(model_options)
params = load_params(mpath_best, params)
elif os.path.exists(mpath):
print "Reloading the model from %s." % mpath
with open(os.path.join(mpath, '%s.pkl' % mpath), 'rb') as f:
models_options = pkl.load(f)
params = init_params(model_options)
params = load_params(mpath, params)
else:
raise IOError("Couldn't open the file.")
else:
print "Couldn't reload the models initializing from scratch."
params = init_params(model_options)
if datasets[0]:
print "Short dataset", datasets[0]
print 'Loading data'
print 'Building model'
if pkl_train_files is None or pkl_valid_files is None:
train, valid, test = load_data(path=datasets[0],
valid_path=valid_datasets[0],
test_path=valid_datasets[1],
batch_size=batch_size,
**opt_ds)
else:
train, valid, test = load_pkl_data(train_file_paths=pkl_train_files,
valid_file_paths=pkl_valid_files,
batch_size=batch_size,
vocab=vocab,
eyem=eyem,
**opt_ds)
tparams = init_tparams(params)
trng, use_noise, inps_d, \
opt_ret, \
cost, errors, ent_errors, ent_derrors, probs = \
build_model(tparams,
model_options,
prepare_data if not opt_ds['use_sent_reps'] \
else prepare_data_sents,
valid,
cost_mask=cost_mask)
alphas = opt_ret['dec_alphas']
if opt_ds['use_sent_reps']:
inps = [inps_d["desc"], \
inps_d["word_mask"], \
inps_d["q"], \
inps_d['q_mask'], \
inps_d['ans'], \
inps_d['wlen'],
inps_d['slen'], inps_d['qlen'],\
inps_d['ent_mask']
]
else:
inps = [inps_d["desc"], \
inps_d["word_mask"], \
inps_d["q"], \
inps_d['q_mask'], \
inps_d['ans'], \
inps_d['wlen'], \
inps_d['qlen'], \
inps_d['ent_mask']]
outs = [cost, errors, probs, alphas]
if ent_errors:
outs += [ent_errors]
if ent_derrors:
outs += [ent_derrors]
# before any regularizer
print 'Building f_log_probs...',
f_log_probs = theano.function(inps, outs, profile=profile)
print 'Done'
# Apply weight decay on the feed-forward connections
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
if "logit" in kk or "ff" in kk:
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
# after any regularizer
print 'Computing gradient...',
grads = safe_grad(cost, itemlist(tparams))
print 'Done'
# Gradient clipping:
if clip_c > 0.:
g2 = get_norms(grads)
for p, g in grads.iteritems():
grads[p] = tensor.switch(g2 > (clip_c**2),
(g / tensor.sqrt(g2 + 1e-8)) * clip_c, g)
inps.pop()
if optimizer.lower() == "adasecant":
learning_rule = Adasecant(delta_clip=25.0,
use_adagrad=True,
grad_clip=0.25,
gamma_clip=0.)
elif optimizer.lower() == "rmsprop":
learning_rule = RMSPropMomentum(init_momentum=momentum)
elif optimizer.lower() == "adam":
learning_rule = Adam()
elif optimizer.lower() == "adadelta":
learning_rule = AdaDelta()
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
learning_rule = None
if learning_rule:
f_grad_shared, f_update = learning_rule.get_funcs(learning_rate=lr,
grads=grads,
inp=inps,
cost=cost,
errors=errors)
else:
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, inps,
cost, errors)
print 'Done'
print 'Optimization'
history_errs = []
# reload history
if reload_ and os.path.exists(mpath):
history_errs = list(numpy.load(mpath)['history_errs'])
best_p = None
bad_count = 0
if validFreq == -1:
validFreq = len(train[0]) / batch_size
if saveFreq == -1:
saveFreq = len(train[0]) / batch_size
best_found = False
uidx = 0
estop = False
train_cost_ave, train_err_ave, \
train_gnorm_ave = reset_train_vals()
for eidx in xrange(max_epochs):
n_samples = 0
if train.done:
train.reset()
for d_, q_, a, em in train:
n_samples += len(a)
uidx += 1
use_noise.set_value(1.)
if opt_ds['use_sent_reps']:
# To mask the description and the question.
d, d_mask, q, q_mask, dlen, slen, qlen = prepare_data_sents(
d_, q_)
if d is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
cost, errors, gnorm, pnorm = f_grad_shared(
d, d_mask, q, q_mask, a, dlen, slen, qlen)
else:
d, d_mask, q, q_mask, dlen, qlen = prepare_data(d_, q_)
if d is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
cost, errors, gnorm, pnorm = f_grad_shared(
d, d_mask, q, q_mask, a, dlen, qlen)
upnorm = f_update(lrate)
ud = time.time() - ud_start
# Collect the running ave train stats.
train_cost_ave = running_ave(train_cost_ave, cost)
train_err_ave = running_ave(train_err_ave, errors)
train_gnorm_ave = running_ave(train_gnorm_ave, gnorm)
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
import ipdb
ipdb.set_trace()
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, ' Update ', uidx, \
' Cost ', cost, ' UD ', ud, \
' UpNorm ', upnorm[0].tolist(), \
' GNorm ', gnorm, \
' Pnorm ', pnorm, 'Terrors ', errors
if numpy.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None and best_found:
numpy.savez(mpath_best,
history_errs=history_errs,
**best_p)
pkl.dump(model_options, open('%s.pkl' % mpath_best, 'wb'))
else:
params = unzip(tparams)
numpy.savez(mpath, history_errs=history_errs, **params)
pkl.dump(model_options, open('%s.pkl' % mpath, 'wb'))
pkl.dump(stats, open("%s.pkl" % mpath_stats, 'wb'))
print 'Done'
print_param_norms(tparams)
if numpy.mod(uidx, validFreq) == 0:
use_noise.set_value(0.)
if valid.done:
valid.reset()
valid_costs, valid_errs, valid_probs, \
valid_alphas, error_ent, error_dent = eval_model(f_log_probs,
prepare_data if not opt_ds['use_sent_reps'] \
else prepare_data_sents,
model_options,
valid,
use_sent_rep=opt_ds['use_sent_reps'])
valid_alphas_ = numpy.concatenate(
[va.argmax(0) for va in valid_alphas.tolist()], axis=0)
valid_err = valid_errs.mean()
valid_cost = valid_costs.mean()
valid_alpha_ent = -negentropy(valid_alphas)
mean_valid_alphas = valid_alphas_.mean()
std_valid_alphas = valid_alphas_.std()
mean_valid_probs = valid_probs.argmax(1).mean()
std_valid_probs = valid_probs.argmax(1).std()
history_errs.append([valid_cost, valid_err])
stats['train_err_ave'].append(train_err_ave)
stats['train_cost_ave'].append(train_cost_ave)
stats['train_gnorm_ave'].append(train_gnorm_ave)
stats['valid_errs'].append(valid_err)
stats['valid_costs'].append(valid_cost)
stats['valid_err_ent'].append(error_ent)
stats['valid_err_desc_ent'].append(error_dent)
stats['valid_alphas_mean'].append(mean_valid_alphas)
stats['valid_alphas_std'].append(std_valid_alphas)
stats['valid_alphas_ent'].append(valid_alpha_ent)
stats['valid_probs_mean'].append(mean_valid_probs)
stats['valid_probs_std'].append(std_valid_probs)
if uidx == 0 or valid_err <= numpy.array(
history_errs)[:, 1].min():
best_p = unzip(tparams)
bad_counter = 0
best_found = True
else:
bst_found = False
if numpy.isnan(valid_err):
import ipdb
ipdb.set_trace()
print "============================"
print '\t>>>Valid error: ', valid_err, \
' Valid cost: ', valid_cost
print '\t>>>Valid pred mean: ', mean_valid_probs, \
' Valid pred std: ', std_valid_probs
print '\t>>>Valid alphas mean: ', mean_valid_alphas, \
' Valid alphas std: ', std_valid_alphas, \
' Valid alpha negent: ', valid_alpha_ent, \
' Valid error ent: ', error_ent, \
' Valid error desc ent: ', error_dent
print "============================"
print "Running average train stats "
print '\t>>>Train error: ', train_err_ave, \
' Train cost: ', train_cost_ave, \
' Train grad norm: ', train_gnorm_ave
print "============================"
train_cost_ave, train_err_ave, \
train_gnorm_ave = reset_train_vals()
print 'Seen %d samples' % n_samples
if estop:
break
if best_p is not None:
zipp(best_p, tparams)
use_noise.set_value(0.)
valid.reset()
valid_cost, valid_error, valid_probs, \
valid_alphas, error_ent = eval_model(f_log_probs,
prepare_data if not opt_ds['use_sent_reps'] \
else prepare_data_sents,
model_options, valid,
use_sent_rep=opt_ds['use_sent_rep'])
print " Final eval resuts: "
print 'Valid error: ', valid_error.mean()
print 'Valid cost: ', valid_cost.mean()
print '\t>>>Valid pred mean: ', valid_probs.mean(), \
' Valid pred std: ', valid_probs.std(), \
' Valid error ent: ', error_ent
params = copy.copy(best_p)
numpy.savez(mpath_last,
zipped_params=best_p,
history_errs=history_errs,
**params)
return valid_err, valid_cost
def train(dim_word_desc=400,# word vector dimensionality
dim_word_q=400,
dim_word_ans=600,
dim_proj=300,
dim=400,# the number of LSTM units
encoder_desc='lstm',
encoder_desc_word='lstm',
encoder_desc_sent='lstm',
use_dq_sims=False,
eyem=None,
learn_h0=False,
use_desc_skip_c_g=False,
debug=False,
encoder_q='lstm',
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
clip_c=-1.,
lrate=0.01,
n_words_q=49145,
n_words_desc=115425,
n_words_ans=409,
pkl_train_files=None,
pkl_valid_files=None,
maxlen=2000, # maximum length of the description
optimizer='rmsprop',
batch_size=2,
vocab=None,
valid_batch_size=16,
use_elu_g=False,
saveto='model.npz',
model_dir=None,
ms_nlayers=3,
validFreq=1000,
saveFreq=1000, # save the parameters after every saveFreq updates
datasets=[None],
truncate=400,
momentum=0.9,
use_bidir=False,
cost_mask=None,
valid_datasets=['/u/yyu/stor/caglar/rc-data/cnn/cnn_test_data.h5',
'/u/yyu/stor/caglar/rc-data/cnn/cnn_valid_data.h5'],
dropout_rate=0.5,
use_dropout=True,
reload_=True,
**opt_ds):
ensure_dir_exists(model_dir)
mpath = os.path.join(model_dir, saveto)
mpath_best = os.path.join(model_dir, prfx("best", saveto))
mpath_last = os.path.join(model_dir, prfx("last", saveto))
mpath_stats = os.path.join(model_dir, prfx("stats", saveto))
# Model options
model_options = locals().copy()
model_options['use_sent_reps'] = opt_ds['use_sent_reps']
stats = defaultdict(list)
del model_options['eyem']
del model_options['cost_mask']
if cost_mask is not None:
cost_mask = sharedX(cost_mask)
# reload options and parameters
if reload_:
print "Reloading the model."
if os.path.exists(mpath_best):
print "Reloading the best model from %s." % mpath_best
with open(os.path.join(mpath_best, '%s.pkl' % mpath_best), 'rb') as f:
models_options = pkl.load(f)
params = init_params(model_options)
params = load_params(mpath_best, params)
elif os.path.exists(mpath):
print "Reloading the model from %s." % mpath
with open(os.path.join(mpath, '%s.pkl' % mpath), 'rb') as f:
models_options = pkl.load(f)
params = init_params(model_options)
params = load_params(mpath, params)
else:
raise IOError("Couldn't open the file.")
else:
print "Couldn't reload the models initializing from scratch."
params = init_params(model_options)
if datasets[0]:
print "Short dataset", datasets[0]
print 'Loading data'
print 'Building model'
if pkl_train_files is None or pkl_valid_files is None:
train, valid, test = load_data(path=datasets[0],
valid_path=valid_datasets[0],
test_path=valid_datasets[1],
batch_size=batch_size,
**opt_ds)
else:
train, valid, test = load_pkl_data(train_file_paths=pkl_train_files,
valid_file_paths=pkl_valid_files,
batch_size=batch_size,
vocab=vocab,
eyem=eyem,
**opt_ds)
tparams = init_tparams(params)
trng, use_noise, inps_d, \
opt_ret, \
cost, errors, ent_errors, ent_derrors, probs = \
build_model(tparams,
model_options,
prepare_data if not opt_ds['use_sent_reps'] \
else prepare_data_sents,
valid,
cost_mask=cost_mask)
alphas = opt_ret['dec_alphas']
if opt_ds['use_sent_reps']:
inps = [inps_d["desc"], \
inps_d["word_mask"], \
inps_d["q"], \
inps_d['q_mask'], \
inps_d['ans'], \
inps_d['wlen'],
inps_d['slen'], inps_d['qlen'],\
inps_d['ent_mask']
]
else:
inps = [inps_d["desc"], \
inps_d["word_mask"], \
inps_d["q"], \
inps_d['q_mask'], \
inps_d['ans'], \
inps_d['wlen'], \
inps_d['qlen'], \
inps_d['ent_mask']]
outs = [cost, errors, probs, alphas]
if ent_errors:
outs += [ent_errors]
if ent_derrors:
outs += [ent_derrors]
# before any regularizer
print 'Building f_log_probs...',
f_log_probs = theano.function(inps, outs, profile=profile)
print 'Done'
# Apply weight decay on the feed-forward connections
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
if "logit" in kk or "ff" in kk:
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
# after any regularizer
print 'Computing gradient...',
grads = safe_grad(cost, itemlist(tparams))
print 'Done'
# Gradient clipping:
if clip_c > 0.:
g2 = get_norms(grads)
for p, g in grads.iteritems():
grads[p] = tensor.switch(g2 > (clip_c**2),
(g / tensor.sqrt(g2 + 1e-8)) * clip_c,
g)
inps.pop()
if optimizer.lower() == "adasecant":
learning_rule = Adasecant(delta_clip=25.0,
use_adagrad=True,
grad_clip=0.25,
gamma_clip=0.)
elif optimizer.lower() == "rmsprop":
learning_rule = RMSPropMomentum(init_momentum=momentum)
elif optimizer.lower() == "adam":
learning_rule = Adam()
elif optimizer.lower() == "adadelta":
learning_rule = AdaDelta()
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
learning_rule = None
if learning_rule:
f_grad_shared, f_update = learning_rule.get_funcs(learning_rate=lr,
grads=grads,
inp=inps,
cost=cost,
errors=errors)
else:
f_grad_shared, f_update = eval(optimizer)(lr,
tparams,
grads,
inps,
cost,
errors)
print 'Done'
print 'Optimization'
history_errs = []
# reload history
if reload_ and os.path.exists(mpath):
history_errs = list(numpy.load(mpath)['history_errs'])
best_p = None
bad_count = 0
if validFreq == -1:
validFreq = len(train[0]) / batch_size
if saveFreq == -1:
saveFreq = len(train[0]) / batch_size
best_found = False
uidx = 0
estop = False
train_cost_ave, train_err_ave, \
train_gnorm_ave = reset_train_vals()
for eidx in xrange(max_epochs):
n_samples = 0
if train.done:
train.reset()
for d_, q_, a, em in train:
n_samples += len(a)
uidx += 1
use_noise.set_value(1.)
if opt_ds['use_sent_reps']:
# To mask the description and the question.
d, d_mask, q, q_mask, dlen, slen, qlen = prepare_data_sents(d_,
q_)
if d is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
cost, errors, gnorm, pnorm = f_grad_shared(d,
d_mask,
q,
q_mask,
a,
dlen,
slen,
qlen)
else:
d, d_mask, q, q_mask, dlen, qlen = prepare_data(d_, q_)
if d is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
cost, errors, gnorm, pnorm = f_grad_shared(d, d_mask,
q, q_mask,
a,
dlen,
qlen)
upnorm = f_update(lrate)
ud = time.time() - ud_start
# Collect the running ave train stats.
train_cost_ave = running_ave(train_cost_ave,
cost)
train_err_ave = running_ave(train_err_ave,
errors)
train_gnorm_ave = running_ave(train_gnorm_ave,
gnorm)
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
import ipdb; ipdb.set_trace()
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, ' Update ', uidx, \
' Cost ', cost, ' UD ', ud, \
' UpNorm ', upnorm[0].tolist(), \
' GNorm ', gnorm, \
' Pnorm ', pnorm, 'Terrors ', errors
if numpy.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None and best_found:
numpy.savez(mpath_best, history_errs=history_errs, **best_p)
pkl.dump(model_options, open('%s.pkl' % mpath_best, 'wb'))
else:
params = unzip(tparams)
numpy.savez(mpath, history_errs=history_errs, **params)
pkl.dump(model_options, open('%s.pkl' % mpath, 'wb'))
pkl.dump(stats, open("%s.pkl" % mpath_stats, 'wb'))
print 'Done'
print_param_norms(tparams)
if numpy.mod(uidx, validFreq) == 0:
use_noise.set_value(0.)
if valid.done:
valid.reset()
valid_costs, valid_errs, valid_probs, \
valid_alphas, error_ent, error_dent = eval_model(f_log_probs,
prepare_data if not opt_ds['use_sent_reps'] \
else prepare_data_sents,
model_options,
valid,
use_sent_rep=opt_ds['use_sent_reps'])
valid_alphas_ = numpy.concatenate([va.argmax(0) for va in valid_alphas.tolist()], axis=0)
valid_err = valid_errs.mean()
valid_cost = valid_costs.mean()
valid_alpha_ent = -negentropy(valid_alphas)
mean_valid_alphas = valid_alphas_.mean()
std_valid_alphas = valid_alphas_.std()
mean_valid_probs = valid_probs.argmax(1).mean()
std_valid_probs = valid_probs.argmax(1).std()
history_errs.append([valid_cost, valid_err])
stats['train_err_ave'].append(train_err_ave)
stats['train_cost_ave'].append(train_cost_ave)
stats['train_gnorm_ave'].append(train_gnorm_ave)
stats['valid_errs'].append(valid_err)
stats['valid_costs'].append(valid_cost)
stats['valid_err_ent'].append(error_ent)
stats['valid_err_desc_ent'].append(error_dent)
stats['valid_alphas_mean'].append(mean_valid_alphas)
stats['valid_alphas_std'].append(std_valid_alphas)
stats['valid_alphas_ent'].append(valid_alpha_ent)
stats['valid_probs_mean'].append(mean_valid_probs)
stats['valid_probs_std'].append(std_valid_probs)
if uidx == 0 or valid_err <= numpy.array(history_errs)[:, 1].min():
best_p = unzip(tparams)
bad_counter = 0
best_found = True
else:
bst_found = False
if numpy.isnan(valid_err):
import ipdb; ipdb.set_trace()
print "============================"
print '\t>>>Valid error: ', valid_err, \
' Valid cost: ', valid_cost
print '\t>>>Valid pred mean: ', mean_valid_probs, \
' Valid pred std: ', std_valid_probs
print '\t>>>Valid alphas mean: ', mean_valid_alphas, \
' Valid alphas std: ', std_valid_alphas, \
' Valid alpha negent: ', valid_alpha_ent, \
' Valid error ent: ', error_ent, \
' Valid error desc ent: ', error_dent
print "============================"
print "Running average train stats "
print '\t>>>Train error: ', train_err_ave, \
' Train cost: ', train_cost_ave, \
' Train grad norm: ', train_gnorm_ave
print "============================"
train_cost_ave, train_err_ave, \
train_gnorm_ave = reset_train_vals()
print 'Seen %d samples' % n_samples
if estop:
break
if best_p is not None:
zipp(best_p, tparams)
use_noise.set_value(0.)
valid.reset()
valid_cost, valid_error, valid_probs, \
valid_alphas, error_ent = eval_model(f_log_probs,
prepare_data if not opt_ds['use_sent_reps'] \
else prepare_data_sents,
model_options, valid,
use_sent_rep=opt_ds['use_sent_rep'])
print " Final eval resuts: "
print 'Valid error: ', valid_error.mean()
print 'Valid cost: ', valid_cost.mean()
print '\t>>>Valid pred mean: ', valid_probs.mean(), \
' Valid pred std: ', valid_probs.std(), \
' Valid error ent: ', error_ent
params = copy.copy(best_p)
numpy.savez(mpath_last,
zipped_params=best_p,
history_errs=history_errs,
**params)
return valid_err, valid_cost
def train(random_seed=1234,
dim_word=256, # word vector dimensionality
ctx_dim=-1, # context vector dimensionality, auto set
dim=1000, # the number of LSTM units
n_layers_out=1,
n_layers_init=1,
encoder='none',
encoder_dim=100,
prev2out=False,
ctx2out=False,
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
alpha_entropy_r=0.,
lrate=0.01,
selector=False,
n_words=100000,
maxlen=100, # maximum length of the description
optimizer='adadelta',
clip_c=2.,
batch_size = 64,
valid_batch_size = 64,
save_model_dir='/data/lisatmp3/yaoli/exp/capgen_vid/attention/test/',
validFreq=10,
saveFreq=10, # save the parameters after every saveFreq updates
sampleFreq=10, # generate some samples after every sampleFreq updates
metric='blue',
dataset='youtube2text',
video_feature='googlenet',
use_dropout=False,
reload_=False,
from_dir=None,
K1=10,
K2=10,
OutOf=240,
verbose=True,
debug=True
):
rng_numpy, rng_theano = utils.get_two_rngs()
model_options = locals().copy()
model_options_c = locals().copy()
if 'self' in model_options:
del model_options['self']
with open('model_files/model_options.pkl', 'wb') as f:
pkl.dump(model_options, f)
with open('model_files/model_options_c3d.pkl', 'wb') as f:
pkl.dump(model_options_c, f)
# instance model
layers = Layers()
model = Model()
model_c = Model()
print 'Loading data'
engine = data_engine.Movie2Caption('attention', dataset,
video_feature,
batch_size, valid_batch_size,
maxlen, n_words,
K1, K2, OutOf)
model_options['ctx_dim'] = engine.ctx_dim
model_options_c['ctx_dim'] = engine.ctx_dim_c
model_options['n_words'] = engine.n_words
model_options_c['n_words'] = engine.n_words
print 'n_words:', model_options['n_words']
print model_options_c['dim'],model_options_c['ctx_dim']
# set test values, for debugging
idx = engine.kf_train[0]
[x_tv, mask_tv,
ctx_tv, ctx_mask_tv,
ctx_tv_c, ctx_mask_tv_c] = data_engine.prepare_data(
engine, [engine.train[index] for index in idx])
print 'init params'
t0 = time.time()
params = model.init_params(model_options)
params_c = model_c.init_params(model_options_c)
# reloading
model_saved = 'model_files/model_resnet.npz'
model_saved_c = 'model_files/model_c3d.npz'
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = utils.load_params(model_saved, params)
params_c = utils.load_params(model_saved_c, params_c)
tparams = utils.init_tparams(params)
tparams_c = utils.init_tparams(params_c)
trng, use_noise, \
x, mask, ctx, mask_ctx, \
cost, extra = \
model.build_model(tparams, model_options)
alphas = extra[1]
betas = extra[2]
trng_c, use_noise_c, \
x_c, mask_c, ctx_c, mask_ctx_c, \
cost_c, extra_c = \
model_c.build_model(tparams_c, model_options_c)
alphas_c = extra_c[1]
betas_c = extra_c[2]
print 'buliding sampler'
f_init, f_next = model.build_sampler(tparams, model_options, use_noise, trng)
f_init_c, f_next_c = model_c.build_sampler(tparams_c, model_options_c, use_noise_c, trng_c)
# before any regularizer
print 'building f_log_probs'
f_log_probs = theano.function([x, mask, ctx, mask_ctx], -cost,
profile=False, on_unused_input='ignore')
f_log_probs_c = theano.function([x_c, mask_c, ctx_c, mask_ctx_c], -cost_c,
profile=False, on_unused_input='ignore')
bad_counter = 0
processes = None
queue = None
rqueue = None
shared_params = None
uidx = 0
uidx_best_blue = 0
uidx_best_valid_err = 0
estop = False
best_p = utils.unzip(tparams)
best_blue_valid = 0
best_valid_err = 999
alphas_ratio = []
for eidx in xrange(max_epochs):
n_samples = 0
train_costs = []
grads_record = []
print 'Epoch ', eidx
for idx in engine.kf_train:
tags = [engine.train[index] for index in idx]
n_samples += len(tags)
use_noise.set_value(1.)
pd_start = time.time()
x, mask, ctx, ctx_mask, ctx_c, ctx_mask_c = data_engine.prepare_data(
engine, tags)
#print 'x:',x.shape,'ctx:',ctx.shape,'ctx_c:',ctx_c.shape
pd_duration = time.time() - pd_start
if x is None:
print 'Minibatch with zero sample under length ', maxlen
continue
if numpy.mod(uidx, saveFreq) == 0:
pass
if numpy.mod(uidx, sampleFreq) == 0:
use_noise.set_value(0.)
print '------------- sampling from train ----------'
x_s = x
mask_s = mask
ctx_s = ctx
ctx_s_c = ctx_c
ctx_mask_s = ctx_mask
ctx_mask_s_c = ctx_mask_c
model.sample_execute_ensemble(engine, model_options,model_options_c, tparams,tparams_c,
f_init,f_init_c, f_next,f_next_c, x_s, ctx_s,
ctx_mask_s, ctx_s_c, ctx_mask_s_c, trng)
print '------------- sampling from valid ----------'
idx = engine.kf_valid[numpy.random.randint(1, len(engine.kf_valid) - 1)]
tags = [engine.valid[index] for index in idx]
x_s, mask_s, ctx_s, mask_ctx_s, ctx_s_c,mask_ctx_s_c = data_engine.prepare_data(engine, tags)
model.sample_execute_ensemble(engine, model_options,model_options_c, tparams,tparams_c,
f_init, f_init_c, f_next, f_next_c, x_s, ctx_s,
mask_ctx_s, ctx_s_c, mask_ctx_s_c, trng)
if validFreq != -1 and numpy.mod(uidx, validFreq) == 0:
current_params = utils.unzip(tparams)
use_noise.set_value(0.)
train_err = -1
train_perp = -1
valid_err = -1
valid_perp = -1
test_err = -1
test_perp = -1
mean_ranking = 0
blue_t0 = time.time()
scores, processes, queue, rqueue, shared_params = \
metrics.compute_score_ensemble(
model_type='attention',
model_archive=current_params,
options=model_options,
options_c=model_options_c,
engine=engine,
save_dir=save_model_dir,
beam=5, n_process=5,
whichset='both',
on_cpu=False,
processes=processes, queue=queue, rqueue=rqueue,
shared_params=shared_params, metric=metric,
one_time=False,
f_init=f_init, f_init_c=f_init_c, f_next=f_next, f_next_c= f_next_c, model=model
)
'''
{'blue': {'test': [-1], 'valid': [77.7, 60.5, 48.7, 38.5, 38.3]},
'alternative_valid': {'Bleu_3': 0.40702270203174923,
'Bleu_4': 0.29276570520368456,
'CIDEr': 0.25247168210607884,
'Bleu_2': 0.529069629270047,
'Bleu_1': 0.6804308797115253,
'ROUGE_L': 0.51083584331688392},
'meteor': {'test': [-1], 'valid': [0.282787550236724]}}
'''
valid_B1 = scores['valid']['Bleu_1']
valid_B2 = scores['valid']['Bleu_2']
valid_B3 = scores['valid']['Bleu_3']
valid_B4 = scores['valid']['Bleu_4']
valid_Rouge = scores['valid']['ROUGE_L']
valid_Cider = scores['valid']['CIDEr']
valid_meteor = scores['valid']['METEOR']
test_B1 = scores['test']['Bleu_1']
test_B2 = scores['test']['Bleu_2']
test_B3 = scores['test']['Bleu_3']
test_B4 = scores['test']['Bleu_4']
test_Rouge = scores['test']['ROUGE_L']
test_Cider = scores['test']['CIDEr']
test_meteor = scores['test']['METEOR']
print 'computing meteor/blue score used %.4f sec, '\
'blue score: %.1f, meteor score: %.1f'%(
time.time()-blue_t0, valid_B4, valid_meteor)
if test_B4>0.52 and test_meteor>0.32:
print 'Saving to %s...'%save_model_dir,
numpy.savez(
save_model_dir+'model_'+str(uidx)+'.npz',
**current_params)
print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err, \
'best valid err so far',best_valid_err
print 'valid took %.2f sec'%(time.time() - t0_valid)
# end of validatioin
sys.exit()
if debug:
break
if estop:
break
if debug:
break
# end for loop over minibatches
print 'This epoch has seen %d samples, train cost %.2f'%(
n_samples, numpy.mean(train_costs))
# end for loop over epochs
print 'Optimization ended.'
if best_p is not None:
utils.zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = 0
test_err = 0
if not debug:
#if valid:
valid_err, valid_perp = model.pred_probs(
engine, 'valid', f_log_probs,
verbose=model_options['verbose'])
#if test:
#test_err, test_perp = self.pred_probs(
# 'test', f_log_probs,
# verbose=model_options['verbose'])
print 'stopped at epoch %d, minibatch %d, '\
'curent Train %.2f, current Valid %.2f, current Test %.2f '%(
eidx,uidx,numpy.mean(train_err),numpy.mean(valid_err),numpy.mean(test_err))
params = copy.copy(best_p)
numpy.savez(save_model_dir+'model_best.npz',
train_err=train_err,
valid_err=valid_err, test_err=test_err, history_errs=history_errs,
**params)
if history_errs != []:
history = numpy.asarray(history_errs)
best_valid_idx = history[:,6].argmin()
numpy.savetxt(save_model_dir+'train_valid_test.txt', history, fmt='%.4f')
print 'final best exp ', history[best_valid_idx]
return train_err, valid_err, test_err
def train(dim_word=100, # word vector dimensionality
dim=1000, # the number of LSTM units
encoder='gru',
decoder='gru_cond',
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
diag_c=0.,
clip_c=-1.,
lrate=0.01,
n_words_src=100000,
n_words=100000,
maxlen=100, # maximum length of the description
optimizer='rmsprop',
batch_size = 16,
valid_batch_size = 16,
saveto='model.npz',
validFreq=1000,
saveFreq=1000, # save the parameters after every saveFreq updates
sampleFreq=100, # generate some samples after every sampleFreq updates
datasets=['/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.en.tok',
'/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.fr.tok'],
valid_datasets=['../data/dev/newstest2011.en.tok', '../data/dev/newstest2011.fr.tok'],
dictionaries=['/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.en.tok.pkl',
'/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.fr.tok.pkl'],
use_dropout=False,
reload_=False):
# Model options
model_options = locals().copy()
worddicts = [None] * len(dictionaries)
worddicts_r = [None] * len(dictionaries)
for ii, dd in enumerate(dictionaries):
with open(dd, 'rb') as f:
worddicts[ii] = pkl.load(f)
worddicts_r[ii] = dict()
for kk, vv in worddicts[ii].iteritems():
worddicts_r[ii][vv] = kk
# reload options
if reload_ and os.path.exists(saveto):
with open('%s.pkl' % saveto, 'rb') as f:
models_options = pkl.load(f)
print 'Loading data'
train = TextIterator(datasets[0], datasets[1],
dictionaries[0], dictionaries[1],
n_words_source=n_words_src, n_words_target=n_words,
batch_size=batch_size,
maxlen=maxlen)
valid = TextIterator(valid_datasets[0], valid_datasets[1],
dictionaries[0], dictionaries[1],
n_words_source=n_words_src, n_words_target=n_words,
batch_size=valid_batch_size,
maxlen=maxlen)
print 'Building model'
params = init_params(model_options)
# reload parameters
if reload_ and os.path.exists(saveto):
params = load_params(saveto, params)
tparams = init_tparams(params)
trng, use_noise, \
x, x_mask, y, y_mask, \
opt_ret, \
cost = \
build_model(tparams, model_options)
inps = [x, x_mask, y, y_mask]
print 'Buliding sampler'
f_init, f_next = build_sampler(tparams, model_options, trng)
# before any regularizer
print 'Building f_log_probs...',
f_log_probs = theano.function(inps, cost, profile=profile)
print 'Done'
cost = cost.mean()
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
if alpha_c > 0. and not model_options['decoder'].endswith('simple'):
alpha_c = theano.shared(numpy.float32(alpha_c), name='alpha_c')
alpha_reg = alpha_c * ((tensor.cast(y_mask.sum(0)//x_mask.sum(0), 'float32')[:,None]-
opt_ret['dec_alphas'].sum(0))**2).sum(1).mean()
cost += alpha_reg
# after any regularizer
print 'Building f_cost...',
f_cost = theano.function(inps, cost, profile=profile)
print 'Done'
print 'Computing gradient...',
grads = tensor.grad(cost, wrt=itemlist(tparams))
print 'Done'
print 'Building f_grad...',
f_grad = theano.function(inps, grads, profile=profile)
print 'Done'
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(tensor.switch(g2 > (clip_c**2),
g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, inps, cost)
print 'Done'
print 'Optimization'
history_errs = []
# reload history
if reload_ and os.path.exists(saveto):
history_errs = list(numpy.load(saveto)['history_errs'])
best_p = None
bad_count = 0
if validFreq == -1:
validFreq = len(train[0])/batch_size
if saveFreq == -1:
saveFreq = len(train[0])/batch_size
if sampleFreq == -1:
sampleFreq = len(train[0])/batch_size
uidx = 0
estop = False
for eidx in xrange(max_epochs):
n_samples = 0
for x, y in train:
n_samples += len(x)
uidx += 1
use_noise.set_value(1.)
x, x_mask, y, y_mask = prepare_data(x, y, maxlen=maxlen,
n_words_src=n_words_src,
n_words=n_words)
if x is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
cost = f_grad_shared(x, x_mask, y, y_mask)
f_update(lrate)
ud = time.time() - ud_start
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
return 1., 1., 1.
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud
if numpy.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None:
params = best_p
else:
params = unzip(tparams)
numpy.savez(saveto, history_errs=history_errs, **params)
pkl.dump(model_options, open('%s.pkl'%saveto, 'wb'))
print 'Done'
if numpy.mod(uidx, sampleFreq) == 0:
# FIXME: random selection?
for jj in xrange(numpy.minimum(5,x.shape[1])):
stochastic = True
sample, score = gen_sample(tparams, f_init, f_next, x[:,jj][:,None],
model_options, trng=trng, k=1, maxlen=30,
stochastic=stochastic, argmax=False)
print 'Source ', jj, ': ',
for vv in x[:, jj]:
if vv == 0:
break
if vv in worddicts_r[0]:
print worddicts_r[0][vv],
else:
print 'UNK',
print
print 'Truth ', jj, ' : ',
for vv in y[:, jj]:
if vv == 0:
break
if vv in worddicts_r[1]:
print worddicts_r[1][vv],
else:
print 'UNK',
print
print 'Sample ', jj, ': ',
if stochastic:
ss = sample
else:
score = score / numpy.array([len(s) for s in sample])
ss = sample[score.argmin()]
for vv in ss:
if vv == 0:
break
if vv in worddicts_r[1]:
print worddicts_r[1][vv],
else:
print 'UNK',
print
if numpy.mod(uidx, validFreq) == 0:
use_noise.set_value(0.)
valid_errs = pred_probs(f_log_probs, prepare_data, model_options, valid)
valid_err = valid_errs.mean()
history_errs.append(valid_err)
if uidx == 0 or valid_err <= numpy.array(history_errs).min():
best_p = unzip(tparams)
bad_counter = 0
if len(history_errs) > patience and valid_err >= numpy.array(history_errs)[:-patience].min():
bad_counter += 1
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if numpy.isnan(valid_err):
import ipdb; ipdb.set_trace()
print 'Valid ', valid_err
print 'Seen %d samples' % n_samples
if estop:
break
if best_p is not None:
zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = pred_probs(f_log_probs, prepare_data, model_options, valid).mean()
print 'Valid ', valid_err
params = copy.copy(best_p)
numpy.savez(saveto, zipped_params=best_p,
history_errs=history_errs,
**params)
return valid_err
def train(dim_word=100, # word vector dimensionality
dim=1000, # the number of LSTM units
encoder='gru',
decoder='gru_cond',
n_words_src=30000,
n_words=30000,
patience=10, # early stopping patience
max_epochs=5000,
finish_after=10000000, # finish after this many updates
dispFreq=100,
decay_c=0., # L2 regularization penalty
alpha_c=0., # alignment regularization
clip_c=-1., # gradient clipping threshold
lrate=1., # learning rate
maxlen=100, # maximum length of the description
optimizer='rmsprop',
batch_size=16,
saveto='model.npz',
saveFreq=1000, # save the parameters after every saveFreq updates
datasets=[
'/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.en.tok',
'/data/lisatmp3/chokyun/europarl/europarl-v7.fr-en.fr.tok'],
picked_train_idxes_file=r'',
use_dropout=False,
reload_=False,
overwrite=False,
preload='',
sort_by_len=False,
convert_embedding=True,
dump_before_train=False,
):
# Model options
model_options = locals().copy()
if reload_:
lrate *= 0.5
# load dictionaries and invert them
# reload options
if reload_ and os.path.exists(preload):
print 'Reloading model options'
with open(r'.\model\en2fr.iter160000.npz.pkl', 'rb') as f:
model_options = pkl.load(f)
print 'Configuration from fy'
vocab_en_filename = './data/dic/en2fr_en_vocabs_top1M.pkl'
vocab_fr_filename = './data/dic/en2fr_fr_vocabs_top1M.pkl'
map_filename = './data/dic/mapFullVocab2Top1MVocab.pkl'
lr_discount_freq = 80000
print 'Done'
print 'Loading data'
text_iterator = TextIterator(
datasets[0],
datasets[1],
vocab_en_filename,
vocab_fr_filename,
batch_size,
maxlen,
n_words_src,
n_words,
)
# sys.stdout.flush()
# train_data_x = pkl.load(open(datasets[0], 'rb'))
# train_data_y = pkl.load(open(datasets[1], 'rb'))
#
# if len(picked_train_idxes_file) != 0:
# picked_idxes = pkl.load(open(picked_train_idxes_file, 'rb'))
# train_data_x = [train_data_x[id] for id in picked_idxes]
# train_data_y = [train_data_y[id] for id in picked_idxes]
#
# print 'Total train:', len(train_data_x)
# print 'Max len:', max([len(x) for x in train_data_x])
# sys.stdout.flush()
#
# if sort_by_len:
# slen = np.array([len(s) for s in train_data_x])
# sidx = slen.argsort()
#
# _sbuf = [train_data_x[i] for i in sidx]
# _tbuf = [train_data_y[i] for i in sidx]
#
# train_data_x = _sbuf
# train_data_y = _tbuf
# print len(train_data_x[0]), len(train_data_x[-1])
# sys.stdout.flush()
# train_batch_idx = get_minibatches_idx(len(train_data_x), batch_size, shuffle=False)
# else:
# train_batch_idx = get_minibatches_idx(len(train_data_x), batch_size, shuffle=True)
print 'Building model'
params = init_params(model_options)
# reload parameters
if reload_ and os.path.exists(preload):
print 'Reloading model parameters'
params = load_params(preload, params)
# for k, v in params.iteritems():
# print '>', k, v.shape, v.dtype
# Only convert parameters when reloading
if convert_embedding:
# =================
# Convert input and output embedding parameters with a exist word embedding
# =================
print 'Convert input and output embedding'
temp_Wemb = params['Wemb']
orig_emb_mean = np.mean(temp_Wemb, axis=0)
params['Wemb'] = np.tile(orig_emb_mean, [params['Wemb'].shape[0], 1])
# Load vocabulary map dicts and do mapping
with open(map_filename, 'rb') as map_file:
map_en = pkl.load(map_file)
map_fr = pkl.load(map_file)
for full, top in map_en.iteritems():
emb_size = temp_Wemb.shape[0]
if full < emb_size and top < emb_size:
params['Wemb'][top] = temp_Wemb[full]
print 'Convert input embedding done'
temp_ff_logit_W = params['ff_logit_W']
temp_Wemb_dec = params['Wemb_dec']
temp_b = params['ff_logit_b']
orig_ff_logit_W_mean = np.mean(temp_ff_logit_W, axis=1)
orig_Wemb_dec_mean = np.mean(temp_Wemb_dec, axis=0)
orig_b_mean = np.mean(temp_b)
params['ff_logit_W'] = np.tile(orig_ff_logit_W_mean, [params['ff_logit_W'].shape[1], 1]).T
params['ff_logit_b'].fill(orig_b_mean)
params['Wemb_dec'] = np.tile(orig_Wemb_dec_mean, [params['Wemb_dec'].shape[0], 1])
for full, top in map_en.iteritems():
emb_size = temp_Wemb.shape[0]
if full < emb_size and top < emb_size:
params['ff_logit_W'][:, top] = temp_ff_logit_W[:, full]
params['ff_logit_b'][top] = temp_b[full]
params['Wemb_dec'][top] = temp_Wemb[full]
print 'Convert output embedding done'
# for k, v in params.iteritems():
# print '>', k, v.shape, v.dtype
# ================
# End Convert
# ================
tparams = init_tparams(params)
trng, use_noise, \
x, x_mask, y, y_mask, \
opt_ret, \
cost, x_emb = \
build_model(tparams, model_options)
inps = [x, x_mask, y, y_mask]
print 'Building sampler'
f_init, f_next = build_sampler(tparams, model_options, trng, use_noise)
# before any regularizer
print 'Building f_log_probs...',
f_log_probs = theano.function(inps, cost, profile=profile)
f_x_emb = theano.function([x, x_mask], x_emb, profile=profile)
print 'Done'
sys.stdout.flush()
cost = cost.mean()
# apply L2 regularization on weights
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
# regularize the alpha weights
if alpha_c > 0. and not model_options['decoder'].endswith('simple'):
alpha_c = theano.shared(np.float32(alpha_c), name='alpha_c')
alpha_reg = alpha_c * (
(tensor.cast(y_mask.sum(0) // x_mask.sum(0), 'float32')[:, None] -
opt_ret['dec_alphas'].sum(0)) ** 2).sum(1).mean()
cost += alpha_reg
# after all regularizers - compile the computational graph for cost
print 'Building f_cost...',
f_cost = theano.function(inps, cost, profile=profile)
print 'Done'
print 'Computing gradient...',
grads = tensor.grad(cost, wrt=itemlist(tparams))
print 'Done'
sys.stdout.flush()
# apply gradient clipping here
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g ** 2).sum()
new_grads = []
for g in grads:
new_grads.append(tensor.switch(g2 > (clip_c ** 2),
g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
# compile the optimizer, the actual computational graph is compiled here
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, inps, cost)
print 'Done'
print 'Optimization'
best_p = None
bad_counter = 0
uidx = 0
if reload_:
m = re.search('.+iter(\d+?)\.npz', preload)
if m:
uidx = int(m.group(1))
print 'uidx', uidx, 'l_rate', lrate
estop = False
history_errs = []
# reload history
if dump_before_train:
print 'Dumping before train...',
saveto_uidx = '{}.iter{}.npz'.format(
os.path.splitext(saveto)[0], uidx)
np.savez(saveto_uidx, history_errs=history_errs,
uidx=uidx, **unzip(tparams))
print 'Done'
if saveFreq == -1:
saveFreq = len(train[0]) / batch_size
for eidx in xrange(max_epochs):
n_samples = 0
# for i, batch_idx in train_batch_idx:
#
# x = [train_data_x[id] for id in batch_idx]
# y = [train_data_y[id] for id in batch_idx]
for i, (x, y) in enumerate(text_iterator):
n_samples += len(x)
uidx += 1
use_noise.set_value(1.)
x, x_mask, y, y_mask = prepare_data(x, y)
if x is None:
print 'Minibatch with zero sample under length ', maxlen
uidx -= 1
continue
ud_start = time.time()
# compute cost, grads and copy grads to shared variables
cost = f_grad_shared(x, x_mask, y, y_mask)
# do the update on parameters
f_update(lrate)
ud = time.time() - ud_start
# check for bad numbers, usually we remove non-finite elements
# and continue training - but not done here
if np.isnan(cost) or np.isinf(cost):
print 'NaN detected'
return 1., 1., 1.
# discount reward
if lr_discount_freq > 0 and np.mod(uidx, lr_discount_freq) == 0:
lrate *= 0.5
print 'Discount learning rate to {} at iteration {}'.format(lrate, uidx)
# verbose
if np.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud
sys.stdout.flush()
if np.mod(uidx, saveFreq) == 0:
# save with uidx
if not overwrite:
# print 'Saving the model at iteration {}...'.format(uidx),
saveto_uidx = '{}.iter{}.npz'.format(
os.path.splitext(saveto)[0], uidx)
np.savez(saveto_uidx, history_errs=history_errs,
uidx=uidx, **unzip(tparams))
# print 'Done'
# sys.stdout.flush()
# generate some samples with the model and display them
# finish after this many updates
if uidx >= finish_after:
print 'Finishing after %d iterations!' % uidx
estop = True
break
print 'Seen %d samples' % n_samples
if estop:
break
if best_p is not None:
zipp(best_p, tparams)
use_noise.set_value(0.)
return 0.
def train(
experiment_id,
model_options,
data_options,
validation_options,
patience, # early stopping patience
max_epochs,
finish_after, # finish after this many updates
decay_c, # L2 regularization penalty
alpha_c, # alignment regularization
clip_c, # gradient clipping threshold
lrate, # learning rate
optimizer,
saveto,
valid_freq,
eval_intv, # time interval for evaluation in minutes
save_freq, # save the parameters after every saveFreq updates
sample_freq, # generate some samples after every sampleFreq
reload_=False):
worddicts_r, train_stream, valid_stream = load_data(**data_options)
LOGGER.info('Building model')
params = init_params(model_options)
# reload parameters
model_filename = '{}.model.npz'.format(experiment_id)
model_option_filename = '{}.config.json'.format(experiment_id)
saveto_filename = '{}.npz'.format(saveto)
if reload_ and os.path.exists(saveto_filename):
LOGGER.info('Loading parameters from {}'.format(saveto_filename))
params = load_params(saveto_filename, params)
LOGGER.info('Initializing parameters')
tparams = init_tparams(params)
# use_noise is for dropout
trng, use_noise, \
x, x_mask, y, y_mask, \
opt_ret, \
cost = \
build_model(tparams, model_options)
inps = [x, x_mask, y, y_mask]
LOGGER.info('Building sampler')
f_init, f_next = build_sampler(tparams, model_options, trng)
# before any regularizer
LOGGER.info('Building f_log_probs')
f_log_probs = theano.function(inps, cost, profile=False)
cost = cost.mean()
# apply L2 regularization on weights
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in six.iteritems(tparams):
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
# regularize the alpha weights
if alpha_c > 0. and not model_options['decoder'].endswith('simple'):
alpha_c = theano.shared(numpy.float32(alpha_c), name='alpha_c')
alpha_reg = alpha_c * (
(tensor.cast(y_mask.sum(0) // x_mask.sum(0), 'float32')[:, None] -
opt_ret['dec_alphas'].sum(0))**2).sum(1).mean()
cost += alpha_reg
# Not used?
# after all regularizers - compile the computational graph for cost
# LOGGER.info('Building f_cost')
# f_cost = theano.function(inps, cost, profile=False)
LOGGER.info('Computing gradient')
grads = tensor.grad(cost, wrt=itemlist(tparams))
# apply gradient clipping here
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(
tensor.switch(g2 > (clip_c**2), g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
# compile the optimizer, the actual computational graph is compiled here
lr = tensor.scalar(name='lr')
LOGGER.info('Building optimizers')
f_grad_shared, f_update = getattr(optimizers, optimizer)(lr, tparams,
grads, inps, cost)
LOGGER.info('Optimization')
log = Logger(filename='{}.log.jsonl.gz'.format(experiment_id))
# evaluation score will be stored into the following queue
valid_ret_queue = Queue.Queue()
process_queue = Queue.Queue()
rt = prepare_validation_timer(tparams, process_queue, model_filename,
model_option_filename, eval_intv,
valid_ret_queue, **validation_options)
rt.start()
def _timer_signal_handler(signum, frame):
LOGGER.info('Received SIGINT')
LOGGER.info('Now attempting to stop the timer')
rt.stop()
LOGGER.info('Please wait for terminating all child processes')
while not process_queue.empty():
proc = process_queue.get()
if proc.poll() is None: # check if the process has terminated
# child process is still working
# LOGGER.info('Attempt to kill', proc.pid)
# terminate it by sending an interrupt signal
proc.send_signal(signal.SIGINT)
# wait for child process while avoiding deadlock
# ignore outputs
proc.communicate()
sys.exit(130)
signal.signal(signal.SIGINT, _timer_signal_handler)
train_start = time.clock()
best_p = None
best_score = 0
bad_counter = 0
uidx = 0
estop = False
for eidx in xrange(max_epochs):
n_samples = 0
for x, x_mask, y, y_mask in train_stream.get_epoch_iterator():
n_samples += len(x)
x, x_mask, y, y_mask = x.T, x_mask.T, y.T, y_mask.T
use_noise.set_value(1.)
uidx += 1
log_entry = {'iteration': uidx, 'epoch': eidx}
# compute cost, grads and copy grads to shared variables
update_start = time.clock()
cost = f_grad_shared(x, x_mask, y, y_mask)
f_update(lrate)
log_entry['cost'] = float(cost)
log_entry['average_source_length'] = float(x_mask.sum(0).mean())
log_entry['average_target_length'] = float(y_mask.sum(0).mean())
log_entry['update_time'] = time.clock() - update_start
log_entry['train_time'] = time.clock() - train_start
# check for bad numbers, usually we remove non-finite elements
# and continue training - but not done here
if not numpy.isfinite(cost):
LOGGER.error('NaN detected')
return 1., 1., 1.
# save the best model so far
if numpy.mod(uidx, save_freq) == 0:
LOGGER.info('Saving best model so far')
if best_p is not None:
params = best_p
else:
params = unzip(tparams)
# save params to exp_id.npz and symlink model.npz to it
save_params(params, model_filename, saveto_filename)
# generate some samples with the model and display them
if numpy.mod(uidx, sample_freq) == 0:
# FIXME: random selection?
log_entry['samples'] = []
for jj in xrange(numpy.minimum(5, x.shape[1])):
log_entry['samples'].append({
'source': '',
'truth': '',
'sample': ''
})
stochastic = True
sample, _, score = gen_sample(tparams,
f_init,
f_next,
x[:, jj][:, None],
model_options,
trng=trng,
k=1,
maxlen=30,
stochastic=stochastic,
argmax=False)
for vv in x[:, jj]:
if vv == 0:
break
if vv in worddicts_r[0]:
token = worddicts_r[0][vv]
else:
token = UNK_TOKEN
log_entry['samples'][-1]['source'] += token + ' '
for vv in y[:, jj]:
if vv == 0:
break
if vv in worddicts_r[1]:
token = worddicts_r[1][vv]
else:
token = UNK_TOKEN
log_entry['samples'][-1]['truth'] += token + ' '
if stochastic:
ss = sample
else:
score = score / numpy.array([len(s) for s in sample])
ss = sample[score.argmin()]
for vv in ss:
if vv == 0:
break
if vv in worddicts_r[1]:
token = worddicts_r[1][vv]
else:
token = UNK_TOKEN
log_entry['samples'][-1]['sample'] += token + ' '
# validate model on validation set and early stop if necessary
if numpy.mod(uidx, valid_freq) == 0:
use_noise.set_value(0.)
valid_errs = pred_probs(f_log_probs, model_options,
valid_stream)
valid_err = valid_errs.mean()
log_entry['validation_cost'] = float(valid_err)
if not numpy.isfinite(valid_err):
raise RuntimeError('NaN detected in validation error')
# collect validation scores (e.g., BLEU) from the child thread
if not valid_ret_queue.empty():
(ret_model, scores) = valid_ret_queue.get()
valid_bleu = scores[0]
# LOGGER.info('BLEU on the validation set: %.2f' % valid_bleu)
log_entry['validation_bleu'] = valid_bleu
if valid_bleu > best_score:
best_p = ret_model
best_score = valid_bleu
bad_counter = 0
else:
bad_counter += 1
if bad_counter > patience:
estop = True
break
# finish after this many updates
if uidx >= finish_after:
LOGGER.info('Finishing after {} iterations'.format(uidx))
estop = True
break
log.log(log_entry)
LOGGER.info('Completed epoch, seen {} samples'.format(n_samples))
if estop:
log.log(log_entry)
break
if best_p is not None:
zipp(best_p, tparams)
use_noise.set_value(0.)
LOGGER.info('Calculating validation cost')
valid_err = pred_probs(f_log_probs, model_options, valid_stream).mean()
if not best_p:
best_p = unzip(tparams)
params = copy.copy(best_p)
save_params(params, model_filename, saveto_filename)
rt.stop()
return valid_err
def train(
experiment_id,
data_base_path,
output_base_path,
model_options,
data_options,
validation_options,
patience, # early stopping patience
max_epochs,
finish_after, # finish after this many updates
clip_c, # gradient clipping threshold
lrate, # learning rate
optimizer,
saveto,
valid_freq,
time_limit,
save_freq, # save the parameters after every saveFreq updates
sample_freq, # generate some samples after every sampleFreq
verbose,
reload_from=None,
pretrained_word_emb=None):
start_time = time.time()
def join_data_base_path(data_base, options):
for kk, vv in six.iteritems(options):
if kk in [
'src', 'trg', 'input_vocab', 'label_vocab', 'valid_src',
'valid_trg'
]:
options[kk] = os.path.join(data_base, options[kk])
return options
data_options = join_data_base_path(data_base_path, data_options)
validation_options = join_data_base_path(data_base_path,
validation_options)
worddicts_r, train_stream, valid_stream = load_data(**data_options)
model_options['n_input_tokens'] = len(worddicts_r[0])
model_options['n_labels'] = len(worddicts_r[1])
if model_options['label_type'] == 'binary':
model_options['n_bins'] = len(worddicts_r[1])
model_options['n_labels'] = 2
max_sample_length = len(worddicts_r[1])
else:
max_sample_length = data_options['max_label_length']
LOGGER.info('Building model')
params = init_params(model_options)
# reload parameters
best_filename = '{}/{}.{}.best.npz'.format(output_base_path, experiment_id,
saveto)
if pretrained_word_emb and os.path.exists(pretrained_word_emb):
assert model_options['input_token_level'] == 'word'
LOGGER.info('Loading pretrained word embeddings from {}'.format(
pretrained_word_emb))
pretrained_emb = load_pretrained_embeddings(pretrained_word_emb)
# TODO check if the size of the pretrained word embedding equals
# the size of the initialized word embeddings
# Also, check whether or not the vocabulary in the pretrained word
# embeddings is identical to the vocabulary in the model.
pvocab = pretrained_emb['vocab'] # (idx, word)
# XXX if the assertians passed, then load the pretrained embeddings
assert pretrained_emb['Wemb'].dtype == numpy.float32, \
'The pretrained word embeddings should be float32\n'
assert pretrained_emb['Wemb'].shape[1] == params['Wemb'].shape[1], \
'{} does not match {}\n'.format(pretrained_emb['Wemb'].shape[1],
params['Wemb'].shape[1])
pretrained_word2id = {word: idx for (idx, word) in pvocab}
param_indices, indices = [], []
for ii in xrange(len(worddicts_r[0])):
if ii >= data_options['n_input_tokens']:
break
word = worddicts_r[0][ii]
if word in pretrained_word2id:
word_idx = pretrained_word2id[word]
indices.append(word_idx)
param_indices.append(ii)
assert len(indices) <= data_options['n_input_tokens']
params['Wemb'][param_indices] = pretrained_emb['Wemb'][indices]
# normalize word embeddings
params['Wemb'] = params['Wemb'] / \
numpy.sqrt((params['Wemb']**2).sum(axis=1)[:, None])
if reload_from and os.path.exists(reload_from):
LOGGER.info('Loading parameters from {}'.format(reload_from))
params = load_params(reload_from, params)
LOGGER.info('Initializing parameters')
tparams = init_tparams(params)
# use_noise is for dropout
trng, use_noise, encoder_vars, decoder_vars, \
opt_ret, costs = build_model(tparams, model_options)
inps = encoder_vars + decoder_vars
LOGGER.info('Building sampler')
f_sample_inits, f_sample_nexts \
= build_sampler(tparams, model_options, trng, use_noise)
# before any regularizer
LOGGER.info('Building functions to compute log prob')
f_log_probs = [
theano.function(inps,
cost_,
name='f_log_probs_%s' % cost_.name,
on_unused_input='ignore') for cost_ in costs
]
assert len(costs) == 1
cost = costs[0]
'''
for cost_ in costs[1:]:
cost += cost_
'''
cost = cost.mean()
LOGGER.info('Computing gradient')
grads = tensor.grad(cost, wrt=itemlist(tparams))
# apply gradient clipping here
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(
tensor.switch(g2 > (clip_c**2), g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
# compile the optimizer, the actual computational graph is compiled here
lr = tensor.scalar(name='lr')
LOGGER.info('Building optimizers')
f_grad_shared, f_update, optimizer_state = \
getattr(optimizers, optimizer)(lr, tparams, grads, inps, cost)
optimizer_state = name_dict(optimizer_state)
# TODO set_value optimizer_state
if reload_from and os.path.exists(reload_from):
LOGGER.info('Loading optimizer state from {}'.format(reload_from))
optimizer_state = load_params(reload_from,
optimizer_state,
theano_var=True)
LOGGER.info('Optimization')
log = Logger(
filename='{}/{}.log.jsonl.gz'.format(output_base_path, experiment_id))
best_valid_err = float('inf')
best_model = None
total_nsamples = 0
uidx = 0
uidx_restore = [0]
estop = False
if reload_from and os.path.exists(reload_from):
rmodel = numpy.load(reload_from)
if 'uidx' in rmodel:
uidx_restore = rmodel['uidx']
if 'best_valid_err' in rmodel:
best_valid_err = rmodel['best_valid_err']
if 'total_nsamples' in rmodel and rmodel['total_nsamples'] > 0:
total_nsamples = rmodel['total_nsamples']
best_model = [unzip(tparams), unzip(optimizer_state), uidx_restore]
train_start = time.clock()
max_updates_per_epoch = total_nsamples / data_options['batch_size']
try:
for epoch in xrange(0, max_epochs):
if total_nsamples > 0 and \
uidx + max_updates_per_epoch < uidx_restore[0]:
uidx += max_updates_per_epoch
continue
n_samples = 0
for x, x_mask, \
y, y_mask in train_stream.get_epoch_iterator():
n_samples += len(x)
uidx += 1
if uidx < uidx_restore[0]:
continue
x_length = x_mask.sum(1).mean()
if model_options['label_type'] == 'binary':
old_y = y
y, y_mask = mul2bin(y, y_mask, model_options['n_bins'])
y, y_mask = y.T, y_mask.T
if data_options['input_token_level'] == 'character':
x, x_mask = prepare_character_tensor(x)
else:
x, x_mask = x.T, x_mask.T
unk_token_ratio = (x == 1).sum(0) / x_mask.sum(0)
non_empty_insts = unk_token_ratio <= 0.5
y = y[:, non_empty_insts]
y_mask = y_mask[:, non_empty_insts]
x = x[:, non_empty_insts]
x_mask = x_mask[:, non_empty_insts]
if x.shape[1] == 0:
continue
encoder_inps = [x, x_mask]
decoder_inps = [y, y_mask]
inps = encoder_inps + decoder_inps
use_noise.set_value(1.)
log_entry = {'iteration': uidx, 'epoch': epoch}
# compute cost, grads and copy grads to shared variables
update_start = time.clock()
cost = f_grad_shared(*inps)
f_update(lrate)
if verbose:
log_entry['cost'] = float(cost)
log_entry['average_source_length'] = \
float(x_length)
log_entry['average_target_length'] = \
float(y_mask.sum(0).mean())
log_entry['update_time'] = time.clock() - update_start
log_entry['train_time'] = time.clock() - train_start
# check for bad numbers, usually we remove non-finite elements
# and continue training - but not done here
if not numpy.isfinite(cost):
LOGGER.error('NaN detected')
return 1., 1., 1.
# validate model on validation set and early stop if necessary
if numpy.mod(uidx, valid_freq) == 0:
use_noise.set_value(0.)
valid_errs = [
numpy.mean(pred_probs(f_, model_options, valid_stream))
for f_ in f_log_probs
]
for f_, err_ in zip(f_log_probs, valid_errs):
log_entry['validation_%s' % f_.name] = float(err_)
valid_scores = do_validation(f_sample_inits,
f_sample_nexts, tparams,
max_sample_length, trng,
model_options, valid_stream)
for eval_type, score in valid_scores.items():
log_entry['validation_%s' % eval_type] = score
for f_, err_ in zip(f_log_probs, valid_errs):
if not numpy.isfinite(err_):
raise RuntimeError(('NaN detected in validation '
'error of %s') % f_.name)
valid_err = numpy.array(valid_errs).sum()
if valid_err < best_valid_err:
best_valid_err = valid_err
best_model = [
unzip(tparams),
unzip(optimizer_state), [uidx]
]
# save the best model so far
if numpy.mod(uidx, save_freq) == 0 and \
uidx > uidx_restore[0]:
LOGGER.info('Saving best model so far')
if best_model is not None:
params, opt_state, save_at_uidx = best_model
else:
params = unzip(tparams)
opt_state = unzip(optimizer_state)
save_at_uidx = [uidx]
# save params to exp_id.npz and symlink model.npz to it
params_and_state = merge(
params, opt_state, {'uidx': save_at_uidx},
{'best_valid_err': best_valid_err},
{'total_nsamples': total_nsamples})
save_params(params_and_state, best_filename)
# generate some samples with the model and display them
if sample_freq > 0 and numpy.mod(uidx, sample_freq) == 0:
# FIXME: random selection?
log_entry['samples'] = []
if data_options['input_token_level'] == 'character':
batch_size = x.shape[2]
else:
batch_size = x.shape[1]
for jj in xrange(numpy.minimum(5, batch_size)):
stats = [('source', ''), ('truth', ''), ('sample', ''),
('align_sample', '')]
log_entry['samples'].append(OrderedDict(stats))
if data_options['input_token_level'] == 'character':
sample_encoder_inps = [
x[:, :, jj][:, :, None], x_mask[:, :, jj][:, :,
None]
]
else:
sample_encoder_inps = [
x[:, jj][:, None], x_mask[:, jj][:, None]
]
solutions = gen_sample(tparams,
f_sample_inits,
f_sample_nexts,
sample_encoder_inps,
model_options,
trng=trng,
k=12,
max_label_len=max_sample_length,
argmax=False)
sample = solutions['samples']
alignment = solutions['alignments']
score = solutions['scores']
score = score / numpy.array([len(s) for s in sample])
ss = sample[score.argmin()]
alignment = alignment[score.argmin()]
if model_options['label_type'] == 'binary':
# print(y[0], y.shape, old_y.shape)
y = old_y.T
assert type(ss) == list
assert len(ss) == max_sample_length
new_ss = [
tidx for tidx, s in enumerate(ss) if s == 1
]
# print(len(ss), numpy.sum(ss), new_ss)
ss = new_ss
assert type(ss) == list
if len(ss) == 0:
ss.append(0)
if ss[0] == 0: # if the first token is <EOS>
ss = ss[1:] + ss[:1]
if data_options['input_token_level'] == 'character':
num_src_words = int(
(x_mask[:, :, jj].sum(0) > 0).sum())
num_chars, num_words, num_samples = x.shape
for widx in xrange(num_words):
if x_mask[:, widx, jj].sum() == 0:
break
for cidx in xrange(num_chars):
cc = x[cidx, widx, jj]
if cc == 0:
break
if cc in worddicts_r[0]:
token = worddicts_r[0][cc]
else:
token = UNK_TOKEN
log_entry['samples'][-1]['source'] \
+= token
log_entry['samples'][-1]['source'] += \
' '
else:
num_src_words = int(x_mask[:, jj].sum())
num_words, num_samples = x.shape
for vv in x[:, jj]:
if vv == 0:
break
if vv in worddicts_r[0]:
token = worddicts_r[0][vv]
else:
token = UNK_TOKEN
log_entry['samples'][-1]['source'] \
+= token + ' '
for vv in y[:, jj]:
if vv == 0:
break
if vv in worddicts_r[1]:
token = worddicts_r[1][vv]
else:
token = UNK_TOKEN
log_entry['samples'][-1]['truth'] += token + ' '
for tidx, vv in enumerate(ss):
if vv == 0:
break
if vv in worddicts_r[1]:
token = worddicts_r[1][vv]
else:
token = UNK_TOKEN
assert tidx >= 0 and tidx < len(alignment), \
'%d\t%d' % (tidx, len(alignment))
align_src_word_idx = \
(alignment[tidx][
:num_src_words-1]).argmax()
aligned_token = '%s_<%d>' % \
(token, align_src_word_idx)
log_entry['samples'][-1]['sample'] += token + ' '
log_entry['samples'][-1]['align_sample'] \
+= aligned_token + ' '
# finish after this many updates
if uidx >= finish_after:
LOGGER.info('Finishing after {} iterations'.format(uidx))
estop = True
break
if time_limit > 0 and \
(time.time() - start_time > time_limit * 60):
LOGGER.info(
'Time limit {} mins is over'.format(time_limit))
estop = True
break
if verbose and len(log_entry) > 2:
log.log(log_entry)
LOGGER.info('Completed epoch, seen {} samples'.format(n_samples))
if total_nsamples == 0:
total_nsamples = n_samples
if estop:
log.log(log_entry)
break
if best_model is not None:
assert len(best_model) == 3
best_p, best_state, best_uidx = best_model
zipp(best_p, tparams)
zipp(best_state, optimizer_state)
'''
use_noise.set_value(0.)
LOGGER.info('Calculating validation cost')
valid_errs = do_validation(f_log_probs, model_options, valid_stream)
'''
if not best_model:
best_p = unzip(tparams)
best_state = unzip(optimizer_state)
best_uidx = [uidx]
best_p = copy.copy(best_p)
best_state = copy.copy(best_state)
params_and_state = merge(best_p, best_state, {'uidx': best_uidx},
{'best_valid_err': best_valid_err},
{'total_nsamples': total_nsamples})
save_params(params_and_state, best_filename)
except Exception:
LOGGER.error(traceback.format_exc())
best_valid_err = -1.
else:
# XXX add something needed
print('Training Done')
return best_valid_err
confusion = sklearn.metrics.confusion_matrix(
labtest, test_pred)
if len(history_errs) > patience and val_err >= np.array(
history_errs)[:-patience, 0].min():
bad_count = bad_counter + 1
if bad_counter > patience:
estop = True
break
if estop:
break
except KeyboardInterrupt:
print('Training interrupted')
end_time = time.time()
if best_p is not None:
zipp(best_p, tparams)
else:
best_p = unzip(tparams)
max_epochs = 100
patience = 10
lrate = 0.002
valid_batch = 100
print('Start Training...')
start_time = time.time()
try:
for eidx in xrange(max_epochs):
batch_index = get_minibatches_idx(train.shape[0],
batch_size,
shuffle=True)
for _, train_index in batch_index: