def test_cocoeval_vtt():
engine = data_engine.Movie2Caption(
'attention',
'lsmdc16',
video_feature='googlenet',
mb_size_train=20,
mb_size_test=20,
maxlen=50,
n_words=20000,
dec='standard',
proc='nostd',
n_frames=20,
outof=None,
data_dir='/PATH/TO/data/lsmdc16/pkls/',
feats_dir='/PATH/TO/lsmdc16/features_googlenet')
samples_valid = common.load_txt_file('/PATH/TO/valid_samples.txt')
samples_test = common.load_txt_file('PATH/TO/test_samples.txt')
samples_valid = [sample.strip() for sample in samples_valid]
samples_test = [sample.strip() for sample in samples_test]
samples_valid = build_sample_pairs(samples_valid, engine.valid_ids)
samples_test = build_sample_pairs(samples_test, engine.test_ids)
valid_score, test_score = score_with_cocoeval(samples_valid, samples_test,
engine)
print valid_score, test_score
def test_cocoeval():
engine = data_engine.Movie2Caption('attention',
'lsmdc16',
video_feature='googlenet',
mb_size_train=20,
mb_size_test=20,
maxlen=50,
n_words=20000,
dec='standard',
proc='nostd',
n_frames=20,
outof=None)
# samples_valid = common.load_txt_file('./test/valid_samples.txt')
# samples_test = common.load_txt_file('./test/test_samples.txt')
samples_valid = common.load_txt_file(
'./results/yao_base/valid_samples.txt')
samples_test = common.load_txt_file('./results/yao_base/test_samples.txt')
samples_valid = [sample.strip() for sample in samples_valid]
samples_test = [sample.strip() for sample in samples_test]
samples_valid = build_sample_pairs(samples_valid, engine.valid_ids)
samples_test = build_sample_pairs(samples_test, engine.test_ids)
valid_score, test_score = score_with_cocoeval(samples_valid, samples_test,
engine)
print valid_score, test_score
def test_cocoeval():
engine = data_engine.Movie2Caption('attention', 'youtube2text',
video_feature='googlenet',
mb_size_train=20,
mb_size_test=20,
maxlen=50, n_words=20000,
n_frames=20, outof=None)
samples_valid = utils.load_txt_file('./model_files/valid_samples.txt')
samples_test = utils.load_txt_file('./model_files/test_samples.txt')
samples_valid = [sample.strip() for sample in samples_valid]
samples_test = [sample.strip() for sample in samples_test]
samples_valid = build_sample_pairs(samples_valid, engine.valid_ids)
samples_test = build_sample_pairs(samples_test, engine.test_ids)
valid_score, test_score = score_with_cocoeval(samples_valid, samples_test, engine)
print valid_score, test_score
def test_cocoeval():
dataset_name = 'MSVD'
cnn_name = 'ResNet152'
train_data_ids_path = config.MSVD_DATA_IDS_TRAIN_PATH
val_data_ids_path = config.MSVD_DATA_IDS_VAL_PATH
test_data_ids_path = config.MSVD_DATA_IDS_TEST_PATH
vocab_path = config.MSVD_VOCAB_PATH
reverse_vocab_path = config.MSVD_REVERSE_VOCAB_PATH
mb_size_train = 64
mb_size_test = 128
maxlen_caption = 30
train_caps_path = config.MSVD_VID_CAPS_TRAIN_PATH
val_caps_path = config.MSVD_VID_CAPS_VAL_PATH
test_caps_path = config.MSVD_VID_CAPS_TEST_PATH
feats_dir = config.MSVD_FEATS_DIR+cnn_name+"/"
engine = data_engine.Movie2Caption(dataset_name,cnn_name,train_data_ids_path, val_data_ids_path, test_data_ids_path,
vocab_path, reverse_vocab_path, mb_size_train, mb_size_test, maxlen_caption,
train_caps_path, val_caps_path, test_caps_path, feats_dir)
samples_valid = engine.val_data_ids
samples_test = engine.test_data_ids
samples_valid = build_sample_pairs_test(samples_valid, engine, mode="val")
samples_test = build_sample_pairs_test(samples_test, engine, mode="test")
valid_score, test_score = score_with_cocoeval(samples_valid, samples_test, engine)
print valid_score
def test(model_options_file='model_options.pkl',
model_file='model_best_so_far.npz'):
from_dir = 'model_files/'
print 'preparing reload'
model_options = utils.load_pkl(from_dir + model_options_file)
print 'Loading data'
engine = data_engine.Movie2Caption(
'attention', model_options['dataset'], model_options['video_feature'],
model_options['batch_size'], model_options['valid_batch_size'],
model_options['maxlen'], model_options['n_words'], model_options['K'],
model_options['OutOf'])
print 'init params'
t0 = time.time()
model = Model()
params = model.init_params(model_options)
model_saved = from_dir + model_file
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = utils.load_params(model_saved, params)
tparams = utils.init_tparams(params)
print tparams.keys
print 'buliding sampler'
use_noise = theano.shared(numpy.float32(0.))
use_noise.set_value(0.)
trng = RandomStreams(1234)
f_init, f_next = model.build_sampler(tparams, model_options, use_noise,
trng)
print 'start test...'
blue_t0 = time.time()
scores, processes, queue, rqueue, shared_params = \
metrics.compute_score(
model_type='attention',
model_archive=params,
options=model_options,
engine=engine,
save_dir=from_dir,
beam=5, n_process=5,
whichset='both',
on_cpu=False,
processes=None, queue=None, rqueue=None,
shared_params=None, metric=model_options['metric'],
one_time=False,
f_init=f_init, f_next=f_next, model=model
)
valid_B1 = scores['valid']['Bleu_1']
valid_B2 = scores['valid']['Bleu_2']
valid_B3 = scores['valid']['Bleu_3']
valid_B4 = scores['valid']['Bleu_4']
valid_Rouge = scores['valid']['ROUGE_L']
valid_Cider = scores['valid']['CIDEr']
valid_meteor = scores['valid']['METEOR']
test_B1 = scores['test']['Bleu_1']
test_B2 = scores['test']['Bleu_2']
test_B3 = scores['test']['Bleu_3']
test_B4 = scores['test']['Bleu_4']
test_Rouge = scores['test']['ROUGE_L']
test_Cider = scores['test']['CIDEr']
test_meteor = scores['test']['METEOR']
print 'computing meteor/blue score used %.4f sec, '\
'B@1: %.3f, B@2: %.3f, B@3: %.3f, B@4: %.3f, M: %.3f'%(
time.time()-blue_t0, test_B1, test_B2, test_B3, test_B4, test_meteor)
def generate(model_options_file='model_options.pkl',
model_file='model_best_so_far.npz'):
from_dir = 'model_files/'
print 'preparing reload'
model_options = utils.load_pkl(from_dir + model_options_file)
print 'Loading data'
engine = data_engine.Movie2Caption(
'attention', model_options['dataset'], model_options['video_feature'],
model_options['batch_size'], model_options['valid_batch_size'],
model_options['maxlen'], model_options['n_words'], model_options['K'],
model_options['OutOf'])
feat = numpy.load('datas/vid1715.npy')
ctx = engine.get_sub_frames(feat)
ctx_mask = engine.get_ctx_mask(ctx)
print 'init params'
t0 = time.time()
model = Model()
params = model.init_params(model_options)
model_saved = from_dir + model_file
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = utils.load_params(model_saved, params)
tparams = utils.init_tparams(params)
print tparams.keys
print 'buliding sampler'
use_noise = theano.shared(numpy.float32(0.))
use_noise.set_value(0.)
trng = RandomStreams(1234)
f_init, f_next = model.build_sampler(tparams, model_options, use_noise,
trng)
print 'start generate...'
g_t0 = time.time()
sample, sample_score, _, _ = model.gen_sample(
None,
f_init,
f_next,
ctx,
ctx_mask,
model_options,
None,
5,
maxlen=model_options['maxlen'])
print sample
# best_one = numpy.argmin(sample_score)
# sample = sample[best_one]
for s in sample:
for kk, ss in enumerate([s]):
for vv in ss:
if vv == 0:
break
if vv in engine.word_idict:
print engine.word_idict[vv],
else:
print 'UNK',
print
def train(self,
random_seed=1234,
reload_=False,
verbose=True,
debug=True,
save_model_dir='',
from_dir=None,
# dataset
dataset='youtube2text',
video_feature='googlenet',
K=10,
OutOf=240,
# network
dim_word=256, # word vector dimensionality
ctx_dim=-1, # context vector dimensionality, auto set
tu_dim=512,
mu_dim=1024,
vu_dim=1024,
n_layers_out=1,
n_layers_init=1,
prev2out=False,
ctx2out=False,
selector=False,
n_words=100000,
maxlen=100, # maximum length of the description
use_dropout=False,
isGlobal=False,
# training
patience=10,
max_epochs=5000,
decay_c=0.,
alpha_c=0.,
alpha_entropy_r=0.,
lrate=0.01,
optimizer='adadelta',
clip_c=2.,
# minibatch
batch_size = 64,
valid_batch_size = 64,
dispFreq=100,
validFreq=10,
saveFreq=10, # save the parameters after every saveFreq updates
sampleFreq=10, # generate some samples after every sampleFreq updates
# metric
metric='blue'
):
self.rng_numpy, self.rng_theano = get_two_rngs()
model_options = locals().copy()
if 'self' in model_options:
del model_options['self']
model_options = validate_options(model_options)
with open('%smodel_options.pkl'%save_model_dir, 'wb') as f:
pkl.dump(model_options, f)
print 'Loading data'
self.engine = data_engine.Movie2Caption('attention', dataset,
video_feature,
batch_size, valid_batch_size,
maxlen, n_words,
K, OutOf)
model_options['ctx_dim'] = self.engine.ctx_dim
print 'init params'
t0 = time.time()
params = self.init_params(model_options)
# reloading
if reload_:
model_saved = from_dir+'/model_best_so_far.npz'
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = load_params(model_saved, params)
tparams = init_tparams(params)
if verbose:
print tparams.keys
trng, use_noise, x, mask, ctx, mask_ctx, alphas, cost, extra, test = \
self.build_model(tparams, model_options)
if debug:
print 'buliding test'
test_fun = theano.function([x, mask, ctx, mask_ctx],
test,
name='f_test',
on_unused_input='ignore')
print 'buliding sampler'
f_init, f_next = self.predict.build_sampler(self.layers, tparams, model_options, use_noise, trng)
# before any regularizer
print 'building f_log_probs'
f_log_probs = theano.function([x, mask, ctx, mask_ctx], -cost,
profile=False, on_unused_input='ignore')
cost = cost.mean()
if decay_c > 0.:
decay_c = theano.shared(np.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
if alpha_c > 0.:
alpha_c = theano.shared(np.float32(alpha_c), name='alpha_c')
alpha_reg = alpha_c * ((1.-alphas.sum(0))**2).sum(0).mean()
cost += alpha_reg
if alpha_entropy_r > 0:
alpha_entropy_r = theano.shared(np.float32(alpha_entropy_r),
name='alpha_entropy_r')
alpha_reg_2 = alpha_entropy_r * (-tensor.sum(alphas *
tensor.log(alphas+1e-8),axis=-1)).sum(0).mean()
cost += alpha_reg_2
else:
alpha_reg_2 = tensor.zeros_like(cost)
print 'building f_alpha'
f_alpha = theano.function([x, mask, ctx, mask_ctx],
[alphas, alpha_reg_2],
name='f_alpha',
on_unused_input='ignore')
print 'compute grad'
grads = tensor.grad(cost, wrt=itemlist(tparams))
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(tensor.switch(g2 > (clip_c**2),
g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
lr = tensor.scalar(name='lr')
print 'build train fns'
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads,
[x, mask, ctx, mask_ctx], cost,
extra + grads)
print 'compilation took %.4f sec'%(time.time()-t0)
print 'Optimization'
history_errs = []
# reload history
if reload_:
print 'loading history error...'
history_errs = np.load(
from_dir+'model_best_so_far.npz')['history_errs'].tolist()
bad_counter = 0
processes = None
queue = None
rqueue = None
shared_params = None
uidx = 0
uidx_best_blue = 0
uidx_best_valid_err = 0
estop = False
best_p = unzip(tparams)
best_blue_valid = 0
best_valid_err = 999
alphas_ratio = []
for eidx in xrange(max_epochs):
n_samples = 0
train_costs = []
grads_record = []
print 'Epoch ', eidx
for idx in self.engine.kf_train:
tags = [self.engine.train[index] for index in idx]
n_samples += len(tags)
uidx += 1
use_noise.set_value(1.)
pd_start = time.time()
x, mask, ctx, ctx_mask,vid_names = data_engine.prepare_data(
self.engine, tags)
if debug:
datas = test_fun(x, mask, ctx, ctx_mask)
for item in datas:
print item[0].shape
pd_duration = time.time() - pd_start
if x is None:
print 'Minibatch with zero sample under length ', maxlen
continue
ud_start = time.time()
rvals = f_grad_shared(x, mask, ctx, ctx_mask)
cost = rvals[0]
probs = rvals[1]
alphas = rvals[2]
betas = rvals[3]
grads = rvals[4:]
grads, NaN_keys = grad_nan_report(grads, tparams)
if len(grads_record) >= 5:
del grads_record[0]
grads_record.append(grads)
if NaN_keys != []:
print 'grads contain NaN'
import pdb; pdb.set_trace()
if np.isnan(cost) or np.isinf(cost):
print 'NaN detected in cost'
import pdb; pdb.set_trace()
# update params
f_update(lrate)
ud_duration = time.time() - ud_start
if eidx == 0:
train_error = cost
else:
train_error = train_error * 0.95 + cost * 0.05
train_costs.append(cost)
if np.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, ', Update ', uidx, \
', Train cost mean so far', train_error, \
', betas mean', np.round(betas.mean(), 3), \
', fetching data time spent (sec)', np.round(pd_duration, 3), \
', update time spent (sec)', np.round(ud_duration, 3)
alphas,reg = f_alpha(x,mask,ctx,ctx_mask)
print 'alpha ratio %.3f, reg %.3f' % (
alphas.min(-1).mean() / (alphas.max(-1)).mean(), reg)
if np.mod(uidx, saveFreq) == 0:
pass
if np.mod(uidx, sampleFreq) == 0:
use_noise.set_value(0.)
print '------------- sampling from train ----------'
self.predict.sample_execute(self.engine, model_options, tparams,
f_init, f_next, x, ctx, ctx_mask, trng,vid_names)
print '------------- sampling from valid ----------'
idx = self.engine.kf_valid[np.random.randint(1, len(self.engine.kf_valid) - 1)]
tags = [self.engine.valid[index] for index in idx]
x_s, mask_s, ctx_s, mask_ctx_s,vid_names = data_engine.prepare_data(self.engine, tags)
self.predict.sample_execute(self.engine, model_options, tparams,
f_init, f_next, x_s, ctx_s, mask_ctx_s, trng, vid_names)
# end of sample
if validFreq != -1 and np.mod(uidx, validFreq) == 0:
t0_valid = time.time()
alphas,_ = f_alpha(x, mask, ctx, ctx_mask)
ratio = alphas.min(-1).mean()/(alphas.max(-1)).mean()
alphas_ratio.append(ratio)
np.savetxt(save_model_dir+'alpha_ratio.txt',alphas_ratio)
current_params = unzip(tparams)
np.savez(save_model_dir+'model_current.npz',
history_errs=history_errs, **current_params)
use_noise.set_value(0.)
train_err = -1
train_perp = -1
valid_err = -1
valid_perp = -1
test_err = -1
test_perp = -1
if not debug:
# first compute train cost
if 0:
print 'computing cost on trainset'
train_err, train_perp = self.pred_probs(
'train', f_log_probs,
verbose=model_options['verbose'])
else:
train_err = 0.
train_perp = 0.
if 1:
print 'validating...'
valid_err, valid_perp = self.pred_probs(
'valid', f_log_probs,
verbose=model_options['verbose'],
)
else:
valid_err = 0.
valid_perp = 0.
if 0:
print 'testing...'
test_err, test_perp = self.pred_probs(
'test', f_log_probs,
verbose=model_options['verbose']
)
else:
test_err = 0.
test_perp = 0.
mean_ranking = 0
blue_t0 = time.time()
scores, processes, queue, rqueue, shared_params = \
metrics.compute_score(model_type='attention',
model_archive=current_params,
options=model_options,
engine=self.engine,
save_dir=save_model_dir,
beam=5, n_process=5,
whichset='both',
on_cpu=False,
processes=processes, queue=queue, rqueue=rqueue,
shared_params=shared_params, metric=metric,
one_time=False,
f_init=f_init, f_next=f_next, model=self.predict
)
valid_B1 = scores['valid']['Bleu_1']
valid_B2 = scores['valid']['Bleu_2']
valid_B3 = scores['valid']['Bleu_3']
valid_B4 = scores['valid']['Bleu_4']
valid_Rouge = scores['valid']['ROUGE_L']
valid_Cider = scores['valid']['CIDEr']
valid_meteor = scores['valid']['METEOR']
test_B1 = scores['test']['Bleu_1']
test_B2 = scores['test']['Bleu_2']
test_B3 = scores['test']['Bleu_3']
test_B4 = scores['test']['Bleu_4']
test_Rouge = scores['test']['ROUGE_L']
test_Cider = scores['test']['CIDEr']
test_meteor = scores['test']['METEOR']
print 'computing meteor/blue score used %.4f sec, '\
'blue score: %.1f, meteor score: %.1f'%(
time.time()-blue_t0, valid_B4, valid_meteor)
history_errs.append([eidx, uidx, train_perp, train_err,
valid_perp, valid_err,
test_perp, test_err,
valid_B1, valid_B2, valid_B3,
valid_B4, valid_meteor, valid_Rouge, valid_Cider,
test_B1, test_B2, test_B3,
test_B4, test_meteor, test_Rouge, test_Cider])
np.savetxt(save_model_dir+'train_valid_test.txt',
history_errs, fmt='%.3f')
print 'save validation results to %s'%save_model_dir
# save best model according to the best blue or meteor
if len(history_errs) > 1 and valid_B4 > np.array(history_errs)[:-1, 11].max():
print 'Saving to %s...'%save_model_dir,
np.savez(
save_model_dir+'model_best_blue_or_meteor.npz',
history_errs=history_errs, **best_p)
if len(history_errs) > 1 and valid_err < np.array(history_errs)[:-1, 5].min():
best_p = unzip(tparams)
bad_counter = 0
best_valid_err = valid_err
uidx_best_valid_err = uidx
print 'Saving to %s...'%save_model_dir,
np.savez(
save_model_dir+'model_best_so_far.npz',
history_errs=history_errs, **best_p)
with open('%smodel_options.pkl'%save_model_dir, 'wb') as f:
pkl.dump(model_options, f)
print 'Done'
elif len(history_errs) > 1 and valid_err >= np.array(history_errs)[:-1, 5].min():
bad_counter += 1
print 'history best ', np.array(history_errs)[:,6].min()
print 'bad_counter ', bad_counter
print 'patience ', patience
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if test_B4 > 0.48 and test_meteor > 0.32:
print 'Saving to %s...' % save_model_dir,
numpy.savez(
save_model_dir + 'model_' + str(uidx) + '.npz',
history_errs=history_errs, **current_params)
if self.channel:
self.channel.save()
print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err, \
'best valid err so far',best_valid_err
print 'valid took %.2f sec'%(time.time() - t0_valid)
# end of validatioin
if debug:
break
if estop:
break
if debug:
break
# end for loop over minibatches
print 'This epoch has seen %d samples, train cost %.2f'%(
n_samples, np.mean(train_costs))
# end for loop over epochs
print 'Optimization ended.'
if best_p is not None:
zipp(best_p, tparams)
print 'stopped at epoch %d, minibatch %d, '\
'curent Train %.2f, current Valid %.2f, current Test %.2f '%(
eidx, uidx, np.mean(train_err), np.mean(valid_err), np.mean(test_err))
params = copy.copy(best_p)
np.savez(save_model_dir+'model_best.npz',
train_err=train_err,
valid_err=valid_err, test_err=test_err, history_errs=history_errs,
**params)
if history_errs != []:
history = np.asarray(history_errs)
best_valid_idx = history[:,6].argmin()
np.savetxt(save_model_dir+'train_valid_test.txt', history, fmt='%.4f')
print 'final best exp ', history[best_valid_idx]
return train_err, valid_err, test_err
def train(self,
random_seed=1234,
dim_word=256, # word vector dimensionality
ctx_dim=-1, # context vector dimensionality, auto set
dim=1000, # the number of LSTM units
n_layers_out=1,
n_layers_init=1,
encoder='none',
encoder_dim=100,
prev2out=False,
ctx2out=False,
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
alpha_entropy_r=0.,
lrate=0.01,
selector=False,
n_words=100000,
maxlen=100, # maximum length of the description
optimizer='adadelta',
clip_c=2.,
batch_size = 64,
valid_batch_size = 64,
save_model_dir='/data/lisatmp3/yaoli/exp/capgen_vid/attention/test/',
validFreq=10,
saveFreq=10, # save the parameters after every saveFreq updates
sampleFreq=10, # generate some samples after every sampleFreq updates
metric='blue',
dataset='youtube2text',
video_feature='googlenet',
use_dropout=False,
reload_=False,
from_dir=None,
K=10,
OutOf=240,
verbose=True,
debug=True
):
self.rng_numpy, self.rng_theano = common.get_two_rngs()
save_model_dir = 'kv_densecap_lstmka_'
model_options = locals().copy()
if 'self' in model_options:
del model_options['self']
model_options = validate_options(model_options)
with open('%smodel_options.pkl'%save_model_dir, 'wb') as f:
pkl.dump(model_options, f)
print 'Loading data, lrate', lrate, validFreq
self.engine = data_engine.Movie2Caption('attention', dataset,
video_feature,
batch_size, valid_batch_size,
maxlen, n_words,
K, OutOf)
model_options['ctx_dim'] = self.engine.ctx_dim
model_options['value_dim'] = self.engine.value_dim
model_options['dim'] = 1024
model_options['dim_key_add'] = 512
# model_options['encoder'] = 'lstm_uni'
# model_options['encoder_dim'] = 1024
# set test values, for debugging
[self.x_tv, self.mask_tv,
self.ctx_tv, self.ctx_mask_tv, self.value_tv, self.value_mask_tv] = data_engine.prepare_data(
self.engine, self.engine.kf_train[0])#[self.engine.train[index] for index in idx])
print ('data deims are', self.ctx_tv.shape, self.value_tv.shape)
print 'init params'
t0 = time.time()
params = self.init_params(model_options)
# reloading
#reload_ = True
if reload_:
model_saved = 'model2.npz'
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = load_params(model_saved, params)
#reload_ = False
tparams = init_tparams(params)
trng, use_noise, \
x, mask, ctx, mask_ctx, value, mask_value, alphas, \
cost, extra = \
self.build_model(tparams, model_options)
print 'buliding sampler'
f_init, f_next = self.build_sampler(tparams, model_options, use_noise, trng)
# before any regularizer
print 'building f_log_probs'
f_log_probs = theano.function([x, mask, ctx, mask_ctx, value, mask_value], -cost,
profile=False, on_unused_input='ignore')
cost = cost.mean()
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
if alpha_c > 0.:
alpha_c = theano.shared(numpy.float32(alpha_c), name='alpha_c')
alpha_reg = alpha_c * ((1.-alphas.sum(0))**2).sum(0).mean()
cost += alpha_reg
if alpha_entropy_r > 0:
alpha_entropy_r = theano.shared(numpy.float32(alpha_entropy_r),
name='alpha_entropy_r')
alpha_reg_2 = alpha_entropy_r * (-tensor.sum(alphas *
tensor.log(alphas+1e-8),axis=-1)).sum(0).mean()
cost += alpha_reg_2
else:
alpha_reg_2 = tensor.zeros_like(cost)
print 'building f_alpha'
f_alpha = theano.function([x, mask, ctx, mask_ctx, value, mask_value],
[alphas, alpha_reg_2],
name='f_alpha',
on_unused_input='ignore')
print 'compute grad'
grads = tensor.grad(cost, wrt=itemlist(tparams))
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(tensor.switch(g2 > (clip_c**2),
g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
lr = tensor.scalar(name='lr')
print 'build train fns'
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads,
[x, mask, ctx, mask_ctx, value, mask_value], cost,
extra + grads)
print 'compilation took %.4f sec'%(time.time()-t0)
print 'Optimization'
history_errs = []
# reload history
if reload_:
print 'loading history error...'
history_errs = numpy.load(
from_dir+'model_best_so_far.npz')['history_errs'].tolist()
bad_counter = 0
processes = None
queue = None
rqueue = None
shared_params = None
uidx = 0
uidx_best_blue = 0
uidx_best_valid_err = 0
estop = False
best_p = unzip(tparams)
best_blue_valid = 0
best_valid_err = 999
alphas_ratio = []
validFreq = 1000
for eidx in xrange(max_epochs):
n_samples = 0
train_costs = []
grads_record = []
print 'Epoch ', eidx
for idx in self.engine.kf_train: #self.engine.kf_train:
# tags = [self.engine.train[index] for index in idx]
# n_samples += len(tags)
uidx += 1
use_noise.set_value(1.)
pd_start = time.time()
x, mask, ctx, ctx_mask, val, val_mask = data_engine.prepare_data(
self.engine, idx)
n_samples += ctx.shape[0]
# print ('n samples are', n_samples, x.shape, mask.shape, ctx.shape, ctx_mask.shape, val.shape)
pd_duration = time.time() - pd_start
if x is None:
print 'Minibatch with zero sample under length ', maxlen
continue
ud_start = time.time()
rvals = f_grad_shared(x, mask, ctx, ctx_mask, val, val_mask)
# print ('rvals computed')
cost = rvals[0]
probs = rvals[1]
alphas = rvals[2]
grads = rvals[3:]
grads, NaN_keys = grad_nan_report(grads, tparams)
if len(grads_record) >= 5:
del grads_record[0]
grads_record.append(grads)
if NaN_keys != []:
print 'grads contain NaN'
import pdb; pdb.set_trace()
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected in cost'
import pdb; pdb.set_trace()
# update params
f_update(lrate)
ud_duration = time.time() - ud_start
if eidx == 0:
train_error = cost
else:
train_error = train_error * 0.95 + cost * 0.05
train_costs.append(cost)
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Train cost mean so far', \
train_error, 'fetching data time spent (sec)', pd_duration, \
'update time spent (sec)', ud_duration, 'save_dir', save_model_dir
alphas,reg = f_alpha(x,mask,ctx,ctx_mask, val, val_mask)
print 'alpha ratio %.3f, reg %.3f'%(
alphas.min(-1).mean() / (alphas.max(-1)).mean(), reg)
if numpy.mod(uidx, saveFreq) == 0:
pass
if numpy.mod(uidx, sampleFreq) == 0:
use_noise.set_value(0.)
def sample_execute(from_which):
print '------------- sampling from %s ----------'%from_which
if from_which == 'train':
x_s = x
mask_s = mask
ctx_s = ctx
ctx_mask_s = ctx_mask
value_s = val
value_mask_s = val_mask
elif from_which == 'valid':
idx = self.engine.kf_valid[numpy.random.randint(1, len(self.engine.kf_valid) - 1)]
# tags = [self.engine.valid[index] for index in idx]
x_s, mask_s, ctx_s, ctx_mask_s, value_s, value_mask_s = data_engine.prepare_data(
self.engine, idx)
stochastic = False
for jj in xrange(numpy.minimum(10, x_s.shape[1])):
sample, score, _, _ = self.gen_sample(
tparams, f_init, f_next, ctx_s[jj], ctx_mask_s[jj], value_s[jj], value_mask_s[jj],
model_options,
trng=trng, k=5, maxlen=30, stochastic=stochastic)
# print ('sample is', sample)
if not stochastic:
best_one = numpy.argmin(score)
sample = sample[0]
else:
sample = sample
print 'Truth ',jj,': ',
for vv in x_s[:,jj]:
if vv == 0:
break
if vv in self.engine.word_idict:
print self.engine.word_idict[vv],
else:
print 'UNK',
print
for kk, ss in enumerate([sample]):
print 'Sample (', kk,') ', jj, ': ',
for vv in ss:
if vv == 0:
break
if vv in self.engine.word_idict:
print self.engine.word_idict[vv],
else:
print 'UNK',
print
sample_execute(from_which='train')
sample_execute(from_which='valid')
def train(
random_seed=1234,
dim_word=256, # word vector dimensionality
ctx_dim=-1, # context vector dimensionality, auto set
dim=1000, # the number of LSTM units
n_layers_out=1,
n_layers_init=1,
encoder='none',
encoder_dim=100,
prev2out=False,
ctx2out=False,
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
alpha_entropy_r=0.,
lrate=0.01,
selector=False,
n_words=100000,
maxlen=100, # maximum length of the description
optimizer='adadelta',
clip_c=2.,
batch_size=64,
valid_batch_size=64,
save_model_dir='/data/lisatmp3/yaoli/exp/capgen_vid/attention/test/',
validFreq=10,
saveFreq=10, # save the parameters after every saveFreq updates
sampleFreq=10, # generate some samples after every sampleFreq updates
metric='blue',
dataset='youtube2text',
video_feature='googlenet',
use_dropout=False,
reload_=False,
from_dir=None,
K=10,
OutOf=240,
verbose=True,
debug=True):
rng_numpy, rng_theano = utils.get_two_rngs()
model_options = locals().copy()
if 'self' in model_options:
del model_options['self']
with open('%smodel_options.pkl' % save_model_dir, 'wb') as f:
pkl.dump(model_options, f)
# instance model
layers = Layers()
model = Model()
print 'Loading data'
engine = data_engine.Movie2Caption('attention', dataset, video_feature,
batch_size, valid_batch_size, maxlen,
n_words, K, OutOf)
model_options['ctx_dim'] = engine.ctx_dim
model_options['n_words'] = engine.n_words
print 'n_words:', model_options['n_words']
# set test values, for debugging
idx = engine.kf_train[0]
[x_tv, mask_tv, ctx_tv, ctx_mask_tv
] = data_engine.prepare_data(engine,
[engine.train[index] for index in idx])
print 'init params'
t0 = time.time()
params = model.init_params(model_options)
# reloading
if reload_:
model_saved = from_dir + '/model_best_so_far.npz'
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = utils.load_params(model_saved, params)
tparams = utils.init_tparams(params)
trng, use_noise, \
x, mask, ctx, mask_ctx, \
cost, extra = \
model.build_model(tparams, model_options)
alphas = extra[1]
betas = extra[2]
print 'buliding sampler'
f_init, f_next = model.build_sampler(tparams, model_options, use_noise,
trng)
# before any regularizer
print 'building f_log_probs'
f_log_probs = theano.function([x, mask, ctx, mask_ctx],
-cost,
profile=False,
on_unused_input='ignore')
cost = cost.mean()
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
if alpha_c > 0.:
alpha_c = theano.shared(numpy.float32(alpha_c), name='alpha_c')
alpha_reg = alpha_c * ((1. - alphas.sum(0))**2).sum(-1).mean()
cost += alpha_reg
if alpha_entropy_r > 0:
alpha_entropy_r = theano.shared(numpy.float32(alpha_entropy_r),
name='alpha_entropy_r')
alpha_reg_2 = alpha_entropy_r * (-tensor.sum(
alphas * tensor.log(alphas + 1e-8), axis=-1)).sum(-1).mean()
cost += alpha_reg_2
else:
alpha_reg_2 = tensor.zeros_like(cost)
print 'building f_alpha'
f_alpha = theano.function([x, mask, ctx, mask_ctx], [alphas, betas],
name='f_alpha',
on_unused_input='ignore')
print 'compute grad'
grads = tensor.grad(cost, wrt=utils.itemlist(tparams))
if clip_c > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(
tensor.switch(g2 > (clip_c**2), g / tensor.sqrt(g2) * clip_c,
g))
grads = new_grads
lr = tensor.scalar(name='lr')
print 'build train fns'
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads,
[x, mask, ctx, mask_ctx], cost,
extra + grads)
print 'compilation took %.4f sec' % (time.time() - t0)
print 'Optimization'
history_errs = []
# reload history
if reload_:
print 'loading history error...'
history_errs = numpy.load(
from_dir + 'model_best_so_far.npz')['history_errs'].tolist()
bad_counter = 0
processes = None
queue = None
rqueue = None
shared_params = None
uidx = 0
uidx_best_blue = 0
uidx_best_valid_err = 0
estop = False
best_p = utils.unzip(tparams)
best_blue_valid = 0
best_valid_err = 999
alphas_ratio = []
for eidx in xrange(max_epochs):
n_samples = 0
train_costs = []
grads_record = []
print 'Epoch ', eidx
for idx in engine.kf_train:
tags = [engine.train[index] for index in idx]
n_samples += len(tags)
uidx += 1
use_noise.set_value(1.)
pd_start = time.time()
x, mask, ctx, ctx_mask = data_engine.prepare_data(engine, tags)
pd_duration = time.time() - pd_start
if x is None:
print 'Minibatch with zero sample under length ', maxlen
continue
ud_start = time.time()
rvals = f_grad_shared(x, mask, ctx, ctx_mask)
cost = rvals[0]
probs = rvals[1]
alphas = rvals[2]
betas = rvals[3]
grads = rvals[4:]
grads, NaN_keys = utils.grad_nan_report(grads, tparams)
if len(grads_record) >= 5:
del grads_record[0]
grads_record.append(grads)
if NaN_keys != []:
print 'grads contain NaN'
import pdb
pdb.set_trace()
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected in cost'
import pdb
pdb.set_trace()
# update params
f_update(lrate)
ud_duration = time.time() - ud_start
if eidx == 0:
train_error = cost
else:
train_error = train_error * 0.95 + cost * 0.05
train_costs.append(cost)
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Train cost mean so far', \
train_error, 'fetching data time spent (sec)', pd_duration, \
'update time spent (sec)', ud_duration, 'save_dir', save_model_dir
alphas, betas = f_alpha(x, mask, ctx, ctx_mask)
counts = mask.sum(0)
betas_mean = (betas * mask).sum(0) / counts
betas_mean = betas_mean.mean()
print 'alpha ratio %.3f, betas mean %.3f' % (
alphas.min(-1).mean() /
(alphas.max(-1)).mean(), betas_mean)
l = 0
for vv in x[:, 0]:
if vv == 0:
break
if vv in engine.word_idict:
print '(', numpy.round(betas[l, 0],
3), ')', engine.word_idict[vv],
else:
print '(', numpy.round(betas[l, 0], 3), ')', 'UNK',
l += 1
print '(', numpy.round(betas[l, 0], 3), ')'
if numpy.mod(uidx, saveFreq) == 0:
pass
if numpy.mod(uidx, sampleFreq) == 0:
use_noise.set_value(0.)
print '------------- sampling from train ----------'
x_s = x
mask_s = mask
ctx_s = ctx
ctx_mask_s = ctx_mask
model.sample_execute(engine, model_options, tparams, f_init,
f_next, x_s, ctx_s, ctx_mask_s, trng)
print '------------- sampling from valid ----------'
idx = engine.kf_valid[numpy.random.randint(
1,
len(engine.kf_valid) - 1)]
tags = [engine.valid[index] for index in idx]
x_s, mask_s, ctx_s, mask_ctx_s = data_engine.prepare_data(
engine, tags)
model.sample_execute(engine, model_options, tparams, f_init,
f_next, x_s, ctx_s, mask_ctx_s, trng)
if validFreq != -1 and numpy.mod(uidx, validFreq) == 0:
t0_valid = time.time()
alphas, _ = f_alpha(x, mask, ctx, ctx_mask)
ratio = alphas.min(-1).mean() / (alphas.max(-1)).mean()
alphas_ratio.append(ratio)
numpy.savetxt(save_model_dir + 'alpha_ratio.txt', alphas_ratio)
current_params = utils.unzip(tparams)
numpy.savez(save_model_dir + 'model_current.npz',
history_errs=history_errs,
**current_params)
use_noise.set_value(0.)
train_err = -1
train_perp = -1
valid_err = -1
valid_perp = -1
test_err = -1
test_perp = -1
if not debug:
# first compute train cost
if 0:
print 'computing cost on trainset'
train_err, train_perp = model.pred_probs(
engine,
'train',
f_log_probs,
verbose=model_options['verbose'])
else:
train_err = 0.
train_perp = 0.
if 1:
print 'validating...'
valid_err, valid_perp = model.pred_probs(
engine,
'valid',
f_log_probs,
verbose=model_options['verbose'],
)
else:
valid_err = 0.
valid_perp = 0.
if 1:
print 'testing...'
test_err, test_perp = model.pred_probs(
engine,
'test',
f_log_probs,
verbose=model_options['verbose'])
else:
test_err = 0.
test_perp = 0.
mean_ranking = 0
blue_t0 = time.time()
scores, processes, queue, rqueue, shared_params = \
metrics.compute_score(
model_type='attention',
model_archive=current_params,
options=model_options,
engine=engine,
save_dir=save_model_dir,
beam=5, n_process=5,
whichset='both',
on_cpu=False,
processes=processes, queue=queue, rqueue=rqueue,
shared_params=shared_params, metric=metric,
one_time=False,
f_init=f_init, f_next=f_next, model=model
)
'''
{'blue': {'test': [-1], 'valid': [77.7, 60.5, 48.7, 38.5, 38.3]},
'alternative_valid': {'Bleu_3': 0.40702270203174923,
'Bleu_4': 0.29276570520368456,
'CIDEr': 0.25247168210607884,
'Bleu_2': 0.529069629270047,
'Bleu_1': 0.6804308797115253,
'ROUGE_L': 0.51083584331688392},
'meteor': {'test': [-1], 'valid': [0.282787550236724]}}
'''
valid_B1 = scores['valid']['Bleu_1']
valid_B2 = scores['valid']['Bleu_2']
valid_B3 = scores['valid']['Bleu_3']
valid_B4 = scores['valid']['Bleu_4']
valid_Rouge = scores['valid']['ROUGE_L']
valid_Cider = scores['valid']['CIDEr']
valid_meteor = scores['valid']['METEOR']
test_B1 = scores['test']['Bleu_1']
test_B2 = scores['test']['Bleu_2']
test_B3 = scores['test']['Bleu_3']
test_B4 = scores['test']['Bleu_4']
test_Rouge = scores['test']['ROUGE_L']
test_Cider = scores['test']['CIDEr']
test_meteor = scores['test']['METEOR']
print 'computing meteor/blue score used %.4f sec, '\
'blue score: %.1f, meteor score: %.1f'%(
time.time()-blue_t0, valid_B4, valid_meteor)
history_errs.append([
eidx, uidx, train_err, train_perp, valid_perp, test_perp,
valid_err, test_err, valid_B1, valid_B2, valid_B3,
valid_B4, valid_meteor, valid_Rouge, valid_Cider, test_B1,
test_B2, test_B3, test_B4, test_meteor, test_Rouge,
test_Cider
])
numpy.savetxt(save_model_dir + 'train_valid_test.txt',
history_errs,
fmt='%.3f')
print 'save validation results to %s' % save_model_dir
# save best model according to the best blue or meteor
if len(history_errs) > 1 and \
valid_B4 > numpy.array(history_errs)[:-1,11].max():
print 'Saving to %s...' % save_model_dir,
numpy.savez(save_model_dir +
'model_best_blue_or_meteor.npz',
history_errs=history_errs,
**best_p)
if len(history_errs) > 1 and \
valid_err < numpy.array(history_errs)[:-1,6].min():
best_p = utils.unzip(tparams)
bad_counter = 0
best_valid_err = valid_err
uidx_best_valid_err = uidx
print 'Saving to %s...' % save_model_dir,
numpy.savez(save_model_dir + 'model_best_so_far.npz',
history_errs=history_errs,
**best_p)
with open('%smodel_options.pkl' % save_model_dir,
'wb') as f:
pkl.dump(model_options, f)
print 'Done'
elif len(history_errs) > 1 and \
valid_err >= numpy.array(history_errs)[:-1,6].min():
bad_counter += 1
print 'history best ', numpy.array(history_errs)[:,
6].min()
print 'bad_counter ', bad_counter
print 'patience ', patience
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if test_B4 > 0.52 and test_meteor > 0.32:
print 'Saving to %s...' % save_model_dir,
numpy.savez(save_model_dir + 'model_' + str(uidx) + '.npz',
history_errs=history_errs,
**current_params)
print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err, \
'best valid err so far',best_valid_err
print 'valid took %.2f sec' % (time.time() - t0_valid)
# end of validatioin
if debug:
break
if estop:
break
if debug:
break
# end for loop over minibatches
print 'This epoch has seen %d samples, train cost %.2f' % (
n_samples, numpy.mean(train_costs))
# end for loop over epochs
print 'Optimization ended.'
if best_p is not None:
utils.zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = 0
test_err = 0
if not debug:
#if valid:
valid_err, valid_perp = model.pred_probs(
engine, 'valid', f_log_probs, verbose=model_options['verbose'])
#if test:
#test_err, test_perp = self.pred_probs(
# 'test', f_log_probs,
# verbose=model_options['verbose'])
print 'stopped at epoch %d, minibatch %d, '\
'curent Train %.2f, current Valid %.2f, current Test %.2f '%(
eidx,uidx,numpy.mean(train_err),numpy.mean(valid_err),numpy.mean(test_err))
params = copy.copy(best_p)
numpy.savez(save_model_dir + 'model_best.npz',
train_err=train_err,
valid_err=valid_err,
test_err=test_err,
history_errs=history_errs,
**params)
if history_errs != []:
history = numpy.asarray(history_errs)
best_valid_idx = history[:, 6].argmin()
numpy.savetxt(save_model_dir + 'train_valid_test.txt',
history,
fmt='%.4f')
print 'final best exp ', history[best_valid_idx]
return train_err, valid_err, test_err
def train(random_seed=1234,
dim_word=256, # word vector dimensionality
ctx_dim=-1, # context vector dimensionality, auto set
dim=1000, # the number of LSTM units
n_layers_out=1,
n_layers_init=1,
encoder='none',
encoder_dim=100,
prev2out=False,
ctx2out=False,
patience=10,
max_epochs=5000,
dispFreq=100,
decay_c=0.,
alpha_c=0.,
alpha_entropy_r=0.,
lrate=0.01,
selector=False,
n_words=100000,
maxlen=100, # maximum length of the description
optimizer='adadelta',
clip_c=2.,
batch_size = 64,
valid_batch_size = 64,
save_model_dir='/data/lisatmp3/yaoli/exp/capgen_vid/attention/test/',
validFreq=10,
saveFreq=10, # save the parameters after every saveFreq updates
sampleFreq=10, # generate some samples after every sampleFreq updates
metric='blue',
dataset='youtube2text',
video_feature='googlenet',
use_dropout=False,
reload_=False,
from_dir=None,
K1=10,
K2=10,
OutOf=240,
verbose=True,
debug=True
):
rng_numpy, rng_theano = utils.get_two_rngs()
model_options = locals().copy()
model_options_c = locals().copy()
if 'self' in model_options:
del model_options['self']
with open('model_files/model_options.pkl', 'wb') as f:
pkl.dump(model_options, f)
with open('model_files/model_options_c3d.pkl', 'wb') as f:
pkl.dump(model_options_c, f)
# instance model
layers = Layers()
model = Model()
model_c = Model()
print 'Loading data'
engine = data_engine.Movie2Caption('attention', dataset,
video_feature,
batch_size, valid_batch_size,
maxlen, n_words,
K1, K2, OutOf)
model_options['ctx_dim'] = engine.ctx_dim
model_options_c['ctx_dim'] = engine.ctx_dim_c
model_options['n_words'] = engine.n_words
model_options_c['n_words'] = engine.n_words
print 'n_words:', model_options['n_words']
print model_options_c['dim'],model_options_c['ctx_dim']
# set test values, for debugging
idx = engine.kf_train[0]
[x_tv, mask_tv,
ctx_tv, ctx_mask_tv,
ctx_tv_c, ctx_mask_tv_c] = data_engine.prepare_data(
engine, [engine.train[index] for index in idx])
print 'init params'
t0 = time.time()
params = model.init_params(model_options)
params_c = model_c.init_params(model_options_c)
# reloading
model_saved = 'model_files/model_resnet.npz'
model_saved_c = 'model_files/model_c3d.npz'
assert os.path.isfile(model_saved)
print "Reloading model params..."
params = utils.load_params(model_saved, params)
params_c = utils.load_params(model_saved_c, params_c)
tparams = utils.init_tparams(params)
tparams_c = utils.init_tparams(params_c)
trng, use_noise, \
x, mask, ctx, mask_ctx, \
cost, extra = \
model.build_model(tparams, model_options)
alphas = extra[1]
betas = extra[2]
trng_c, use_noise_c, \
x_c, mask_c, ctx_c, mask_ctx_c, \
cost_c, extra_c = \
model_c.build_model(tparams_c, model_options_c)
alphas_c = extra_c[1]
betas_c = extra_c[2]
print 'buliding sampler'
f_init, f_next = model.build_sampler(tparams, model_options, use_noise, trng)
f_init_c, f_next_c = model_c.build_sampler(tparams_c, model_options_c, use_noise_c, trng_c)
# before any regularizer
print 'building f_log_probs'
f_log_probs = theano.function([x, mask, ctx, mask_ctx], -cost,
profile=False, on_unused_input='ignore')
f_log_probs_c = theano.function([x_c, mask_c, ctx_c, mask_ctx_c], -cost_c,
profile=False, on_unused_input='ignore')
bad_counter = 0
processes = None
queue = None
rqueue = None
shared_params = None
uidx = 0
uidx_best_blue = 0
uidx_best_valid_err = 0
estop = False
best_p = utils.unzip(tparams)
best_blue_valid = 0
best_valid_err = 999
alphas_ratio = []
for eidx in xrange(max_epochs):
n_samples = 0
train_costs = []
grads_record = []
print 'Epoch ', eidx
for idx in engine.kf_train:
tags = [engine.train[index] for index in idx]
n_samples += len(tags)
use_noise.set_value(1.)
pd_start = time.time()
x, mask, ctx, ctx_mask, ctx_c, ctx_mask_c = data_engine.prepare_data(
engine, tags)
#print 'x:',x.shape,'ctx:',ctx.shape,'ctx_c:',ctx_c.shape
pd_duration = time.time() - pd_start
if x is None:
print 'Minibatch with zero sample under length ', maxlen
continue
if numpy.mod(uidx, saveFreq) == 0:
pass
if numpy.mod(uidx, sampleFreq) == 0:
use_noise.set_value(0.)
print '------------- sampling from train ----------'
x_s = x
mask_s = mask
ctx_s = ctx
ctx_s_c = ctx_c
ctx_mask_s = ctx_mask
ctx_mask_s_c = ctx_mask_c
model.sample_execute_ensemble(engine, model_options,model_options_c, tparams,tparams_c,
f_init,f_init_c, f_next,f_next_c, x_s, ctx_s,
ctx_mask_s, ctx_s_c, ctx_mask_s_c, trng)
print '------------- sampling from valid ----------'
idx = engine.kf_valid[numpy.random.randint(1, len(engine.kf_valid) - 1)]
tags = [engine.valid[index] for index in idx]
x_s, mask_s, ctx_s, mask_ctx_s, ctx_s_c,mask_ctx_s_c = data_engine.prepare_data(engine, tags)
model.sample_execute_ensemble(engine, model_options,model_options_c, tparams,tparams_c,
f_init, f_init_c, f_next, f_next_c, x_s, ctx_s,
mask_ctx_s, ctx_s_c, mask_ctx_s_c, trng)
if validFreq != -1 and numpy.mod(uidx, validFreq) == 0:
current_params = utils.unzip(tparams)
use_noise.set_value(0.)
train_err = -1
train_perp = -1
valid_err = -1
valid_perp = -1
test_err = -1
test_perp = -1
mean_ranking = 0
blue_t0 = time.time()
scores, processes, queue, rqueue, shared_params = \
metrics.compute_score_ensemble(
model_type='attention',
model_archive=current_params,
options=model_options,
options_c=model_options_c,
engine=engine,
save_dir=save_model_dir,
beam=5, n_process=5,
whichset='both',
on_cpu=False,
processes=processes, queue=queue, rqueue=rqueue,
shared_params=shared_params, metric=metric,
one_time=False,
f_init=f_init, f_init_c=f_init_c, f_next=f_next, f_next_c= f_next_c, model=model
)
'''
{'blue': {'test': [-1], 'valid': [77.7, 60.5, 48.7, 38.5, 38.3]},
'alternative_valid': {'Bleu_3': 0.40702270203174923,
'Bleu_4': 0.29276570520368456,
'CIDEr': 0.25247168210607884,
'Bleu_2': 0.529069629270047,
'Bleu_1': 0.6804308797115253,
'ROUGE_L': 0.51083584331688392},
'meteor': {'test': [-1], 'valid': [0.282787550236724]}}
'''
valid_B1 = scores['valid']['Bleu_1']
valid_B2 = scores['valid']['Bleu_2']
valid_B3 = scores['valid']['Bleu_3']
valid_B4 = scores['valid']['Bleu_4']
valid_Rouge = scores['valid']['ROUGE_L']
valid_Cider = scores['valid']['CIDEr']
valid_meteor = scores['valid']['METEOR']
test_B1 = scores['test']['Bleu_1']
test_B2 = scores['test']['Bleu_2']
test_B3 = scores['test']['Bleu_3']
test_B4 = scores['test']['Bleu_4']
test_Rouge = scores['test']['ROUGE_L']
test_Cider = scores['test']['CIDEr']
test_meteor = scores['test']['METEOR']
print 'computing meteor/blue score used %.4f sec, '\
'blue score: %.1f, meteor score: %.1f'%(
time.time()-blue_t0, valid_B4, valid_meteor)
if test_B4>0.52 and test_meteor>0.32:
print 'Saving to %s...'%save_model_dir,
numpy.savez(
save_model_dir+'model_'+str(uidx)+'.npz',
**current_params)
print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err, \
'best valid err so far',best_valid_err
print 'valid took %.2f sec'%(time.time() - t0_valid)
# end of validatioin
sys.exit()
if debug:
break
if estop:
break
if debug:
break
# end for loop over minibatches
print 'This epoch has seen %d samples, train cost %.2f'%(
n_samples, numpy.mean(train_costs))
# end for loop over epochs
print 'Optimization ended.'
if best_p is not None:
utils.zipp(best_p, tparams)
use_noise.set_value(0.)
valid_err = 0
test_err = 0
if not debug:
#if valid:
valid_err, valid_perp = model.pred_probs(
engine, 'valid', f_log_probs,
verbose=model_options['verbose'])
#if test:
#test_err, test_perp = self.pred_probs(
# 'test', f_log_probs,
# verbose=model_options['verbose'])
print 'stopped at epoch %d, minibatch %d, '\
'curent Train %.2f, current Valid %.2f, current Test %.2f '%(
eidx,uidx,numpy.mean(train_err),numpy.mean(valid_err),numpy.mean(test_err))
params = copy.copy(best_p)
numpy.savez(save_model_dir+'model_best.npz',
train_err=train_err,
valid_err=valid_err, test_err=test_err, history_errs=history_errs,
**params)
if history_errs != []:
history = numpy.asarray(history_errs)
best_valid_idx = history[:,6].argmin()
numpy.savetxt(save_model_dir+'train_valid_test.txt', history, fmt='%.4f')
print 'final best exp ', history[best_valid_idx]
return train_err, valid_err, test_err
def train(model_options,
dataset_name = 'MSVD',
cnn_name = 'ResNet50',
train_data_ids_path = config.MSVD_DATA_IDS_TRAIN_PATH,
val_data_ids_path = config.MSVD_DATA_IDS_VAL_PATH,
test_data_ids_path = config.MSVD_DATA_IDS_TEST_PATH,
vocab_path = config.MSVD_VOCAB_PATH,
reverse_vocab_path = config.MSVD_REVERSE_VOCAB_PATH,
mb_size_train = 64,
mb_size_test = 128,
train_caps_path = config.MSVD_VID_CAPS_TRAIN_PATH,
val_caps_path = config.MSVD_VID_CAPS_VAL_PATH,
test_caps_path = config.MSVD_VID_CAPS_TEST_PATH,
feats_dir = config.MSVD_FEATS_DIR,
save_dir = config.SAVE_DIR_PATH,
word_dim = 512, # word embeddings size
ctx_dim = 2048, # video cnn feature dimension
lstm_dim = 512, # lstm unit size
patience = 20,
max_epochs = 500,
decay_c = 1e-4,
alpha_entropy_r = 0.,
alpha_c = 0.70602,
clip_c = 10.,
lrate = 0.0001,
vocab_size = 20000, # n_words
maxlen_caption = 30, # max length of the descprition
optimizer = 'adadelta',
batch_size = 64, # for trees use 25
metric = 'everything', # set to perplexity on DVS # blue, meteor, or both
use_dropout = True,
selector = True,
ctx2out = True,
prev2out = True,
dispFreq = 10,
validFreq = 2000,
saveFreq = -1, # save the parameters after every saveFreq updates
sampleFreq = 100, # generate some samples after every sampleFreq updates
verbose = True,
debug = False,
reload_model = False,
from_dir = '',
ctx_frames = 28, # 26 when compare
random_seed = 1234,
beam_search = True
):
tf.set_random_seed(random_seed)
model = Model()
print 'loading data'
engine = data_engine.Movie2Caption(dataset_name,cnn_name,train_data_ids_path, val_data_ids_path, test_data_ids_path,
vocab_path, reverse_vocab_path, mb_size_train, mb_size_test, maxlen_caption,
train_caps_path, val_caps_path, test_caps_path, feats_dir)
model_options['ctx_dim'] = engine.ctx_dim
ctx_dim = engine.ctx_dim
model_options['vocab_size'] = engine.vocab_size
vocab_size = engine.vocab_size
print 'n_words:', model_options['vocab_size']
print 'ctx_dim:', model_options['ctx_dim']
utils.write_to_json(model_options, '%smodel_options.json'%save_dir)
# set test values, for debugging
idx = engine.kf_train[0]
x_tv, mask_tv, ctx_tv, ctx_mask_tv = data_engine.prepare_data(engine, [engine.train_data_ids[index] for index in idx], mode="train")
print 'init params'
t0 = time.time()
params = model.init_params(model_options)
# description string: #words x #samples
X = tf.placeholder(tf.int32, shape=(None, None), name='word_seq_x') # word seq input (t,m)
MASK = tf.placeholder(tf.float32, shape=(None, None), name='word_seq_mask') # (t,m)
# context: #samples x #annotations x dim
CTX = tf.placeholder(tf.float32, shape=(None, ctx_frames, ctx_dim), name='ctx')
CTX_MASK = tf.placeholder(tf.float32, shape=(None, ctx_frames), name='ctx_mask')
CTX_SAMPLER = tf.placeholder(tf.float32, shape=(ctx_frames, ctx_dim), name='ctx_sampler')
CTX_MASK_SAMPLER = tf.placeholder(tf.float32, shape=(ctx_frames), name='ctx_mask_sampler')
X_SAMPLER = tf.placeholder(tf.int32, shape=(None,), name='x_sampler') # DOUBT 1 or None ?
BO_INIT_STATE_SAMPLER = tf.placeholder(tf.float32, shape=(None,lstm_dim), name='bo_init_state_sampler')
TO_INIT_STATE_SAMPLER = tf.placeholder(tf.float32, shape=(None,lstm_dim), name='to_init_state_sampler')
BO_INIT_MEMORY_SAMPLER = tf.placeholder(tf.float32, shape=(None,lstm_dim), name='bo_init_memory_sampler')
TO_INIT_MEMORY_SAMPLER = tf.placeholder(tf.float32, shape=(None,lstm_dim), name='to_init_memory_sampler')
# create tensorflow variables
print 'buliding model'
tfparams = utils.init_tfparams(params)
use_noise, COST, extra = model.build_model(tfparams, model_options, X, MASK, CTX, CTX_MASK)
ALPHAS = extra[1] # (t,64,28)
BETAS = extra[2] # (t,64)
print 'buliding sampler'
f_init, f_next = model.build_sampler(tfparams, model_options, use_noise,
CTX_SAMPLER, CTX_MASK_SAMPLER, X_SAMPLER, BO_INIT_STATE_SAMPLER,
TO_INIT_STATE_SAMPLER, BO_INIT_MEMORY_SAMPLER, TO_INIT_MEMORY_SAMPLER)
print 'building f_log_probs'
f_log_probs = -COST
print 'check trainables'
wrt = utils.itemlist(tfparams, model_options)
trainables = tf.trainable_variables()
print len(wrt),len(trainables)
# assert len(wrt)==len(trainables)
COST = tf.reduce_mean(COST, name="LOSS")
if decay_c > 0.:
decay_c = tf.Variable(np.float32(decay_c), trainable=False, name='decay_c')
weight_decay = 0.
for vv in wrt:
weight_decay += tf.reduce_sum(vv ** 2)
weight_decay *= decay_c
COST += weight_decay
if alpha_c > 0.:
alpha_c = tf.Variable(np.float32(alpha_c), trainable=False, name='alpha_c')
alpha_reg = alpha_c * tf.reduce_mean(tf.reduce_sum(((1.-tf.reduce_sum(ALPHAS, axis=0))**2), axis=-1))
COST += alpha_reg
if alpha_entropy_r > 0:
alpha_entropy_r = tf.Variable(np.float32(alpha_entropy_r),
name='alpha_entropy_r')
alpha_reg_2 = alpha_entropy_r * tf.reduce_mean(tf.reduce_sum((-tf.add(ALPHAS *
tf.log(ALPHAS+1e-8),axis=-1)), axis=-1))
COST += alpha_reg_2
else:
alpha_reg_2 = tf.zeros_like(COST)
print 'building f_alpha'
f_alpha = [ALPHAS, BETAS]
print 'build train fns'
UPDATE_OPS = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(UPDATE_OPS):
optimizer = tf.train.AdadeltaOptimizer(learning_rate=1.0, rho=0.95, epsilon=1e-6).minimize(loss=COST, var_list=wrt)
# Initialize all variables
var_init = tf.global_variables_initializer()
# Ops to save and restore all the variables.
saver = tf.train.Saver()
print 'compilation took %.4f sec'%(time.time()-t0)
print 'Optimization'
history_errs = []
# reload history
if reload_model:
print 'loading history error...'
history_errs = np.load(from_dir+'model_best_so_far.npz')['history_errs'].tolist()
bad_counter = 0
processes = None
queue = None
rqueue = None
shared_params = None
uidx = 0
uidx_best_blue = 0
uidx_best_valid_err = 0
estop = False
# best_p = utils.unzip(tparams)
best_blue_valid = 0
best_valid_err = 999
alphas_ratio = []
train_err = -1
train_perp = -1
valid_err = -1
valid_perp = -1
test_err = -1
test_perp = -1
# Launch the graph
with tf.Session() as sess:
sess.run(var_init)
if reload_model:
print 'restoring model...'
saver.restore(sess, from_dir+"model_best_so_far.ckpt")
for eidx in xrange(max_epochs):
n_samples = 0
train_costs = []
grads_record = []
for idx in engine.kf_train:
tags = [engine.train_data_ids[index] for index in idx]
n_samples += len(tags)
uidx += 1
sess.run(tf.assign(use_noise, True))
pd_start = time.time()
x, mask, ctx, ctx_mask = data_engine.prepare_data(engine, tags, mode="train")
pd_duration = time.time() - pd_start
if x is None:
print 'Minibatch with zero sample under length ', maxlen
continue
ud_start = time.time()
sess.run(optimizer, feed_dict={
X: x,
MASK: mask,
CTX: ctx,
CTX_MASK: ctx_mask})
ud_duration = time.time() - ud_start
# writer = tf.summary.FileWriter("graph_cost", sess.graph)
cost = sess.run(COST, feed_dict={
X: x,
MASK: mask,
CTX: ctx,
CTX_MASK: ctx_mask})
# writer.close()
if np.isnan(cost) or np.isinf(cost):
print 'NaN detected in cost'
import pdb; pdb.set_trace()
if eidx == 0:
train_error = cost
else:
train_error = train_error * 0.95 + cost * 0.05
train_costs.append(cost)
if np.mod(uidx, dispFreq) == 0:
print 'Epoch: ', eidx, \
', Update: ', uidx, \
', train cost mean so far: ', train_error, \
', fetching data time spent (sec): ', pd_duration, \
', update time spent (sec): ', ud_duration, \
', save_dir: ', save_dir, '\n'
alphas, betas = sess.run(f_alpha, feed_dict={
X: x,
MASK: mask,
CTX: ctx,
CTX_MASK: ctx_mask})
counts = mask.sum(0)
betas_mean = (betas * mask).sum(0) / counts
betas_mean = betas_mean.mean()
print 'alpha ratio %.3f, betas mean %.3f\n'%(
alphas.min(-1).mean() / (alphas.max(-1)).mean(), betas_mean)
l = 0
for vv in x[:, 0]:
if vv == 0: # eos
break
if vv in engine.reverse_vocab:
print '(', np.round(betas[l, 0], 3), ')', engine.reverse_vocab[vv],
else:
print '(', np.round(betas[l, 0], 3), ')', 'UNK',
print ",",
l += 1
print '(', np.round(betas[l, 0], 3), ')\n'
if np.mod(uidx, saveFreq) == 0:
pass
if np.mod(uidx, sampleFreq) == 0:
sess.run(tf.assign(use_noise, False))
print '------------- sampling from train ----------'
x_s = x # (t,m)
mask_s = mask # (t,m)
ctx_s = ctx # (m,28,2048)
ctx_mask_s = ctx_mask # (m,28)
model.sample_execute(sess, engine, model_options, tfparams, f_init, f_next, x_s, ctx_s, ctx_mask_s)
print '------------- sampling from valid ----------'
# idx = engine.kf_val[np.random.randint(1, len(engine.kf_val) - 1)]
# tags = [engine.val_data_ids[index] for index in idx]
# x_s, mask_s, ctx_s, mask_ctx_s = data_engine.prepare_data(engine, tags,"val")
# model.sample_execute(sess, engine, model_options, tfparams, f_init, f_next, x_s, ctx_s, ctx_mask_s)
print ""
if validFreq != -1 and np.mod(uidx, validFreq) == 0:
t0_valid = time.time()
alphas, _ = sess.run(f_alpha, feed_dict={
X: x,
MASK: mask,
CTX: ctx,
CTX_MASK: ctx_mask})
ratio = alphas.min(-1).mean()/(alphas.max(-1)).mean()
alphas_ratio.append(ratio)
np.savetxt(save_dir+'alpha_ratio.txt',alphas_ratio)
np.savez(save_dir+'model_current.npz', history_errs=history_errs)
saver.save(sess, save_dir+'model_current.ckpt')
sess.run(tf.assign(use_noise, False))
train_err = -1
train_perp = -1
valid_err = -1
valid_perp = -1
test_err = -1
test_perp = -1
if not debug:
# first compute train cost
if 0:
print 'computing cost on trainset'
train_err, train_perp = model.pred_probs(sess, engine, 'train',
f_log_probs, verbose=model_options['verbose'])
else:
train_err = 0.
train_perp = 0.
if 1:
print 'validating...'
valid_err, valid_perp = model.pred_probs(sess, engine, 'val',
f_log_probs, verbose=model_options['verbose'])
else:
valid_err = 0.
valid_perp = 0.
if 0:
print 'testing...'
test_err, test_perp = model.pred_probs(sess, engine, 'test',
f_log_probs, verbose=model_options['verbose'])
else:
test_err = 0.
test_perp = 0.
mean_ranking = 0
blue_t0 = time.time()
scores, processes, queue, rqueue, shared_params = \
metrics.compute_score(sess=sess,
model_type='attention',
model_archive=None,
options=model_options,
engine=engine,
save_dir=save_dir,
beam=5, n_process=5,
whichset='both',
on_cpu=False,
processes=processes, queue=queue, rqueue=rqueue,
shared_params=shared_params, metric=metric,
one_time=False,
f_init=f_init, f_next=f_next, model=model
)
'''
{'blue': {'test': [-1], 'valid': [77.7, 60.5, 48.7, 38.5, 38.3]},
'alternative_valid': {'Bleu_3': 0.40702270203174923,
'Bleu_4': 0.29276570520368456,
'CIDEr': 0.25247168210607884,
'Bleu_2': 0.529069629270047,
'Bleu_1': 0.6804308797115253,
'ROUGE_L': 0.51083584331688392},
'meteor': {'test': [-1], 'valid': [0.282787550236724]}}
'''
valid_B1 = scores['valid']['Bleu_1']
valid_B2 = scores['valid']['Bleu_2']
valid_B3 = scores['valid']['Bleu_3']
valid_B4 = scores['valid']['Bleu_4']
valid_Rouge = scores['valid']['ROUGE_L']
valid_Cider = scores['valid']['CIDEr']
valid_meteor = scores['valid']['METEOR']
test_B1 = scores['test']['Bleu_1']
test_B2 = scores['test']['Bleu_2']
test_B3 = scores['test']['Bleu_3']
test_B4 = scores['test']['Bleu_4']
test_Rouge = scores['test']['ROUGE_L']
test_Cider = scores['test']['CIDEr']
test_meteor = scores['test']['METEOR']
print 'computing meteor/blue score used %.4f sec, '\
'blue score: %.1f, meteor score: %.1f'%(
time.time()-blue_t0, valid_B4, valid_meteor)
history_errs.append([eidx, uidx, train_err, train_perp,
valid_perp, test_perp,
valid_err, test_err,
valid_B1, valid_B2, valid_B3,
valid_B4, valid_meteor, valid_Rouge, valid_Cider,
test_B1, test_B2, test_B3,
test_B4, test_meteor, test_Rouge, test_Cider])
np.savetxt(save_dir+'train_valid_test.txt',
history_errs, fmt='%.3f')
print 'save validation results to %s'%save_dir
# save best model according to the best blue or meteor
if len(history_errs) > 1 and \
valid_B4 > np.array(history_errs)[:-1,11].max():
print 'Saving to %s...'%save_dir,
np.savez(
save_dir+'model_best_blue_or_meteor.npz',
history_errs=history_errs)
saver.save(sess, save_dir+'model_best_blue_or_meteor.ckpt') # DOUBT
if len(history_errs) > 1 and \
valid_err < np.array(history_errs)[:-1,6].min():
# best_p = utils.unzip(tparams) # DOUBT
bad_counter = 0
best_valid_err = valid_err
uidx_best_valid_err = uidx
print 'Saving to %s...'%save_dir,
np.savez(save_dir+'model_best_so_far.npz',
history_errs=history_errs)
saver.save(sess, save_dir+'model_best_so_far.ckpt')
utils.write_to_json(model_options, '%smodel_options.json'%save_dir)
print 'Done'
elif len(history_errs) > 1 and \
valid_err >= np.array(history_errs)[:-1,6].min():
bad_counter += 1
print 'history best ',np.array(history_errs)[:,6].min()
print 'bad_counter ',bad_counter
print 'patience ',patience
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
if test_B4>0.52 and test_meteor>0.32:
print 'Saving to %s...'%save_dir,
np.savez(
save_dir+'model_'+str(uidx)+'.npz',
history_errs=history_errs)
saver.save(sess, save_dir+'model_'+str(uidx)+'.ckpt')
print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err, \
'best valid err so far',best_valid_err
print 'valid took %.2f sec'%(time.time() - t0_valid)
# end of validatioin
if debug:
break
if estop:
break
if debug:
break
# end for loop over minibatches
print 'This epoch has seen %d samples, train cost %.2f'%(
n_samples, np.mean(train_costs))
# end for loop over epochs
print 'Optimization ended.'
print 'stopped at epoch %d, minibatch %d, '\
'curent Train %.2f, current Valid %.2f, current Test %.2f '%(
eidx,uidx,np.mean(train_err),np.mean(valid_err),np.mean(test_err))
if history_errs != []:
history = np.asarray(history_errs)
best_valid_idx = history[:,6].argmin()
np.savetxt(save_dir+'train_valid_test.txt', history, fmt='%.4f')
print 'final best exp ', history[best_valid_idx]
np.savez(
save_dir+'model_train_end.npz',
history_errs=history_errs)
saver.save(sess, save_dir+'model_train_end.ckpt')
return
