def load_model(path_to_model=default_model):
"""
Load all model components
"""
print path_to_model
# Load the worddict
print 'Loading dictionary...'
with open(path_to_model + '.dictionary.pkl', 'rb') as f:
worddict = pkl.load(f)
# Create inverted dictionary
print 'Creating inverted dictionary...'
word_idict = dict()
for kk, vv in worddict.iteritems():
word_idict[vv] = kk
word_idict[0] = '<eos>'
word_idict[1] = 'UNK'
# Load model options
print 'Loading model options...'
with open(path_to_model + '.pkl', 'rb') as f:
options = pkl.load(f)
# Load parameters
print 'Loading model parameters...'
params = init_params(options)
params = load_params(path_to_model, params)
tparams = init_tparams(params)
# Extractor functions
print 'Compiling image encoder...'
trng, [im], images = build_image_encoder(tparams, options)
f_ienc = theano.function([im], images, name='f_ienc')
print 'Compiling sentence encoder...'
trng = RandomStreams(1234)
trng, [x, x_mask], sentences = build_sentence_encoder(tparams, options)
f_senc = theano.function([x, x_mask], sentences, name='f_senc')
# Store everything we need in a dictionary
print 'Packing up...'
model = {}
model['options'] = options
model['worddict'] = worddict
model['word_idict'] = word_idict
model['f_senc'] = f_senc
model['f_ienc'] = f_ienc
return model
def load_model(path_to_model=default_model):
"""
Load all model components
"""
print path_to_model
# Load the worddict
print 'Loading dictionary...'
with open('%s.dictionary.pkl'%path_to_model, 'rb') as f:
worddict = pkl.load(f)
# Create inverted dictionary
print 'Creating inverted dictionary...'
word_idict = dict()
for kk, vv in worddict.iteritems():
word_idict[vv] = kk
word_idict[0] = '<eos>'
word_idict[1] = 'UNK'
# Load model options
print 'Loading model options...'
with open('%s.pkl'%path_to_model, 'rb') as f:
options = pkl.load(f)
# Load parameters
print 'Loading model parameters...'
params = init_params(options)
params = load_params(path_to_model, params)
tparams = init_tparams(params)
# Extractor functions
print 'Compiling sentence encoder...'
trng = RandomStreams(1234)
trng, [x, x_mask], sentences = build_sentence_encoder(tparams, options)
f_senc = theano.function([x, x_mask], sentences, name='f_senc')
print 'Compiling image encoder...'
trng, [im], images = build_image_encoder(tparams, options)
f_ienc = theano.function([im], images, name='f_ienc')
# Store everything we need in a dictionary
print 'Packing up...'
model = {}
model['options'] = options
model['worddict'] = worddict
model['word_idict'] = word_idict
model['f_senc'] = f_senc
model['f_ienc'] = f_ienc
return model
def load_model():
"""
Load all model components
"""
print path_to_model
# Load the worddict
print "Loading dictionary..."
with open("%s.dictionary.pkl" % path_to_model, "rb") as f:
worddict = pkl.load(f)
# Create inverted dictionary
print "Creating inverted dictionary..."
word_idict = dict()
for kk, vv in worddict.iteritems():
word_idict[vv] = kk
word_idict[0] = "<eos>"
word_idict[1] = "UNK"
# Load model options
print "Loading model options..."
with open("%s.pkl" % path_to_model, "rb") as f:
options = pkl.load(f)
# Load parameters
print "Loading model parameters..."
params = init_params(options)
params = load_params(path_to_model, params)
tparams = init_tparams(params)
# Extractor functions
print "Compiling sentence encoder..."
trng = RandomStreams(1234)
trng, [x, x_mask], sentences = build_sentence_encoder(tparams, options)
f_senc = theano.function([x, x_mask], sentences, name="f_senc")
print "Compiling image encoder..."
trng, [im], images = build_image_encoder(tparams, options)
f_ienc = theano.function([im], images, name="f_ienc")
# Store everything we need in a dictionary
print "Packing up..."
model = {}
model["options"] = options
model["worddict"] = worddict
model["word_idict"] = word_idict
model["f_senc"] = f_senc
model["f_ienc"] = f_ienc
return model
def load_model(save_dir, model_name, best=True):
"""
Load all model components
Input are only save_dir and model_name since it is assumed that filenames keep convention
"""
model_options = {}
model_options['save_dir'] = save_dir
model_options['model_name'] = model_name
# Load model
print 'Loading model'
opt_filename_reload = get_opt_filename(model_options, previous=True)
with open(opt_filename_reload, 'rb') as f:
model = pkl.load(f)
options = model['options']
# Load parameters
print 'Loading model parameters...'
params = init_params(options)
params_filename = get_npz_filename(model_options, best=best, previous=True)
params = load_params(params_filename, params)
tparams = init_tparams(params)
# Extractor functions
print 'Compiling sentence encoder...'
[x, x_mask], sentences = build_sentence_encoder(tparams, options)
f_senc = theano.function([x, x_mask], sentences, name='f_senc')
print 'Compiling image encoder...'
[im], images = build_image_encoder(tparams, options)
f_ienc = theano.function([im], images, name='f_ienc')
print 'Compiling error computation...'
[s, im], errs = build_errors(options)
f_err = theano.function([s, im], errs, name='f_err')
# Store everything we need in a dictionary
print 'Packing up...'
model['f_senc'] = f_senc
model['f_ienc'] = f_ienc
model['f_err'] = f_err
return model
def load_model_path(path_to_model):
"""
Load all model components
"""
print path_to_model
# Load model
print 'Loading model'
with open(path_to_model + '.pkl', 'rb') as f:
model = pkl.load(f)
options = model['options']
# Load parameters
print 'Loading model parameters...'
params = init_params(options)
params = load_params(path_to_model + '.npz', params)
tparams = init_tparams(params)
# Extractor functions
print 'Compiling sentence encoder...'
[x, x_mask], sentences = build_sentence_encoder(tparams, options)
f_senc = theano.function([x, x_mask], sentences, name='f_senc')
print 'Compiling image encoder...'
[im], images = build_image_encoder(tparams, options)
f_ienc = theano.function([im], images, name='f_ienc')
print 'Compiling error computation...'
[s, im], errs = build_errors(options)
f_err = theano.function([s, im], errs, name='f_err')
# Store everything we need in a dictionary
print 'Packing up...'
model['f_senc'] = f_senc
model['f_ienc'] = f_ienc
model['f_err'] = f_err
return model
def load_model(path_to_model):
"""
Load all model components
"""
print path_to_model
# Load model
print 'Loading model'
with open(path_to_model + '.pkl', 'rb') as f:
model = pkl.load(f)
options = model['options']
# Load parameters
print 'Loading model parameters...'
params = init_params(options)
params = load_params(path_to_model + '.npz', params)
tparams = init_tparams(params)
# Extractor functions
print 'Compiling sentence encoder...'
[x, x_mask], sentences = build_sentence_encoder(tparams, options)
f_senc = theano.function([x, x_mask], sentences, name='f_senc')
print 'Compiling image encoder...'
[im], images = build_image_encoder(tparams, options)
f_ienc = theano.function([im], images, name='f_ienc')
print 'Compiling error computation...'
[s, im], errs = build_errors(options)
f_err = theano.function([s,im], errs, name='f_err')
# Store everything we need in a dictionary
print 'Packing up...'
model['f_senc'] = f_senc
model['f_ienc'] = f_ienc
model['f_err'] = f_err
return model
def trainer(data='coco', #f8k, f30k, coco
margin=0.2,
dim=1024,
dim_image=4096,
dim_word=300,
encoder='gru', # gru OR bow
max_epochs=15,
dispFreq=10,
decay_c=0.,
grad_clip=2.,
maxlen_w=100,
optimizer='adam',
batch_size = 128,
saveto='/ais/gobi3/u/rkiros/uvsmodels/coco.npz',
validFreq=100,
lrate=0.0002,
reload_=False):
# Model options
model_options = {}
model_options['data'] = data
model_options['margin'] = margin
model_options['dim'] = dim
model_options['dim_image'] = dim_image
model_options['dim_word'] = dim_word
model_options['encoder'] = encoder
model_options['max_epochs'] = max_epochs
model_options['dispFreq'] = dispFreq
model_options['decay_c'] = decay_c
model_options['grad_clip'] = grad_clip
model_options['maxlen_w'] = maxlen_w
model_options['optimizer'] = optimizer
model_options['batch_size'] = batch_size
model_options['saveto'] = saveto
model_options['validFreq'] = validFreq
model_options['lrate'] = lrate
model_options['reload_'] = reload_
print model_options
# reload options
if reload_ and os.path.exists(saveto):
print 'reloading...' + saveto
with open('%s.pkl'%saveto, 'rb') as f:
models_options = pkl.load(f)
# Load training and development sets
print 'Loading dataset'
train, dev = load_dataset(data)[:2]
# Create and save dictionary
print 'Creating dictionary'
worddict = build_dictionary(train[0]+dev[0])[0]
n_words = len(worddict)
model_options['n_words'] = n_words
print 'Dictionary size: ' + str(n_words)
with open('%s.dictionary.pkl'%saveto, 'wb') as f:
pkl.dump(worddict, f)
# Inverse dictionary
word_idict = dict()
for kk, vv in worddict.iteritems():
word_idict[vv] = kk
word_idict[0] = '<eos>'
word_idict[1] = 'UNK'
print 'Building model'
params = init_params(model_options)
# reload parameters
if reload_ and os.path.exists(saveto):
params = load_params(saveto, params)
tparams = init_tparams(params)
trng, inps, cost = build_model(tparams, model_options)
# before any regularizer
print 'Building f_log_probs...',
f_log_probs = theano.function(inps, cost, profile=False)
print 'Done'
# weight decay, if applicable
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv ** 2).sum()
weight_decay *= decay_c
cost += weight_decay
# after any regularizer
print 'Building f_cost...',
f_cost = theano.function(inps, cost, profile=False)
print 'Done'
print 'Building sentence encoder'
trng, inps_se, sentences = build_sentence_encoder(tparams, model_options)
f_senc = theano.function(inps_se, sentences, profile=False)
print 'Building image encoder'
trng, inps_ie, images = build_image_encoder(tparams, model_options)
f_ienc = theano.function(inps_ie, images, profile=False)
print 'Building f_grad...',
grads = tensor.grad(cost, wrt=itemlist(tparams))
f_grad_norm = theano.function(inps, [(g**2).sum() for g in grads], profile=False)
f_weight_norm = theano.function([], [(t**2).sum() for k,t in tparams.iteritems()], profile=False)
if grad_clip > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(tensor.switch(g2 > (grad_clip**2),
g / tensor.sqrt(g2) * grad_clip,
g))
grads = new_grads
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
# (compute gradients), (updates parameters)
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, inps, cost)
print 'Optimization'
# Each sentence in the minibatch have same length (for encoder)
train_iter = homogeneous_data.HomogeneousData([train[0], train[1]], batch_size=batch_size, maxlen=maxlen_w)
uidx = 0
curr = 0.
n_samples = 0
for eidx in xrange(max_epochs):
print 'Epoch ', eidx
for x, im in train_iter:
n_samples += len(x)
uidx += 1
x, mask, im = homogeneous_data.prepare_data(x, im, worddict, maxlen=maxlen_w, n_words=n_words)
if x == None:
print 'Minibatch with zero sample under length ', maxlen_w
uidx -= 1
continue
# Update
ud_start = time.time()
cost = f_grad_shared(x, mask, im)
f_update(lrate)
ud = time.time() - ud_start
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
return 1., 1., 1.
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud
if numpy.mod(uidx, validFreq) == 0:
print 'Computing results...'
curr_model = {}
curr_model['options'] = model_options
curr_model['worddict'] = worddict
curr_model['word_idict'] = word_idict
curr_model['f_senc'] = f_senc
curr_model['f_ienc'] = f_ienc
ls = encode_sentences(curr_model, dev[0])
lim = encode_images(curr_model, dev[1])
(r1, r5, r10, medr) = i2t(lim, ls)
print "Image to text: %.1f, %.1f, %.1f, %.1f" % (r1, r5, r10, medr)
(r1i, r5i, r10i, medri) = t2i(lim, ls)
print "Text to image: %.1f, %.1f, %.1f, %.1f" % (r1i, r5i, r10i, medri)
currscore = r1 + r5 + r10 + r1i + r5i + r10i
if currscore > curr:
curr = currscore
# Save model
print 'Saving...',
params = unzip(tparams)
numpy.savez(saveto, **params)
pkl.dump(model_options, open('%s.pkl'%saveto, 'wb'))
print 'Done'
print 'Seen %d samples'%n_samples
def trainer(
data='coco', #f8k, f30k, coco
margin=0.2,
dim=1024,
dim_image=4096,
dim_word=300,
encoder='gru', # gru OR bow
max_epochs=15,
dispFreq=10,
decay_c=0.,
grad_clip=2.,
maxlen_w=100,
optimizer='adam',
batch_size=128,
saveto='/ais/gobi3/u/rkiros/uvsmodels/coco.npz',
validFreq=100,
lrate=0.0002,
reload_=False):
# Model options
model_options = {}
model_options['data'] = data
model_options['margin'] = margin
model_options['dim'] = dim
model_options['dim_image'] = dim_image
model_options['dim_word'] = dim_word
model_options['encoder'] = encoder
model_options['max_epochs'] = max_epochs
model_options['dispFreq'] = dispFreq
model_options['decay_c'] = decay_c
model_options['grad_clip'] = grad_clip
model_options['maxlen_w'] = maxlen_w
model_options['optimizer'] = optimizer
model_options['batch_size'] = batch_size
model_options['saveto'] = saveto
model_options['validFreq'] = validFreq
model_options['lrate'] = lrate
model_options['reload_'] = reload_
print(model_options)
# reload options
if reload_ and os.path.exists(saveto):
print('reloading...' + saveto)
with open('%s.pkl' % saveto, 'rb') as f:
models_options = pkl.load(f)
# Load training and development sets
print('Loading dataset')
train, dev = load_dataset(data)[:2]
# Create and save dictionary
print('Creating dictionary')
worddict = build_dictionary(train[0] + dev[0])[0]
n_words = len(worddict)
model_options['n_words'] = n_words
print('Dictionary size: ' + str(n_words))
with open('%s.dictionary.pkl' % saveto, 'wb') as f:
pkl.dump(worddict, f)
# Inverse dictionary
word_idict = dict()
for kk, vv in worddict.iteritems():
word_idict[vv] = kk
word_idict[0] = '<eos>'
word_idict[1] = 'UNK'
print('Building model')
params = init_params(model_options)
# reload parameters
if reload_ and os.path.exists(saveto):
params = load_params(saveto, params)
tparams = init_tparams(params)
trng, inps, cost = build_model(tparams, model_options)
# before any regularizer
print('Building f_log_probs...', )
f_log_probs = theano.function(inps, cost, profile=False)
print('Done')
# weight decay, if applicable
if decay_c > 0.:
decay_c = theano.shared(numpy.float32(decay_c), name='decay_c')
weight_decay = 0.
for kk, vv in tparams.iteritems():
weight_decay += (vv**2).sum()
weight_decay *= decay_c
cost += weight_decay
# after any regularizer
print('Building f_cost...', )
f_cost = theano.function(inps, cost, profile=False)
print('Done')
print('Building sentence encoder')
trng, inps_se, sentences = build_sentence_encoder(tparams, model_options)
f_senc = theano.function(inps_se, sentences, profile=False)
print('Building image encoder')
trng, inps_ie, images = build_image_encoder(tparams, model_options)
f_ienc = theano.function(inps_ie, images, profile=False)
print('Building f_grad...', )
grads = tensor.grad(cost, wrt=itemlist(tparams))
f_grad_norm = theano.function(inps, [(g**2).sum() for g in grads],
profile=False)
f_weight_norm = theano.function([], [(t**2).sum()
for k, t in tparams.iteritems()],
profile=False)
if grad_clip > 0.:
g2 = 0.
for g in grads:
g2 += (g**2).sum()
new_grads = []
for g in grads:
new_grads.append(
tensor.switch(g2 > (grad_clip**2),
g / tensor.sqrt(g2) * grad_clip, g))
grads = new_grads
lr = tensor.scalar(name='lr')
print('Building optimizers...', )
# (compute gradients), (updates parameters)
f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, inps, cost)
print('Optimization')
# Each sentence in the minibatch have same length (for encoder)
train_iter = homogeneous_data.HomogeneousData([train[0], train[1]],
batch_size=batch_size,
maxlen=maxlen_w)
uidx = 0
curr = 0.
n_samples = 0
for eidx in xrange(max_epochs):
print('Epoch ', eidx)
for x, im in train_iter:
n_samples += len(x)
uidx += 1
x, mask, im = homogeneous_data.prepare_data(x,
im,
worddict,
maxlen=maxlen_w,
n_words=n_words)
if x == None:
print('Minibatch with zero sample under length ', maxlen_w)
uidx -= 1
continue
# Update
ud_start = time.time()
cost = f_grad_shared(x, mask, im)
f_update(lrate)
ud = time.time() - ud_start
if numpy.isnan(cost) or numpy.isinf(cost):
print('NaN detected')
return 1., 1., 1.
if numpy.mod(uidx, dispFreq) == 0:
print('Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ',
ud)
if numpy.mod(uidx, validFreq) == 0:
print('Computing results...')
curr_model = {}
curr_model['options'] = model_options
curr_model['worddict'] = worddict
curr_model['word_idict'] = word_idict
curr_model['f_senc'] = f_senc
curr_model['f_ienc'] = f_ienc
ls = encode_sentences(curr_model, dev[0])
lim = encode_images(curr_model, dev[1])
(r1, r5, r10, medr) = i2t(lim, ls)
print("Image to text: %.1f, %.1f, %.1f, %.1f" %
(r1, r5, r10, medr))
(r1i, r5i, r10i, medri) = t2i(lim, ls)
print("Text to image: %.1f, %.1f, %.1f, %.1f" %
(r1i, r5i, r10i, medri))
currscore = r1 + r5 + r10 + r1i + r5i + r10i
if currscore > curr:
curr = currscore
# Save model
print('Saving...', )
params = unzip(tparams)
numpy.savez(saveto, **params)
pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'))
print('Done')
print('Seen %d samples' % n_samples)
def trainer(load_from=None, save_dir='snapshots', name='anon', **kwargs):
"""
:param load_from: location to load parameters + options from
:param name: name of model, used as location to save parameters + options
"""
curr_model = dict()
# load old model, including parameters, but overwrite with new options
if load_from:
print 'reloading...' + load_from
with open('%s.pkl' % load_from, 'rb') as f:
curr_model = pkl.load(f)
else:
curr_model['options'] = {}
for k, v in kwargs.iteritems():
curr_model['options'][k] = v
model_options = curr_model['options']
# initialize logger
import datetime
timestampedName = datetime.datetime.now().strftime(
'%Y_%m_%d_%H_%M_%S') + '_' + name
from logger import Log
log = Log(name=timestampedName,
hyperparams=model_options,
saveDir='vis/training',
xLabel='Examples Seen',
saveFrequency=1)
print curr_model['options']
# Load training and development sets
print 'Loading dataset'
dataset = load_dataset(model_options['data'],
cnn=model_options['cnn'],
load_train=True)
train = dataset['train']
dev = dataset['dev']
# Create dictionary
print 'Creating dictionary'
worddict = build_dictionary(train['caps'] + dev['caps'])
print 'Dictionary size: ' + str(len(worddict))
curr_model['worddict'] = worddict
curr_model['options']['n_words'] = len(worddict) + 2
# save model
pkl.dump(curr_model, open('%s/%s.pkl' % (save_dir, name), 'wb'))
print 'Loading data'
train_iter = datasource.Datasource(train,
batch_size=model_options['batch_size'],
worddict=worddict)
dev = datasource.Datasource(dev, worddict=worddict)
dev_caps, dev_ims = dev.all()
print 'Building model'
params = init_params(model_options)
# reload parameters
if load_from is not None and os.path.exists(load_from):
params = load_params(load_from, params)
tparams = init_tparams(params)
inps, cost = build_model(tparams, model_options)
print 'Building sentence encoder'
inps_se, sentences = build_sentence_encoder(tparams, model_options)
f_senc = theano.function(inps_se, sentences, profile=False)
print 'Building image encoder'
inps_ie, images = build_image_encoder(tparams, model_options)
f_ienc = theano.function(inps_ie, images, profile=False)
print 'Building f_grad...',
grads = tensor.grad(cost, wrt=itemlist(tparams))
print 'Building errors..'
inps_err, errs = build_errors(model_options)
f_err = theano.function(inps_err, errs, profile=False)
curr_model['f_senc'] = f_senc
curr_model['f_ienc'] = f_ienc
curr_model['f_err'] = f_err
if model_options['grad_clip'] > 0.:
grads = [maxnorm(g, model_options['grad_clip']) for g in grads]
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
# (compute gradients), (updates parameters)
f_grad_shared, f_update = eval(model_options['optimizer'])(lr, tparams,
grads, inps,
cost)
print 'Optimization'
uidx = 0
curr = 0
n_samples = 0
for eidx in xrange(model_options['max_epochs']):
print 'Epoch ', eidx
for x, mask, im in train_iter:
n_samples += x.shape[1]
uidx += 1
# Update
ud_start = time.time()
cost = f_grad_shared(x, mask, im)
f_update(model_options['lrate'])
ud = time.time() - ud_start
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
return 1., 1., 1.
if numpy.mod(uidx, model_options['dispFreq']) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud
log.update({'Error': float(cost)}, n_samples)
if numpy.mod(uidx, model_options['validFreq']) == 0:
print 'Computing results...'
# encode sentences efficiently
dev_s = encode_sentences(
curr_model,
dev_caps,
batch_size=model_options['batch_size'])
dev_i = encode_images(curr_model, dev_ims)
# compute errors
dev_errs = compute_errors(curr_model, dev_s, dev_i)
# compute ranking error
(r1, r5, r10, medr, meanr), vis_details = t2i(dev_errs,
vis_details=True)
(r1i, r5i, r10i, medri, meanri) = i2t(dev_errs)
print "Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % (
r1, r5, r10, medr, meanr)
log.update(
{
'R@1': r1,
'R@5': r5,
'R@10': r10,
'median_rank': medr,
'mean_rank': meanr
}, n_samples)
print "Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % (
r1i, r5i, r10i, medri, meanri)
log.update(
{
'Image2Caption_R@1': r1i,
'Image2Caption_R@5': r5i,
'Image2CaptionR@10': r10i,
'Image2Caption_median_rank': medri,
'Image2Caption_mean_rank': meanri
}, n_samples)
tot = r1 + r5 + r10
if tot > curr:
curr = tot
# Save parameters
print 'Saving...',
numpy.savez('%s/%s' % (save_dir, name), **unzip(tparams))
print 'Done'
vis_details['hyperparams'] = model_options
# Save visualization details
with open(
'vis/roc/%s/%s.json' %
(model_options['data'], timestampedName), 'w') as f:
json.dump(vis_details, f)
# Add the new model to the index
try:
index = json.load(open('vis/roc/index.json', 'r'))
except IOError:
index = {model_options['data']: []}
models = index[model_options['data']]
if timestampedName not in models:
models.append(timestampedName)
with open('vis/roc/index.json', 'w') as f:
json.dump(index, f)
print 'Seen %d samples' % n_samples
def trainer(load_from=None, save_dir="snapshots", name="anon", **kwargs):
"""
:param load_from: location to load parameters + options from
:param name: name of model, used as location to save parameters + options
"""
curr_model = dict()
# load old model, including parameters, but overwrite with new options
if load_from:
print "reloading..." + load_from
with open("%s.pkl" % load_from, "rb") as f:
curr_model = pkl.load(f)
else:
curr_model["options"] = {}
for k, v in kwargs.iteritems():
curr_model["options"][k] = v
model_options = curr_model["options"]
# initialize logger
import datetime
timestampedName = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + "_" + name
from logger import Log
log = Log(
name=timestampedName, hyperparams=model_options, saveDir="vis/training", xLabel="Examples Seen", saveFrequency=1
)
print curr_model["options"]
# Load training and development sets
print "Loading dataset"
dataset = load_dataset(model_options["data"], cnn=model_options["cnn"], load_train=True)
train = dataset["train"]
dev = dataset["dev"]
# Create dictionary
print "Creating dictionary"
worddict = build_dictionary(train["caps"] + dev["caps"])
print "Dictionary size: " + str(len(worddict))
curr_model["worddict"] = worddict
curr_model["options"]["n_words"] = len(worddict) + 2
# save model
pkl.dump(curr_model, open("%s/%s.pkl" % (save_dir, name), "wb"))
print "Loading data"
train_iter = datasource.Datasource(train, batch_size=model_options["batch_size"], worddict=worddict)
dev = datasource.Datasource(dev, worddict=worddict)
dev_caps, dev_ims = dev.all()
print "Building model"
params = init_params(model_options)
# reload parameters
if load_from is not None and os.path.exists(load_from):
params = load_params(load_from, params)
tparams = init_tparams(params)
inps, cost = build_model(tparams, model_options)
print "Building sentence encoder"
inps_se, sentences = build_sentence_encoder(tparams, model_options)
f_senc = theano.function(inps_se, sentences, profile=False)
print "Building image encoder"
inps_ie, images = build_image_encoder(tparams, model_options)
f_ienc = theano.function(inps_ie, images, profile=False)
print "Building f_grad...",
grads = tensor.grad(cost, wrt=itemlist(tparams))
print "Building errors.."
inps_err, errs = build_errors(model_options)
f_err = theano.function(inps_err, errs, profile=False)
curr_model["f_senc"] = f_senc
curr_model["f_ienc"] = f_ienc
curr_model["f_err"] = f_err
if model_options["grad_clip"] > 0.0:
grads = [maxnorm(g, model_options["grad_clip"]) for g in grads]
lr = tensor.scalar(name="lr")
print "Building optimizers...",
# (compute gradients), (updates parameters)
f_grad_shared, f_update = eval(model_options["optimizer"])(lr, tparams, grads, inps, cost)
print "Optimization"
uidx = 0
curr = 0
n_samples = 0
for eidx in xrange(model_options["max_epochs"]):
print "Epoch ", eidx
for x, mask, im in train_iter:
n_samples += x.shape[1]
uidx += 1
# Update
ud_start = time.time()
cost = f_grad_shared(x, mask, im)
f_update(model_options["lrate"])
ud = time.time() - ud_start
if numpy.isnan(cost) or numpy.isinf(cost):
print "NaN detected"
return 1.0, 1.0, 1.0
if numpy.mod(uidx, model_options["dispFreq"]) == 0:
print "Epoch ", eidx, "Update ", uidx, "Cost ", cost, "UD ", ud
log.update({"Error": float(cost)}, n_samples)
if numpy.mod(uidx, model_options["validFreq"]) == 0:
print "Computing results..."
# encode sentences efficiently
dev_s = encode_sentences(curr_model, dev_caps, batch_size=model_options["batch_size"])
dev_i = encode_images(curr_model, dev_ims)
# compute errors
dev_errs = compute_errors(curr_model, dev_s, dev_i)
# compute ranking error
(r1, r5, r10, medr, meanr), vis_details = t2i(dev_errs, vis_details=True)
(r1i, r5i, r10i, medri, meanri) = i2t(dev_errs)
print "Text to image (dev set): %.1f, %.1f, %.1f, %.1f, %.1f" % (r1, r5, r10, medr, meanr)
log.update({"R@1": r1, "R@5": r5, "R@10": r10, "median_rank": medr, "mean_rank": meanr}, n_samples)
print "Image to text (dev set): %.1f, %.1f, %.1f, %.1f, %.1f" % (r1i, r5i, r10i, medri, meanri)
log.update(
{
"Image2Caption_R@1": r1i,
"Image2Caption_R@5": r5i,
"Image2CaptionR@10": r10i,
"Image2Caption_median_rank": medri,
"Image2Caption_mean_rank": meanri,
},
n_samples,
)
tot = r1 + r5 + r10
if tot > curr:
curr = tot
# Save parameters
print "Saving...",
numpy.savez("%s/%s" % (save_dir, name), **unzip(tparams))
print "Done"
vis_details["hyperparams"] = model_options
# Save visualization details
with open("vis/roc/%s/%s.json" % (model_options["data"], timestampedName), "w") as f:
json.dump(vis_details, f)
# Add the new model to the index
try:
index = json.load(open("vis/roc/index.json", "r"))
except IOError:
index = {model_options["data"]: []}
models = index[model_options["data"]]
if timestampedName not in models:
models.append(timestampedName)
with open("vis/roc/index.json", "w") as f:
json.dump(index, f)
print "Seen %d samples" % n_samples
def load_model(path_to_model):
"""
Load all model components
"""
print path_to_model
# Load model
print 'Loading model'
with open(path_to_model + '.pkl', 'rb') as f:
model = pkl.load(f)
options = model['options']
options['use_topic'] = True
# Load parameters
print 'Loading model parameters...'
params = init_params(options)
params = load_params(path_to_model + '.npz', params)
tparams = init_tparams(params)
# Extractor functions
print 'Compiling sentence encoder...'
[x, x_mask], sentences = build_sentence_encoder(tparams, options)
f_senc = theano.function([x, x_mask], sentences, name='f_senc')
#print 'Compiling sentence encoder with topics...'
#[x, x_mask, topics], sentences = build_sentence_encoder_with_topicvector(tparams, options)
#f_senc_t = theano.function([x, x_mask, topics], sentences, name='f_senc_t')
print 'Compiling image encoder...'
[im], images = build_image_encoder(tparams, options)
f_ienc = theano.function([im], images, name='f_ienc')
print 'Compiling topic encoder...'
[t], topics = build_topic_encoder(tparams, options)
f_tenc = theano.function([t], topics, name='f_tenc')
'''
print 'Compiling topic_vector1 encoder...'
[t], topics = build_topic_vector1_encoder(tparams, options)
f_tv1enc = theano.function([t], topics, name='f_tv1enc')
print 'Compiling topic_vector2 encoder...'
[t], topics = build_topic_encoder(tparams, options)
f_tv2enc = theano.function([t], topics, name='f_tv2enc')
'''
print 'Compiling error computation...'
[s, im], errs = build_errors(options)
f_err = theano.function([s, im], errs, name='f_err')
'''
[s, im, t], errs_t1 = build_errors_3level(options)
f_err_t1 = theano.function([s,im,t], errs_t1, name='f_err_t1')
[s_t, im_t], errs_t2 = build_errors_t2(options)
f_err_t2 = theano.function([s_t, im_t], errs_t2, name='f_err_t2')
'''
# Store everything we need in a dictionary
print 'Packing up...'
model['f_senc'] = f_senc
#model['f_senc_t'] = f_senc_t
model['f_ienc'] = f_ienc
model['f_tenc'] = f_tenc
#model['f_tv1enc'] = f_tv1enc
#model['f_tv2enc'] = f_tv2enc
model['f_err'] = f_err
#model['f_err_t1'] = f_err_t1
#model['f_err_t2'] = f_err_t2
return model
def trainer(**kwargs):
"""
Train the model according to input params
Info about input params is available in parameters.py
"""
# Timing
print('Starting time:', datetime.now())
sys.stdout.flush()
t_start_train = time.time()
# Model options
# load old model, including parameters, but overwrite with new options
# Extract model options from arguments
model_options = {}
for k, v in kwargs.iteritems():
model_options[k] = v
# Print input options
print('PARAMETERS BEFORE LOADING:')
for k, v in model_options.items():
print('{:>26}: {}'.format(k, v))
sys.stdout.flush()
# Reload options if required
curr_model = dict()
if model_options['reload_']:
# Reload model parameters
opt_filename_reload = get_opt_filename(model_options, previous=True)
print('reloading...', opt_filename_reload)
sys.stdout.flush()
try:
with open(opt_filename_reload, 'rb') as f:
curr_model = pkl.load(f)
except:
print(
'Failed to reload parameters, try to use only feeded parameters'
)
curr_model['options'] = {}
# Check if we reload from best model or last model
if model_options['load_from'] in ['Best', 'best', 'B', 'b']:
load_from_best = True
print('Loading from Best saved model in validation results')
elif model_options['load_from'] in ['Last', 'last', 'L', 'l']:
load_from_best = False
print('Loading from Last saved model')
else:
print('Unkown choice for "load_from" parameter',
model_options['load_from'])
print('Please choose one of:', ['Best', 'best', 'B', 'b'],
['Last', 'last', 'L', 'l'])
print('Using Last as default')
load_from_best = False
# Reload end-point parameters
state_filename = get_sol_filename(model_options,
best=load_from_best,
previous=True)
print('reloading...', state_filename)
sys.stdout.flush()
try:
with open(state_filename, 'rb') as f:
state_params = pkl.load(f)
if load_from_best:
init_epoch = state_params['epoch']
solution = state_params
else:
init_epoch = state_params['epoch_done'] + 1
solution = state_params['solution']
best_val_score = solution['best_val_score']
n_samples = solution['samples_seen']
except:
print('Failed to reload state parameters, starting from 0')
init_epoch = 0
best_val_score = 0
n_samples = 0
else:
curr_model['options'] = {}
init_epoch = 0
best_val_score = 0
n_samples = 0
# Overwrite loaded options with input options
for k, v in kwargs.iteritems():
curr_model['options'][k] = v
model_options = curr_model['options']
# Print final options loaded
if model_options['reload_']:
print('PARAMETERS AFTER LOADING:')
for k, v in model_options.items():
print('{:>26}: {}'.format(k, v))
sys.stdout.flush()
# Load training and development sets
print('Loading dataset')
sys.stdout.flush()
dataset = load_dataset(dataset_name=model_options['data'],
embedding=model_options['embedding'],
path_to_data=model_options['data_path'],
test_subset=model_options['test_subset'],
load_train=True,
fold=0)
train = dataset['train']
dev = dataset['val']
# Create word dictionary
print('Creating dictionary')
sys.stdout.flush()
worddict = build_dictionary(train['caps'] + dev['caps'])
print('Dictionary size: ' + str(len(worddict)))
sys.stdout.flush()
curr_model['worddict'] = worddict
curr_model['options']['n_words'] = len(worddict) + 2
# save model
opt_filename_save = get_opt_filename(model_options, previous=False)
print('Saving model parameters in', opt_filename_save)
sys.stdout.flush()
try:
os.makedirs(os.path.dirname(opt_filename_save))
except:
pass
pkl.dump(curr_model, open(opt_filename_save, 'wb'))
# Load data from dataset
print('Loading data')
sys.stdout.flush()
train_iter = datasource.Datasource(train,
batch_size=model_options['batch_size'],
worddict=worddict)
dev = datasource.Datasource(dev, worddict=worddict)
dev_caps, dev_ims = dev.all()
print('Building model')
sys.stdout.flush()
params = init_params(model_options)
# reload network parameters, ie. weights
if model_options['reload_']:
params_filename = get_npz_filename(model_options,
best=load_from_best,
previous=True)
params = load_params(params_filename, params)
tparams = init_tparams(params)
inps, cost = build_model(tparams, model_options)
print('Building sentence encoder')
sys.stdout.flush()
inps_se, sentences = build_sentence_encoder(tparams, model_options)
f_senc = theano.function(inps_se, sentences, profile=False)
print('Building image encoder')
sys.stdout.flush()
inps_ie, images = build_image_encoder(tparams, model_options)
f_ienc = theano.function(inps_ie, images, profile=False)
print('Building f_grad...')
sys.stdout.flush()
grads = tensor.grad(cost, wrt=itemlist(tparams))
print('Building errors...')
sys.stdout.flush()
inps_err, errs = build_errors(model_options)
f_err = theano.function(inps_err, errs, profile=False)
curr_model['f_senc'] = f_senc
curr_model['f_ienc'] = f_ienc
curr_model['f_err'] = f_err
if model_options['grad_clip'] > 0.:
grads = [maxnorm(g, model_options['grad_clip']) for g in grads]
lr = tensor.scalar(name='lr')
print('Building optimizers...')
sys.stdout.flush()
# (compute gradients), (updates parameters)
f_grad_shared, f_update = eval(model_options['optimizer'])(lr, tparams,
grads, inps,
cost)
# Get names for the files to save model and solution
sol_filename_best = get_sol_filename(model_options,
best=True,
previous=False)
sol_filename_last = get_sol_filename(model_options,
best=False,
previous=False)
params_filename_best = get_npz_filename(model_options,
best=True,
previous=False)
params_filename_last = get_npz_filename(model_options,
best=False,
previous=False)
print('PATHS TO MODELS:')
for filename in [
sol_filename_best, sol_filename_last, params_filename_best,
params_filename_last
]:
print(filename)
sys.stdout.flush()
try:
os.makedirs(os.path.dirname(filename))
except:
pass
# Start optimization
print('Optimization')
sys.stdout.flush()
uidx = 0
# Timing
t_start = time.time()
print('Starting time:', datetime.now())
for eidx in range(init_epoch, model_options['max_epochs']):
t_start_epoch = time.time()
print('Epoch ', eidx)
sys.stdout.flush()
for x, mask, im in train_iter:
n_samples += x.shape[1]
uidx += 1
# Update
ud_start = time.time()
cost = f_grad_shared(x, mask, im)
f_update(model_options['lrate'])
ud = time.time() - ud_start
if numpy.isnan(cost) or numpy.isinf(cost):
print('NaN detected')
sys.stdout.flush()
return 1., 1., 1.
if numpy.mod(uidx, model_options['dispFreq']) == 0:
print('Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ',
ud)
sys.stdout.flush()
if numpy.mod(uidx, model_options['validFreq']) == 0:
print('Computing results...')
sys.stdout.flush()
# encode sentences efficiently
dev_s = encode_sentences(
curr_model,
dev_caps,
batch_size=model_options['batch_size'])
dev_i = encode_images(curr_model, dev_ims)
# compute errors
dev_errs = compute_errors(curr_model, dev_s, dev_i)
# compute ranking error
(r1, r5, r10, medr, meanr) = i2t(dev_errs)
(r1i, r5i, r10i, medri, meanri) = t2i(dev_errs)
print("Text to image (dev set): %.1f, %.1f, %.1f, %.1f, %.1f" %
(r1i, r5i, r10i, medri, meanri))
sys.stdout.flush()
print("Image to text (dev set): %.1f, %.1f, %.1f, %.1f, %.1f" %
(r1, r5, r10, medr, meanr))
sys.stdout.flush()
# Score
val_score = r1 + r5 + r10 + r1i + r5i + r10i
if val_score > best_val_score:
print('BEST MODEL FOUND')
print('Score:', val_score)
print('Previous best score:', best_val_score)
best_val_score = val_score
# Join in a results dict
results_dict = build_results_dict(r1, r5, r10, medr, r1i,
r5i, r10i, medri)
# Save parameters
print('Saving...', end=' ')
sys.stdout.flush()
numpy.savez(params_filename_best, **unzip(tparams))
print('Done')
sys.stdout.flush()
# Update solution
solution = OrderedDict([
('epoch', eidx), ('update', uidx),
('samples_seen', n_samples),
('best_val_score', best_val_score),
('best_val_res', results_dict),
('time_until_results',
str(timedelta(seconds=(time.time() - t_start_train))))
])
pkl.dump(solution, open(sol_filename_best, 'wb'))
print('Seen %d samples' % n_samples)
sys.stdout.flush()
# Timing
t_epoch = time.time() - t_start_epoch
t_epoch_avg = (time.time() - t_start) / (eidx + 1 - (init_epoch))
print('Time for this epoch:', str(timedelta(seconds=t_epoch)),
'Average:', str(timedelta(seconds=t_epoch_avg)))
t_2_complete = t_epoch_avg * (model_options['max_epochs'] - (eidx + 1))
print('Time since start session:',
str(timedelta(seconds=time.time() - t_start)),
'Estimated time to complete training:',
str(timedelta(seconds=t_2_complete)))
print('Current time:', datetime.now())
sys.stdout.flush()
# Save current model
try:
state_params = OrderedDict([('epoch_done', eidx),
('solution', solution)])
except:
solution = OrderedDict([
('epoch', eidx), ('update', uidx), ('samples_seen', n_samples),
('best_val_score', best_val_score),
('time_until_results',
str(timedelta(seconds=(time.time() - t_start_train))))
])
state_params = OrderedDict([('epoch_done', eidx),
('solution', solution)])
pkl.dump(state_params, open(sol_filename_last, 'wb'))
# Save parameters
print('Saving LAST npz...', end=' ')
sys.stdout.flush()
numpy.savez(params_filename_last, **unzip(tparams))
print('Done')
sys.stdout.flush()
return solution
def trainer(load_from=None,
save_dir='snapshots',
name='anon',
**kwargs):
"""
:param load_from: location to load parameters + options from
:param name: name of model, used as location to save parameters + options
"""
curr_model = dict()
# load old model, including parameters, but overwrite with new options
if load_from:
print 'reloading...' + load_from
with open('%s.pkl'%load_from, 'rb') as f:
curr_model = pkl.load(f)
else:
curr_model['options'] = {}
for k, v in kwargs.iteritems():
curr_model['options'][k] = v
model_options = curr_model['options']
# initialize logger
import datetime
timestampedName = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + '_' + name
from logger import Log
log = Log(name=timestampedName, hyperparams=model_options, saveDir='vis/training',
xLabel='Examples Seen', saveFrequency=1)
print curr_model['options']
# Load training and development sets
print 'Loading dataset'
dataset = load_dataset(model_options['data'], cnn=model_options['cnn'], load_train=True)
train = dataset['train']
dev = dataset['dev']
# Create dictionary
print 'Creating dictionary'
worddict = build_dictionary(train['caps']+dev['caps'])
print 'Dictionary size: ' + str(len(worddict))
curr_model['worddict'] = worddict
curr_model['options']['n_words'] = len(worddict) + 2
# save model
pkl.dump(curr_model, open('%s/%s.pkl' % (save_dir, name), 'wb'))
print 'Loading data'
train_iter = datasource.Datasource(train, batch_size=model_options['batch_size'], worddict=worddict)
dev = datasource.Datasource(dev, worddict=worddict)
dev_caps, dev_ims = dev.all()
print 'Building model'
params = init_params(model_options)
# reload parameters
if load_from is not None and os.path.exists(load_from):
params = load_params(load_from, params)
tparams = init_tparams(params)
inps, cost = build_model(tparams, model_options)
print 'Building sentence encoder'
inps_se, sentences = build_sentence_encoder(tparams, model_options)
f_senc = theano.function(inps_se, sentences, profile=False)
print 'Building image encoder'
inps_ie, images = build_image_encoder(tparams, model_options)
f_ienc = theano.function(inps_ie, images, profile=False)
print 'Building f_grad...',
grads = tensor.grad(cost, wrt=itemlist(tparams))
print 'Building errors..'
inps_err, errs = build_errors(model_options)
f_err = theano.function(inps_err, errs, profile=False)
curr_model['f_senc'] = f_senc
curr_model['f_ienc'] = f_ienc
curr_model['f_err'] = f_err
if model_options['grad_clip'] > 0.:
grads = [maxnorm(g, model_options['grad_clip']) for g in grads]
lr = tensor.scalar(name='lr')
print 'Building optimizers...',
# (compute gradients), (updates parameters)
f_grad_shared, f_update = eval(model_options['optimizer'])(lr, tparams, grads, inps, cost)
print 'Optimization'
uidx = 0
curr = 0
n_samples = 0
for eidx in xrange(model_options['max_epochs']):
print 'Epoch ', eidx
for x, mask, im in train_iter:
n_samples += x.shape[1]
uidx += 1
# Update
ud_start = time.time()
cost = f_grad_shared(x, mask, im)
f_update(model_options['lrate'])
ud = time.time() - ud_start
if numpy.isnan(cost) or numpy.isinf(cost):
print 'NaN detected'
return 1., 1., 1.
if numpy.mod(uidx, model_options['dispFreq']) == 0:
print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud
log.update({'Error': float(cost)}, n_samples)
if numpy.mod(uidx, model_options['validFreq']) == 0:
print 'Computing results...'
# encode sentences efficiently
dev_s = encode_sentences(curr_model, dev_caps, batch_size=model_options['batch_size'])
dev_i = encode_images(curr_model, dev_ims)
# compute errors
dev_errs = compute_errors(curr_model, dev_s, dev_i)
# compute ranking error
(r1, r5, r10, medr, meanr), vis_details = t2i(dev_errs, vis_details=True)
(r1i, r5i, r10i, medri, meanri) = i2t(dev_errs)
print "Text to image: %.1f, %.1f, %.1f, %.1f, %.1f" % (r1, r5, r10, medr, meanr)
log.update({'R@1': r1, 'R@5': r5, 'R@10': r10, 'median_rank': medr, 'mean_rank': meanr}, n_samples)
print "Image to text: %.1f, %.1f, %.1f, %.1f, %.1f" % (r1i, r5i, r10i, medri, meanri)
log.update({'Image2Caption_R@1': r1i, 'Image2Caption_R@5': r5i, 'Image2CaptionR@10': r10i, 'Image2Caption_median_rank': medri, 'Image2Caption_mean_rank': meanri}, n_samples)
tot = r1 + r5 + r10
if tot > curr:
curr = tot
# Save parameters
print 'Saving...',
numpy.savez('%s/%s'%(save_dir, name), **unzip(tparams))
print 'Done'
vis_details['hyperparams'] = model_options
# Save visualization details
with open('vis/roc/%s/%s.json' % (model_options['data'], timestampedName), 'w') as f:
json.dump(vis_details, f)
# Add the new model to the index
index = json.load(open('vis/roc/index.json', 'r'))
models = index[model_options['data']]
if timestampedName not in models:
models.append(timestampedName)
with open('vis/roc/index.json', 'w') as f:
json.dump(index, f)
print 'Seen %d samples'%n_samples