class GroundedTranslation(object):
def __init__(self, args, datagen=None):
'''
Initialise the model and set Theano debugging model if
self.args.debug is true. Prepare the data generator if necessary.
'''
self.args = args
self.data_generator = datagen
self.use_sourcelang = args.source_vectors is not None
self.use_image = not args.no_image
self.log_run_arguments()
self.data_generator=datagen
self.prepare_datagenerator()
if self.args.debug:
theano.config.optimizer = 'fast_compile'
theano.config.exception_verbosity = 'high'
def train_model(self):
'''
Initialise the data generator to process the data in a memory-friendly
manner. Then build the Keras model, given the user-specified arguments
(or the initial defaults). Train the model for self.args.max_epochs
and return the training and validation losses.
The losses object contains a history variable. The history variable is
a dictionary with a list of training and validation losses:
losses.history.['loss']
losses.history.['val_loss']
'''
if not self.use_sourcelang:
hsn_size = 0
else:
hsn_size = self.data_generator.hsn_size # ick
if self.args.mrnn:
m = models.MRNN(self.args.embed_size, self.args.hidden_size,
self.V, self.args.dropin,
self.args.optimiser, self.args.l2reg,
hsn_size=hsn_size,
weights=self.args.init_from_checkpoint,
gru=self.args.gru,
clipnorm=self.args.clipnorm,
t=self.data_generator.max_seq_len,
lr=self.args.lr)
else:
m = models.NIC(self.args.embed_size, self.args.hidden_size,
self.V, self.args.dropin,
self.args.optimiser, self.args.l2reg,
hsn_size=hsn_size,
weights=self.args.init_from_checkpoint,
gru=self.args.gru,
clipnorm=self.args.clipnorm,
t=self.data_generator.max_seq_len,
lr=self.args.lr)
model = m.buildKerasModel(use_sourcelang=self.use_sourcelang,
use_image=self.use_image)
callbacks = CompilationOfCallbacks(self.data_generator.word2index,
self.data_generator.index2word,
self.args,
self.args.dataset,
self.data_generator,
use_sourcelang=self.use_sourcelang,
use_image=self.use_image)
train_generator = self.data_generator.random_generator('train')
train_size = self.data_generator.split_sizes['train']
val_generator = self.data_generator.fixed_generator('val')
val_size = self.data_generator.split_sizes['val']
losses = model.fit_generator(generator=train_generator,
samples_per_epoch=train_size,
nb_epoch= self.args.max_epochs,
verbose=1,
callbacks=[callbacks],
nb_worker=1,
validation_data=val_generator,
nb_val_samples=val_size)
return losses
def prepare_datagenerator(self):
'''
Initialise the data generator and its datastructures, unless a valid
data generator was already passed into the
GroundedTranslation.__init() function.
'''
# Initialise the data generator if it has not yet been initialised
if self.data_generator == None:
self.data_generator = VisualWordDataGenerator(self.args,
self.args.dataset)
# Extract the working vocabulary from the training dataset
if self.args.existing_vocab != "":
self.data_generator.set_vocabulary(self.args.existing_vocab)
else:
self.data_generator.extract_vocabulary()
self.V = self.data_generator.get_vocab_size()
def log_run_arguments(self):
'''
Save the command-line arguments, along with the method defaults,
used to parameterise this run.
'''
logger.info("Run arguments:")
for arg, value in self.args.__dict__.iteritems():
logger.info("%s: %s" % (arg, str(value)))
class Sweep(object):
def __init__(self, args):
'''
Initialise the model and set Theano debugging model if
self.args.debug is true
'''
self.args = args
self.use_sourcelang = args.source_vectors is not None
self.use_image = not args.no_image
self.data_generator = None
self.prepare_datagenerator()
if self.args.debug:
theano.config.optimizer = 'fast_compile'
theano.config.exception_verbosity = 'high'
def random_sweep(self):
'''
Start randomly sweeping through hyperparameter ranges.
This current only supports sweeping through the L2 regularisation
strength, the learning rate, and the dropout probability.
'''
model = GroundedTranslation(self.args, datagen=self.data_generator)
handle = open("../logs/sweeper-%s.log" % self.args.run_string, "w")
handle.write("{:3} | {:10} | {:10} | {:10} | {:10} | {:10} \n".format("Run",
"loss", "val_loss", "lr", "reg", "dropin"))
handle.close()
for sweep in xrange(self.args.num_sweeps):
# randomly sample a learning rate and an L2 regularisation
handle = open("../logs/sweeper-%s.log" % self.args.run_string, "a")
if self.args.min_lr == ceil(self.args.min_lr):
# you provided an exponent, we'll search in log-space
lr = 10**uniform(self.args.min_lr, self.args.max_lr)
else:
# you provided a specific number
lr = 10**uniform(log10(self.args.min_lr),
log10(self.args.max_lr))
if self.args.min_l2 == ceil(self.args.min_l2):
# you provided an exponent, we'll search in log-space
l2 = 10**uniform(self.args.min_l2, self.args.max_l2)
else:
# you provide a specific number
l2 = 10**uniform(log10(self.args.min_l2),
log10(self.args.max_l2))
drop_in = uniform(self.args.min_dropin, self.args.max_dropin)
# modify the arguments that will be used to create the graph
model.args.lr = lr
model.args.l2reg = l2
model.args.dropin = drop_in
logger.info("Setting learning rate to: %.5e", lr)
logger.info("Setting l2reg to: %.5e", l2)
logger.info("Setting dropout to: %f", drop_in)
# initialise and compile a new model
losses = model.train_model()
handle.write("{:3d} | {:5.5f} | {:5.5f} | {:5e} | {:5e} | {:5.4f} \n".format(sweep,
losses.history['loss'][-1],
losses.history['val_loss'][-1], lr, l2, drop_in))
handle.close()
def prepare_datagenerator(self):
'''
Initialise the data generator and its datastructures, unless a valid
data generator was already passed into the
GroundedTranslation.__init() function.
'''
# Initialise the data generator if it has not yet been initialised
if self.data_generator == None:
self.data_generator = VisualWordDataGenerator(self.args,
self.args.dataset)
# Extract the working vocabulary from the training dataset
if self.args.existing_vocab != "":
self.data_generator.set_vocabulary(self.args.existing_vocab)
else:
self.data_generator.extract_vocabulary()
self.V = self.data_generator.get_vocab_size()
class GroundedTranslation(object):
def __init__(self, args, datagen=None):
'''
Initialise the model and set Theano debugging model if
self.args.debug is true. Prepare the data generator if necessary.
'''
self.args = args
self.data_generator = datagen
self.use_sourcelang = args.source_vectors is not None
self.use_image = not args.no_image
self.log_run_arguments()
self.data_generator = datagen
self.prepare_datagenerator()
if self.args.debug:
theano.config.optimizer = 'fast_compile'
theano.config.exception_verbosity = 'high'
def train_model(self):
'''
Initialise the data generator to process the data in a memory-friendly
manner. Then build the Keras model, given the user-specified arguments
(or the initial defaults). Train the model for self.args.max_epochs
and return the training and validation losses.
The losses object contains a history variable. The history variable is
a dictionary with a list of training and validation losses:
losses.history.['loss']
losses.history.['val_loss']
'''
if not self.use_sourcelang:
hsn_size = 0
else:
hsn_size = self.data_generator.hsn_size # ick
if self.args.mrnn:
m = models.MRNN(self.args.embed_size,
self.args.hidden_size,
self.V,
self.args.dropin,
self.args.optimiser,
self.args.l2reg,
hsn_size=hsn_size,
weights=self.args.init_from_checkpoint,
gru=self.args.gru,
clipnorm=self.args.clipnorm,
t=self.data_generator.max_seq_len,
lr=self.args.lr)
else:
m = models.NIC(self.args.embed_size,
self.args.hidden_size,
self.V,
self.args.dropin,
self.args.optimiser,
self.args.l2reg,
hsn_size=hsn_size,
weights=self.args.init_from_checkpoint,
gru=self.args.gru,
clipnorm=self.args.clipnorm,
t=self.data_generator.max_seq_len,
lr=self.args.lr)
model = m.buildKerasModel(use_sourcelang=self.use_sourcelang,
use_image=self.use_image)
callbacks = CompilationOfCallbacks(self.data_generator.word2index,
self.data_generator.index2word,
self.args,
self.args.dataset,
self.data_generator,
use_sourcelang=self.use_sourcelang,
use_image=self.use_image)
train_generator = self.data_generator.random_generator('train')
train_size = self.data_generator.split_sizes['train']
val_generator = self.data_generator.fixed_generator('val')
val_size = self.data_generator.split_sizes['val']
losses = model.fit_generator(generator=train_generator,
samples_per_epoch=train_size,
nb_epoch=self.args.max_epochs,
verbose=1,
callbacks=[callbacks],
nb_worker=1,
validation_data=val_generator,
nb_val_samples=val_size)
return losses
def prepare_datagenerator(self):
'''
Initialise the data generator and its datastructures, unless a valid
data generator was already passed into the
GroundedTranslation.__init() function.
'''
# Initialise the data generator if it has not yet been initialised
if self.data_generator == None:
self.data_generator = VisualWordDataGenerator(
self.args, self.args.dataset)
# Extract the working vocabulary from the training dataset
if self.args.existing_vocab != "":
self.data_generator.set_vocabulary(self.args.existing_vocab)
else:
self.data_generator.extract_vocabulary()
self.V = self.data_generator.get_vocab_size()
def log_run_arguments(self):
'''
Save the command-line arguments, along with the method defaults,
used to parameterise this run.
'''
logger.info("Run arguments:")
for arg, value in self.args.__dict__.iteritems():
logger.info("%s: %s" % (arg, str(value)))
class VisualWordLSTM(object):
"""LSTM that combines visual features with textual descriptions.
TODO: more details. Inherits from object as new-style class.
"""
def __init__(self, args):
self.args = args
# consistent with models.py
self.use_sourcelang = args.source_vectors is not None
self.use_image = not args.no_image
if self.args.debug:
theano.config.optimizer = 'None'
theano.config.exception_verbosity = 'high'
def train_model(self):
'''
In the model, we will merge
the word embeddings with
the VGG image representation (if used)
and the source-language multimodal vectors (if used).
We need to feed the data as a list, in which the order of the elements
in the list is _crucial_.
'''
self.log_run_arguments()
self.data_generator = VisualWordDataGenerator(
self.args, self.args.dataset)
self.data_generator.extract_vocabulary()
self.V = self.data_generator.get_vocab_size()
# Keras doesn't do batching of val set, so
# assume val data is small enough to get all at once.
# val_input is the list passed to model.fit()
# val_input can contain image, source features as well (or not)
if not self.args.enable_val_pplx:
val_input, valY = self.data_generator.get_data_by_split('val',
self.use_sourcelang, self.use_image)
if not self.use_sourcelang:
hsn_size = 0
else:
hsn_size = self.data_generator.hsn_size # ick
m = models.OneLayerLSTM(self.args.hidden_size, self.V,
self.args.dropin,
self.args.optimiser, self.args.l2reg,
hsn_size=hsn_size,
weights=self.args.init_from_checkpoint,
gru=self.args.gru)
model = m.buildKerasModel(use_sourcelang=self.use_sourcelang,
use_image=self.use_image)
callbacks = CompilationOfCallbacks(self.data_generator.word2index,
self.data_generator.index2word,
self.args,
self.args.dataset,
self.data_generator,
use_sourcelang=self.use_sourcelang,
use_image=self.use_image)
big_batch_size = self.args.big_batch_size
if big_batch_size > 0:
if self.args.small:
batches = ceil(SMALL_NUM_DESCRIPTIONS/self.args.big_batch_size)
else:
batches = ceil(float(self.data_generator.split_sizes['train']) /
self.args.big_batch_size)
batches = int(batches)
else: # if big_batch_size == 0, reset to training set size.
big_batch_size = self.data_generator.split_sizes['train']
batches = 1
# for epoch in range(self.args.epochs):
epoch = 0
while True:
# the program will exit with sys.exit(0) in
# Callbacks.early_stop_decision(). Do not put any clean-up
# after this loop. It will NEVER be executed!
batch = 1
for train_input, trainY, indicator in\
self.data_generator.yield_training_batch(big_batch_size,
self.use_sourcelang,
self.use_image):
if self.args.predefined_epochs:
logger.info("Epoch %d/%d, big-batch %d/%d", epoch+1,
self.args.max_epochs, batch, batches)
else:
logger.info("Epoch %d, big-batch %d/%d", epoch+1,
batch, batches)
if indicator is True:
# let's test on the val after training on these batches
model.fit(train_input,
trainY,
validation_data=None if
self.args.enable_val_pplx
else (val_input, valY),
callbacks=[callbacks],
nb_epoch=1,
verbose=1,
batch_size=self.args.batch_size,
shuffle=True)
else:
model.fit(train_input,
trainY,
nb_epoch=1,
verbose=1,
batch_size=self.args.batch_size,
shuffle=True)
batch += 1
epoch += 1
if self.args.predefined_epochs and epoch >= self.args.max_epochs:
# stop training because we've exceeded self.args.max_epochs
break
def log_run_arguments(self):
'''
Save the command-line arguments, along with the method defaults,
used to parameterise this run.
'''
logger.info("Run arguments:")
for arg, value in self.args.__dict__.iteritems():
logger.info("%s: %s" % (arg, str(value)))
class Sweep(object):
def __init__(self, args):
'''
Initialise the model and set Theano debugging model if
self.args.debug is true
'''
self.args = args
self.use_sourcelang = args.source_vectors is not None
self.use_image = not args.no_image
self.data_generator = None
self.prepare_datagenerator()
if self.args.debug:
theano.config.optimizer = 'fast_compile'
theano.config.exception_verbosity = 'high'
def random_sweep(self):
'''
Start randomly sweeping through hyperparameter ranges.
This current only supports sweeping through the L2 regularisation
strength, the learning rate, and the dropout probability.
'''
model = GroundedTranslation(self.args, datagen=self.data_generator)
handle = open("../logs/sweeper-%s.log" % self.args.run_string, "w")
handle.write("{:3} | {:10} | {:10} | {:10} | {:10} | {:10} \n".format(
"Run", "loss", "val_loss", "lr", "reg", "dropin"))
handle.close()
for sweep in xrange(self.args.num_sweeps):
# randomly sample a learning rate and an L2 regularisation
handle = open("../logs/sweeper-%s.log" % self.args.run_string, "a")
if self.args.min_lr == ceil(self.args.min_lr):
# you provided an exponent, we'll search in log-space
lr = 10**uniform(self.args.min_lr, self.args.max_lr)
else:
# you provided a specific number
lr = 10**uniform(log10(self.args.min_lr),
log10(self.args.max_lr))
if self.args.min_l2 == ceil(self.args.min_l2):
# you provided an exponent, we'll search in log-space
l2 = 10**uniform(self.args.min_l2, self.args.max_l2)
else:
# you provide a specific number
l2 = 10**uniform(log10(self.args.min_l2),
log10(self.args.max_l2))
drop_in = uniform(self.args.min_dropin, self.args.max_dropin)
# modify the arguments that will be used to create the graph
model.args.lr = lr
model.args.l2reg = l2
model.args.dropin = drop_in
logger.info("Setting learning rate to: %.5e", lr)
logger.info("Setting l2reg to: %.5e", l2)
logger.info("Setting dropout to: %f", drop_in)
# initialise and compile a new model
losses = model.train_model()
handle.write(
"{:3d} | {:5.5f} | {:5.5f} | {:5e} | {:5e} | {:5.4f} \n".
format(sweep, losses.history['loss'][-1],
losses.history['val_loss'][-1], lr, l2, drop_in))
handle.close()
def prepare_datagenerator(self):
'''
Initialise the data generator and its datastructures, unless a valid
data generator was already passed into the
GroundedTranslation.__init() function.
'''
# Initialise the data generator if it has not yet been initialised
if self.data_generator == None:
self.data_generator = VisualWordDataGenerator(
self.args, self.args.dataset)
# Extract the working vocabulary from the training dataset
if self.args.existing_vocab != "":
self.data_generator.set_vocabulary(self.args.existing_vocab)
else:
self.data_generator.extract_vocabulary()
self.V = self.data_generator.get_vocab_size()
