class GroundedTranslation(object): def __init__(self, args, datagen=None): ''' Initialise the model and set Theano debugging model if self.args.debug is true. Prepare the data generator if necessary. ''' self.args = args self.data_generator = datagen self.use_sourcelang = args.source_vectors is not None self.use_image = not args.no_image self.log_run_arguments() self.data_generator=datagen self.prepare_datagenerator() if self.args.debug: theano.config.optimizer = 'fast_compile' theano.config.exception_verbosity = 'high' def train_model(self): ''' Initialise the data generator to process the data in a memory-friendly manner. Then build the Keras model, given the user-specified arguments (or the initial defaults). Train the model for self.args.max_epochs and return the training and validation losses. The losses object contains a history variable. The history variable is a dictionary with a list of training and validation losses: losses.history.['loss'] losses.history.['val_loss'] ''' if not self.use_sourcelang: hsn_size = 0 else: hsn_size = self.data_generator.hsn_size # ick if self.args.mrnn: m = models.MRNN(self.args.embed_size, self.args.hidden_size, self.V, self.args.dropin, self.args.optimiser, self.args.l2reg, hsn_size=hsn_size, weights=self.args.init_from_checkpoint, gru=self.args.gru, clipnorm=self.args.clipnorm, t=self.data_generator.max_seq_len, lr=self.args.lr) else: m = models.NIC(self.args.embed_size, self.args.hidden_size, self.V, self.args.dropin, self.args.optimiser, self.args.l2reg, hsn_size=hsn_size, weights=self.args.init_from_checkpoint, gru=self.args.gru, clipnorm=self.args.clipnorm, t=self.data_generator.max_seq_len, lr=self.args.lr) model = m.buildKerasModel(use_sourcelang=self.use_sourcelang, use_image=self.use_image) callbacks = CompilationOfCallbacks(self.data_generator.word2index, self.data_generator.index2word, self.args, self.args.dataset, self.data_generator, use_sourcelang=self.use_sourcelang, use_image=self.use_image) train_generator = self.data_generator.random_generator('train') train_size = self.data_generator.split_sizes['train'] val_generator = self.data_generator.fixed_generator('val') val_size = self.data_generator.split_sizes['val'] losses = model.fit_generator(generator=train_generator, samples_per_epoch=train_size, nb_epoch= self.args.max_epochs, verbose=1, callbacks=[callbacks], nb_worker=1, validation_data=val_generator, nb_val_samples=val_size) return losses def prepare_datagenerator(self): ''' Initialise the data generator and its datastructures, unless a valid data generator was already passed into the GroundedTranslation.__init() function. ''' # Initialise the data generator if it has not yet been initialised if self.data_generator == None: self.data_generator = VisualWordDataGenerator(self.args, self.args.dataset) # Extract the working vocabulary from the training dataset if self.args.existing_vocab != "": self.data_generator.set_vocabulary(self.args.existing_vocab) else: self.data_generator.extract_vocabulary() self.V = self.data_generator.get_vocab_size() def log_run_arguments(self): ''' Save the command-line arguments, along with the method defaults, used to parameterise this run. ''' logger.info("Run arguments:") for arg, value in self.args.__dict__.iteritems(): logger.info("%s: %s" % (arg, str(value)))
class Sweep(object): def __init__(self, args): ''' Initialise the model and set Theano debugging model if self.args.debug is true ''' self.args = args self.use_sourcelang = args.source_vectors is not None self.use_image = not args.no_image self.data_generator = None self.prepare_datagenerator() if self.args.debug: theano.config.optimizer = 'fast_compile' theano.config.exception_verbosity = 'high' def random_sweep(self): ''' Start randomly sweeping through hyperparameter ranges. This current only supports sweeping through the L2 regularisation strength, the learning rate, and the dropout probability. ''' model = GroundedTranslation(self.args, datagen=self.data_generator) handle = open("../logs/sweeper-%s.log" % self.args.run_string, "w") handle.write("{:3} | {:10} | {:10} | {:10} | {:10} | {:10} \n".format("Run", "loss", "val_loss", "lr", "reg", "dropin")) handle.close() for sweep in xrange(self.args.num_sweeps): # randomly sample a learning rate and an L2 regularisation handle = open("../logs/sweeper-%s.log" % self.args.run_string, "a") if self.args.min_lr == ceil(self.args.min_lr): # you provided an exponent, we'll search in log-space lr = 10**uniform(self.args.min_lr, self.args.max_lr) else: # you provided a specific number lr = 10**uniform(log10(self.args.min_lr), log10(self.args.max_lr)) if self.args.min_l2 == ceil(self.args.min_l2): # you provided an exponent, we'll search in log-space l2 = 10**uniform(self.args.min_l2, self.args.max_l2) else: # you provide a specific number l2 = 10**uniform(log10(self.args.min_l2), log10(self.args.max_l2)) drop_in = uniform(self.args.min_dropin, self.args.max_dropin) # modify the arguments that will be used to create the graph model.args.lr = lr model.args.l2reg = l2 model.args.dropin = drop_in logger.info("Setting learning rate to: %.5e", lr) logger.info("Setting l2reg to: %.5e", l2) logger.info("Setting dropout to: %f", drop_in) # initialise and compile a new model losses = model.train_model() handle.write("{:3d} | {:5.5f} | {:5.5f} | {:5e} | {:5e} | {:5.4f} \n".format(sweep, losses.history['loss'][-1], losses.history['val_loss'][-1], lr, l2, drop_in)) handle.close() def prepare_datagenerator(self): ''' Initialise the data generator and its datastructures, unless a valid data generator was already passed into the GroundedTranslation.__init() function. ''' # Initialise the data generator if it has not yet been initialised if self.data_generator == None: self.data_generator = VisualWordDataGenerator(self.args, self.args.dataset) # Extract the working vocabulary from the training dataset if self.args.existing_vocab != "": self.data_generator.set_vocabulary(self.args.existing_vocab) else: self.data_generator.extract_vocabulary() self.V = self.data_generator.get_vocab_size()
class GroundedTranslation(object): def __init__(self, args, datagen=None): ''' Initialise the model and set Theano debugging model if self.args.debug is true. Prepare the data generator if necessary. ''' self.args = args self.data_generator = datagen self.use_sourcelang = args.source_vectors is not None self.use_image = not args.no_image self.log_run_arguments() self.data_generator = datagen self.prepare_datagenerator() if self.args.debug: theano.config.optimizer = 'fast_compile' theano.config.exception_verbosity = 'high' def train_model(self): ''' Initialise the data generator to process the data in a memory-friendly manner. Then build the Keras model, given the user-specified arguments (or the initial defaults). Train the model for self.args.max_epochs and return the training and validation losses. The losses object contains a history variable. The history variable is a dictionary with a list of training and validation losses: losses.history.['loss'] losses.history.['val_loss'] ''' if not self.use_sourcelang: hsn_size = 0 else: hsn_size = self.data_generator.hsn_size # ick if self.args.mrnn: m = models.MRNN(self.args.embed_size, self.args.hidden_size, self.V, self.args.dropin, self.args.optimiser, self.args.l2reg, hsn_size=hsn_size, weights=self.args.init_from_checkpoint, gru=self.args.gru, clipnorm=self.args.clipnorm, t=self.data_generator.max_seq_len, lr=self.args.lr) else: m = models.NIC(self.args.embed_size, self.args.hidden_size, self.V, self.args.dropin, self.args.optimiser, self.args.l2reg, hsn_size=hsn_size, weights=self.args.init_from_checkpoint, gru=self.args.gru, clipnorm=self.args.clipnorm, t=self.data_generator.max_seq_len, lr=self.args.lr) model = m.buildKerasModel(use_sourcelang=self.use_sourcelang, use_image=self.use_image) callbacks = CompilationOfCallbacks(self.data_generator.word2index, self.data_generator.index2word, self.args, self.args.dataset, self.data_generator, use_sourcelang=self.use_sourcelang, use_image=self.use_image) train_generator = self.data_generator.random_generator('train') train_size = self.data_generator.split_sizes['train'] val_generator = self.data_generator.fixed_generator('val') val_size = self.data_generator.split_sizes['val'] losses = model.fit_generator(generator=train_generator, samples_per_epoch=train_size, nb_epoch=self.args.max_epochs, verbose=1, callbacks=[callbacks], nb_worker=1, validation_data=val_generator, nb_val_samples=val_size) return losses def prepare_datagenerator(self): ''' Initialise the data generator and its datastructures, unless a valid data generator was already passed into the GroundedTranslation.__init() function. ''' # Initialise the data generator if it has not yet been initialised if self.data_generator == None: self.data_generator = VisualWordDataGenerator( self.args, self.args.dataset) # Extract the working vocabulary from the training dataset if self.args.existing_vocab != "": self.data_generator.set_vocabulary(self.args.existing_vocab) else: self.data_generator.extract_vocabulary() self.V = self.data_generator.get_vocab_size() def log_run_arguments(self): ''' Save the command-line arguments, along with the method defaults, used to parameterise this run. ''' logger.info("Run arguments:") for arg, value in self.args.__dict__.iteritems(): logger.info("%s: %s" % (arg, str(value)))
class VisualWordLSTM(object): """LSTM that combines visual features with textual descriptions. TODO: more details. Inherits from object as new-style class. """ def __init__(self, args): self.args = args # consistent with models.py self.use_sourcelang = args.source_vectors is not None self.use_image = not args.no_image if self.args.debug: theano.config.optimizer = 'None' theano.config.exception_verbosity = 'high' def train_model(self): ''' In the model, we will merge the word embeddings with the VGG image representation (if used) and the source-language multimodal vectors (if used). We need to feed the data as a list, in which the order of the elements in the list is _crucial_. ''' self.log_run_arguments() self.data_generator = VisualWordDataGenerator( self.args, self.args.dataset) self.data_generator.extract_vocabulary() self.V = self.data_generator.get_vocab_size() # Keras doesn't do batching of val set, so # assume val data is small enough to get all at once. # val_input is the list passed to model.fit() # val_input can contain image, source features as well (or not) if not self.args.enable_val_pplx: val_input, valY = self.data_generator.get_data_by_split('val', self.use_sourcelang, self.use_image) if not self.use_sourcelang: hsn_size = 0 else: hsn_size = self.data_generator.hsn_size # ick m = models.OneLayerLSTM(self.args.hidden_size, self.V, self.args.dropin, self.args.optimiser, self.args.l2reg, hsn_size=hsn_size, weights=self.args.init_from_checkpoint, gru=self.args.gru) model = m.buildKerasModel(use_sourcelang=self.use_sourcelang, use_image=self.use_image) callbacks = CompilationOfCallbacks(self.data_generator.word2index, self.data_generator.index2word, self.args, self.args.dataset, self.data_generator, use_sourcelang=self.use_sourcelang, use_image=self.use_image) big_batch_size = self.args.big_batch_size if big_batch_size > 0: if self.args.small: batches = ceil(SMALL_NUM_DESCRIPTIONS/self.args.big_batch_size) else: batches = ceil(float(self.data_generator.split_sizes['train']) / self.args.big_batch_size) batches = int(batches) else: # if big_batch_size == 0, reset to training set size. big_batch_size = self.data_generator.split_sizes['train'] batches = 1 # for epoch in range(self.args.epochs): epoch = 0 while True: # the program will exit with sys.exit(0) in # Callbacks.early_stop_decision(). Do not put any clean-up # after this loop. It will NEVER be executed! batch = 1 for train_input, trainY, indicator in\ self.data_generator.yield_training_batch(big_batch_size, self.use_sourcelang, self.use_image): if self.args.predefined_epochs: logger.info("Epoch %d/%d, big-batch %d/%d", epoch+1, self.args.max_epochs, batch, batches) else: logger.info("Epoch %d, big-batch %d/%d", epoch+1, batch, batches) if indicator is True: # let's test on the val after training on these batches model.fit(train_input, trainY, validation_data=None if self.args.enable_val_pplx else (val_input, valY), callbacks=[callbacks], nb_epoch=1, verbose=1, batch_size=self.args.batch_size, shuffle=True) else: model.fit(train_input, trainY, nb_epoch=1, verbose=1, batch_size=self.args.batch_size, shuffle=True) batch += 1 epoch += 1 if self.args.predefined_epochs and epoch >= self.args.max_epochs: # stop training because we've exceeded self.args.max_epochs break def log_run_arguments(self): ''' Save the command-line arguments, along with the method defaults, used to parameterise this run. ''' logger.info("Run arguments:") for arg, value in self.args.__dict__.iteritems(): logger.info("%s: %s" % (arg, str(value)))
class Sweep(object): def __init__(self, args): ''' Initialise the model and set Theano debugging model if self.args.debug is true ''' self.args = args self.use_sourcelang = args.source_vectors is not None self.use_image = not args.no_image self.data_generator = None self.prepare_datagenerator() if self.args.debug: theano.config.optimizer = 'fast_compile' theano.config.exception_verbosity = 'high' def random_sweep(self): ''' Start randomly sweeping through hyperparameter ranges. This current only supports sweeping through the L2 regularisation strength, the learning rate, and the dropout probability. ''' model = GroundedTranslation(self.args, datagen=self.data_generator) handle = open("../logs/sweeper-%s.log" % self.args.run_string, "w") handle.write("{:3} | {:10} | {:10} | {:10} | {:10} | {:10} \n".format( "Run", "loss", "val_loss", "lr", "reg", "dropin")) handle.close() for sweep in xrange(self.args.num_sweeps): # randomly sample a learning rate and an L2 regularisation handle = open("../logs/sweeper-%s.log" % self.args.run_string, "a") if self.args.min_lr == ceil(self.args.min_lr): # you provided an exponent, we'll search in log-space lr = 10**uniform(self.args.min_lr, self.args.max_lr) else: # you provided a specific number lr = 10**uniform(log10(self.args.min_lr), log10(self.args.max_lr)) if self.args.min_l2 == ceil(self.args.min_l2): # you provided an exponent, we'll search in log-space l2 = 10**uniform(self.args.min_l2, self.args.max_l2) else: # you provide a specific number l2 = 10**uniform(log10(self.args.min_l2), log10(self.args.max_l2)) drop_in = uniform(self.args.min_dropin, self.args.max_dropin) # modify the arguments that will be used to create the graph model.args.lr = lr model.args.l2reg = l2 model.args.dropin = drop_in logger.info("Setting learning rate to: %.5e", lr) logger.info("Setting l2reg to: %.5e", l2) logger.info("Setting dropout to: %f", drop_in) # initialise and compile a new model losses = model.train_model() handle.write( "{:3d} | {:5.5f} | {:5.5f} | {:5e} | {:5e} | {:5.4f} \n". format(sweep, losses.history['loss'][-1], losses.history['val_loss'][-1], lr, l2, drop_in)) handle.close() def prepare_datagenerator(self): ''' Initialise the data generator and its datastructures, unless a valid data generator was already passed into the GroundedTranslation.__init() function. ''' # Initialise the data generator if it has not yet been initialised if self.data_generator == None: self.data_generator = VisualWordDataGenerator( self.args, self.args.dataset) # Extract the working vocabulary from the training dataset if self.args.existing_vocab != "": self.data_generator.set_vocabulary(self.args.existing_vocab) else: self.data_generator.extract_vocabulary() self.V = self.data_generator.get_vocab_size()