def _optimizer(c: Configs): optimizer = OptimizerConfigs() optimizer.parameters = c.model.parameters() optimizer.optimizer = 'Adam' optimizer.d_model = c.d_model return optimizer
def _generator_optimizer(c: Configs): opt_conf = OptimizerConfigs() opt_conf.optimizer = 'Adam' opt_conf.parameters = c.generator.parameters() opt_conf.learning_rate = 2.5e-4 # Setting exponent decay rate for first moment of gradient, # $\beta_`$ to `0.5` is important. # Default of `0.9` fails. opt_conf.betas = (0.5, 0.999) return opt_conf
def transformer_optimizer(c: Configs): """ Create a configurable optimizer. Parameters like learning rate can be changed by passing a dictionary when starting the experiment. """ optimizer = OptimizerConfigs() optimizer.parameters = c.model.parameters() optimizer.d_model = c.transformer.d_model optimizer.optimizer = 'Noam' return optimizer
def init(self): # Create a configurable optimizer. # Parameters like learning rate can be changed by passing a dictionary when starting the experiment. optimizer = OptimizerConfigs() optimizer.parameters = self.model.parameters() optimizer.d_model = self.transformer.d_model optimizer.optimizer = 'Noam' self.optimizer = optimizer # Create a sequential data loader for training self.train_loader = SequentialDataLoader(text=self.text.train, dataset=self.text, batch_size=self.batch_size, seq_len=self.seq_len) # Create a sequential data loader for validation self.valid_loader = SequentialDataLoader(text=self.text.valid, dataset=self.text, batch_size=self.batch_size, seq_len=self.seq_len) self.state_modules = [self.accuracy]