def get_optimizer_parameters(self, config): base_lr = config.optimizer.params.lr bert_params = get_bert_configured_parameters(self.text_encoder, base_lr * 0.1) backbone_params = [{ "params": filter_grads(self.image_encoder.parameters()), "lr": base_lr * 0.1, }] rest_params = [ { "params": filter_grads(self.image_proj.parameters()), "lr": base_lr }, { "params": filter_grads(self.text_proj.parameters()), "lr": base_lr }, { "params": filter_grads(self.image_pool.parameters()), "lr": base_lr }, { "params": filter_grads(self.text_pool.parameters()), "lr": base_lr }, { "params": filter_grads(self.shared_transformer.parameters()), "lr": base_lr, }, ] training_parameters = bert_params + backbone_params + rest_params return training_parameters
def get_optimizer_parameters(self, config: DictConfig) -> List[Dict[str, Any]]: combine_layer = self.image_text_multi_modal_combine_layer params = [ { "params": filter_grads(self.word_embedding.parameters()) }, { "params": filter_grads( self.image_feature_embeddings_list.sga.parameters()) }, { "params": filter_grads( self.image_feature_embeddings_list.sga_pool.parameters()) }, { "params": filter_grads( self.image_feature_embeddings_list.cbn.parameters()), "lr": (config.optimizer.params.lr * config.training.encoder_lr_multiply), }, { "params": filter_grads(self.text_embeddings.parameters()) }, { "params": filter_grads(combine_layer.parameters()) }, { "params": filter_grads(self.classifier.parameters()) }, { "params": filter_grads(self.image_feature_encoders.parameters()) }, ] return params