def get_optimizer_parameters(self, config):
base_lr = config.optimizer.params.lr
bert_params = get_bert_configured_parameters(self.text_encoder,
base_lr * 0.1)
backbone_params = [{
"params":
filter_grads(self.image_encoder.parameters()),
"lr":
base_lr * 0.1,
}]
rest_params = [
{
"params": filter_grads(self.image_proj.parameters()),
"lr": base_lr
},
{
"params": filter_grads(self.text_proj.parameters()),
"lr": base_lr
},
{
"params": filter_grads(self.image_pool.parameters()),
"lr": base_lr
},
{
"params": filter_grads(self.text_pool.parameters()),
"lr": base_lr
},
{
"params": filter_grads(self.shared_transformer.parameters()),
"lr": base_lr,
},
]
training_parameters = bert_params + backbone_params + rest_params
return training_parameters
def get_optimizer_parameters(self,
config: DictConfig) -> List[Dict[str, Any]]:
combine_layer = self.image_text_multi_modal_combine_layer
params = [
{
"params": filter_grads(self.word_embedding.parameters())
},
{
"params":
filter_grads(
self.image_feature_embeddings_list.sga.parameters())
},
{
"params":
filter_grads(
self.image_feature_embeddings_list.sga_pool.parameters())
},
{
"params":
filter_grads(
self.image_feature_embeddings_list.cbn.parameters()),
"lr": (config.optimizer.params.lr *
config.training.encoder_lr_multiply),
},
{
"params": filter_grads(self.text_embeddings.parameters())
},
{
"params": filter_grads(combine_layer.parameters())
},
{
"params": filter_grads(self.classifier.parameters())
},
{
"params":
filter_grads(self.image_feature_encoders.parameters())
},
]
return params