def qa_build_model(experiment_config_dict, mlm_model_path, task):
backbone_name = experiment_config_dict["training"].backbone_name
XLMRConfig = transformers.AutoConfig.from_pretrained(backbone_name)
setattr(XLMRConfig, "discriminators", experiment_config_dict["discriminators"])
finetune_model = MultitaskModel.create_untrained(
backbone_name=backbone_name,
task_dict={
"mlm": {
"type": transformers.XLMRobertaForMaskedLM,
"config": transformers.XLMRobertaConfig.from_pretrained(backbone_name),
},
"disentangle": {
"type": XLMRobertaForDisentanglement,
"config": XLMRConfig,
},
},
)
import torch
finetune_model.load_state_dict(torch.load(mlm_model_path))
finetune_model.taskmodels_dict.pop("mlm")
finetune_model.add_task(
task,
transformers.XLMRobertaForQuestionAnswering,
transformers.XLMRobertaConfig.from_pretrained(finetune_model.backbone_name),
)
return finetune_model
def tag_load_finetuned_model(experiment_config_dict, mlm_model_path, task):
backbone_name = experiment_config_dict["training"].backbone_name
XLMRConfig = transformers.AutoConfig.from_pretrained(backbone_name)
setattr(XLMRConfig, "discriminators", experiment_config_dict["discriminators"])
finetuned_model = MultitaskModel.create_untrained(
backbone_name=backbone_name,
task_dict={
task: {
"type": transformers.XLMRobertaForTokenClassification,
"config": transformers.XLMRobertaConfig.from_pretrained(
backbone_name,
num_labels=xtreme_ds.TASK[task]["num_labels"],
),
},
"disentangle": {
"type": XLMRobertaForDisentanglement,
"config": XLMRConfig,
},
},
)
import torch
finetuned_model.load_state_dict(torch.load(mlm_model_path))
return finetuned_model
"mlm_beta1": 0.9,
"mlm_beta2": 0.98,
"mlm_eps": 1e-6
}
}""",
cls=ExperimentConfigSerializer,
)
XLMRConfig = XLMRobertaConfig.from_pretrained("xlm-roberta-large")
setattr(XLMRConfig, "discriminators", exp_config["discriminators"])
model = MultitaskModel.create_untrained(
backbone_name="xlm-roberta-large",
task_dict={
"mlm": {
"type": XLMRobertaForMaskedLM,
"config": XLMRobertaConfig.from_pretrained("xlm-roberta-large"),
},
"disentangle": {
"type": XLMRobertaForDisentanglement,
"config": XLMRConfig,
},
},
)
import torch
model.load_state_dict(
torch.load(
"/gpfs1/home/ckchan666/mlm_disentangle_experiment/model/mlm/default/pytorch_model.bin"
)
)
disentangled_model = model.backbone
ds = xtreme_ds.udposTestDataset()
experiment_config_dict = json.load(outfile, cls=ExperimentConfigSerializer)
experiment_config_dict["training"].model_name = (os.path.abspath(
args.config_json).split("/")[-1].split(".")[0])
backbone_name = experiment_config_dict["training"].backbone_name
XLMRConfig = transformers.AutoConfig.from_pretrained(backbone_name)
setattr(XLMRConfig, "discriminators", experiment_config_dict["discriminators"])
multitask_model = MultitaskModel.create(
backbone_name=backbone_name,
task_dict={
"mlm": {
"type": transformers.XLMRobertaForMaskedLM,
"config":
transformers.XLMRobertaConfig.from_pretrained(backbone_name),
},
"disentangle": {
"type": XLMRobertaForDisentanglement,
"config": XLMRConfig,
},
},
)
for layers in multitask_model.backbone.encoder.layer[:experiment_config_dict[
"training"].num_frozen_layers]: # 0:24
for param in layers.parameters():
param.requires_grad = False
optimizermlm = torch.optim.Adam(
multitask_model.parameters(),
lr=experiment_config_dict["training"].mlm_lr,
betas=(