def create_transfo_xl_lm_head_trainer_incompatible_tuple(
self, config, input_ids_1, input_ids_2, lm_labels):
config.trainer_compatible = False
model = TransfoXLLMHeadModel(config)
model.to(torch_device)
model.eval()
lm_logits_1 = model(input_ids_1, return_dict=False)[0]
outputs1 = model(input_ids_1, labels=lm_labels, return_dict=False)
losses_1, _, mems_1 = outputs1[:3]
loss_1 = outputs1[-1]
lm_logits_2 = model(input_ids_2, mems=mems_1, return_dict=False)[0]
outputs2 = model(input_ids_2, labels=lm_labels, mems=mems_1)
losses_2, _, mems_2 = outputs2[:3]
loss_2 = outputs2[-1]
outputs = {
"losses_1": losses_1,
"mems_1": mems_1,
"lm_logits_1": lm_logits_1,
"loss_1": loss_1,
"losses_2": losses_2,
"mems_2": mems_2,
"lm_logits_2": lm_logits_2,
"loss_2": loss_2,
}
config.trainer_compatible = None
return outputs
def create_transfo_xl_lm_head(self, config, input_ids_1, input_ids_2, lm_labels):
model = TransfoXLLMHeadModel(config)
model.eval()
lm_logits_1, mems_1 = model(input_ids_1)
loss_1, _, mems_1 = model(input_ids_1, labels=lm_labels)
lm_logits_2, mems_2 = model(input_ids_2, mems=mems_1)
loss_2, _, mems_2 = model(input_ids_2, labels=lm_labels, mems=mems_1)
outputs = {
"loss_1": loss_1,
"mems_1": mems_1,
"lm_logits_1": lm_logits_1,
"loss_2": loss_2,
"mems_2": mems_2,
"lm_logits_2": lm_logits_2,
}
return outputs
def create_transfo_xl_lm_head(self, config, input_ids_1, input_ids_2, lm_labels):
model = TransfoXLLMHeadModel(config)
model.to(torch_device)
model.eval()
lm_logits_1 = model(input_ids_1)["prediction_scores"]
outputs1 = model(input_ids_1, labels=lm_labels)
lm_logits_2 = model(input_ids_2, mems=outputs1["mems"])["prediction_scores"]
outputs2 = model(input_ids_2, labels=lm_labels, mems=outputs1["mems"])
outputs = {
"loss_1": outputs1["losses"],
"mems_1": outputs1["mems"],
"lm_logits_1": lm_logits_1,
"loss_2": outputs2["losses"],
"mems_2": outputs2["mems"],
"lm_logits_2": lm_logits_2,
}
return outputs
