def init_params(self, model_name, pre_trained_model, f_lr=5e-5, f_eps = 1e-8):
MODEL_CLASSES = { "xlnet": (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer) }
# self._config, self._model_class, self._tokenizer = MODEL_CLASSES[model_name]
self._tokenizer = XLNetTokenizer.from_pretrained(pre_trained_model, do_lower_case=True)
self._config = XLNetConfig.from_pretrained(pre_trained_model, do_lower_case=True)
self._model = XLNetForQuestionAnswering.from_pretrained(pre_trained_model, config=self._config)
self._model.to(self._device)
no_decay = ['bias', 'LayerNorm.weight']
weight_decay = 0.0 # Author's default parameter
optimizer_grouped_parameters = [
{'params': [p for n, p in self._model.named_parameters() \
if not any(nd in n for nd in no_decay)], 'weight_decay': weight_decay},
{'params': [p for n, p in self._model.named_parameters() \
if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
# warmup_steps = 0.0
self._optimizer = AdamW(optimizer_grouped_parameters, lr=f_lr, eps=f_eps)
def convert_xlnet_checkpoint_to_pytorch(tf_checkpoint_path,
bert_config_file,
pytorch_dump_folder_path,
finetuning_task=None):
# Initialise PyTorch model
config = XLNetConfig.from_json_file(bert_config_file)
finetuning_task = finetuning_task.lower(
) if finetuning_task is not None else ""
if finetuning_task in GLUE_TASKS_NUM_LABELS:
print(
"Building PyTorch XLNetForSequenceClassification model from configuration: {}"
.format(str(config)))
config.finetuning_task = finetuning_task
config.num_labels = GLUE_TASKS_NUM_LABELS[finetuning_task]
model = XLNetForSequenceClassification(config)
elif 'squad' in finetuning_task:
config.finetuning_task = finetuning_task
model = XLNetForQuestionAnswering(config)
else:
model = XLNetLMHeadModel(config)
# Load weights from tf checkpoint
load_tf_weights_in_xlnet(model, config, tf_checkpoint_path)
# Save pytorch-model
pytorch_weights_dump_path = os.path.join(pytorch_dump_folder_path,
WEIGHTS_NAME)
pytorch_config_dump_path = os.path.join(pytorch_dump_folder_path,
CONFIG_NAME)
print("Save PyTorch model to {}".format(
os.path.abspath(pytorch_weights_dump_path)))
torch.save(model.state_dict(), pytorch_weights_dump_path)
print("Save configuration file to {}".format(
os.path.abspath(pytorch_config_dump_path)))
with open(pytorch_config_dump_path, "w", encoding="utf-8") as f:
f.write(config.to_json_string())
def _load_model(self, model_path, device):
self.model = XLNetForQuestionAnswering.from_pretrained(model_path)
self.model.to(device)
self.model.eval()
def create_and_check_xlnet_qa(self, config, input_ids_1, input_ids_2,
input_ids_q, perm_mask, input_mask,
target_mapping, segment_ids, lm_labels,
sequence_labels, is_impossible_labels):
model = XLNetForQuestionAnswering(config)
model.eval()
outputs = model(input_ids_1)
start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits, mems = outputs
outputs = model(input_ids_1,
start_positions=sequence_labels,
end_positions=sequence_labels,
cls_index=sequence_labels,
is_impossible=is_impossible_labels,
p_mask=input_mask)
outputs = model(input_ids_1,
start_positions=sequence_labels,
end_positions=sequence_labels,
cls_index=sequence_labels,
is_impossible=is_impossible_labels)
total_loss, mems = outputs
outputs = model(input_ids_1,
start_positions=sequence_labels,
end_positions=sequence_labels)
total_loss, mems = outputs
result = {
"loss": total_loss,
"start_top_log_probs": start_top_log_probs,
"start_top_index": start_top_index,
"end_top_log_probs": end_top_log_probs,
"end_top_index": end_top_index,
"cls_logits": cls_logits,
"mems": mems,
}
self.parent.assertListEqual(list(result["loss"].size()), [])
self.parent.assertListEqual(
list(result["start_top_log_probs"].size()),
[self.batch_size, model.config.start_n_top])
self.parent.assertListEqual(
list(result["start_top_index"].size()),
[self.batch_size, model.config.start_n_top])
self.parent.assertListEqual(
list(result["end_top_log_probs"].size()), [
self.batch_size,
model.config.start_n_top * model.config.end_n_top
])
self.parent.assertListEqual(list(result["end_top_index"].size()), [
self.batch_size,
model.config.start_n_top * model.config.end_n_top
])
self.parent.assertListEqual(list(result["cls_logits"].size()),
[self.batch_size])
self.parent.assertListEqual(
list(list(mem.size()) for mem in result["mems"]),
[[self.seq_length, self.batch_size, self.hidden_size]] *
self.num_hidden_layers)