def create_and_check_for_entity_pair_classification(
self,
config,
input_ids,
attention_mask,
token_type_ids,
entity_ids,
entity_attention_mask,
entity_token_type_ids,
entity_position_ids,
sequence_labels,
labels,
entity_labels,
entity_classification_labels,
entity_pair_classification_labels,
entity_span_classification_labels,
):
config.num_labels = self.num_entity_pair_classification_labels
model = LukeForEntityClassification(config)
model.to(torch_device)
model.eval()
result = model(
input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
entity_ids=entity_ids,
entity_attention_mask=entity_attention_mask,
entity_token_type_ids=entity_token_type_ids,
entity_position_ids=entity_position_ids,
labels=entity_pair_classification_labels,
)
self.parent.assertEqual(
result.logits.shape,
(self.batch_size, self.num_entity_pair_classification_labels))
def evaluate_transformers_checkpoint(
data_path: str,
model_config_path: str,
checkpoint_model_name: str,
checkpoint_tokenizer_name: str,
batch_size: int,
cuda_device: int,
result_save_path: str,
):
"""
Expected results for ``test.json`` from the Open Entity dataset:
{'micro_precision': 0.7997806072235107, 'micro_recall': 0.7657563090324402, 'micro_fscore': 0.7823987007141113}.
Parameters
----------
data_path : str
Data path to the input file.
model_config_path : str
A config file that defines the model architecture to evaluate.
checkpoint_model_name : str
The name of the checkpoint in Hugging Face Model Hub.
checkpoint_tokenizer_name : str
This should be the name of the base pre-training model because sometimes
the tokenizer of downstream task is not compatible with allennlp.
batch_size : int
cuda_device : int
result_save_path : str
"""
import_module_and_submodules("examples_allennlp")
tokenizer_kwargs = {"additional_special_tokens": [ENT]}
reader = EntityTypingReader(
tokenizer=PretrainedTransformerTokenizer(
model_name=checkpoint_tokenizer_name,
add_special_tokens=True,
tokenizer_kwargs=tokenizer_kwargs),
token_indexers={
"tokens":
PretrainedTransformerIndexer(model_name=checkpoint_tokenizer_name,
tokenizer_kwargs=tokenizer_kwargs)
},
use_entity_feature=True,
)
transformers_tokenizer = LukeTokenizer.from_pretrained(
checkpoint_model_name)
transformers_model = LukeForEntityClassification.from_pretrained(
checkpoint_model_name)
vocab = Vocabulary()
vocab.add_transformer_vocab(transformers_tokenizer, "tokens")
num_labels = len(transformers_model.config.id2label)
labels = [transformers_model.config.id2label[i] for i in range(num_labels)]
vocab.add_tokens_to_namespace(labels, namespace="labels")
# read model
params = Params.from_file(
model_config_path,
ext_vars={"TRANSFORMERS_MODEL_NAME": checkpoint_model_name})
model = Model.from_params(params, vocab=vocab)
model.classifier = transformers_model.classifier
model.eval()
# set the GPU device to use
if cuda_device < 0:
device = torch.device("cpu")
else:
device = torch.device(f"cuda:{cuda_device}")
model = model.to(device)
loader = MultiProcessDataLoader(reader,
data_path,
batch_size=batch_size,
shuffle=False)
loader.index_with(model.vocab)
with torch.no_grad():
for batch in tqdm.tqdm(loader):
batch = nn_util.move_to_device(batch, device)
output_dict = model(**batch)
metrics = model.get_metrics(reset=True)
print(metrics)
if result_save_path is not None:
with open(result_save_path, "w") as f:
json.dump(metrics, f)