def test_inference_tiny_model(self):
batch_size = 13
sequence_length = 7
input_ids = torch.arange(0,
batch_size * sequence_length).long().reshape(
batch_size, sequence_length)
lengths = [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
token_type_ids = torch.tensor(
[[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths])
model = FunnelModel.from_pretrained("sgugger/funnel-random-tiny")
output = model(input_ids, token_type_ids=token_type_ids)[0].abs()
expected_output_sum = torch.tensor(2344.9023)
expected_output_mean = torch.tensor(0.8053)
self.assertTrue(
torch.allclose(output.sum(), expected_output_sum, atol=1e-4))
self.assertTrue(
torch.allclose(output.mean(), expected_output_mean, atol=1e-4))
attention_mask = torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 +
[[0, 1, 1, 0, 0, 1, 1]])
output = model(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids)[0].abs()
expected_output_sum = torch.tensor(2363.2178)
expected_output_mean = torch.tensor(0.8115)
self.assertTrue(
torch.allclose(output.sum(), expected_output_sum, atol=1e-4))
self.assertTrue(
torch.allclose(output.mean(), expected_output_mean, atol=1e-4))
def test_inference_model(self):
tokenizer = FunnelTokenizer.from_pretrained("huggingface/funnel-small")
model = FunnelModel.from_pretrained("huggingface/funnel-small")
inputs = tokenizer("Hello! I am the Funnel Transformer model.", return_tensors="pt")
output = model(**inputs)[0]
expected_output_sum = torch.tensor(235.7246)
expected_output_mean = torch.tensor(0.0256)
self.assertTrue(torch.allclose(output.sum(), expected_output_sum, atol=1e-4))
self.assertTrue(torch.allclose(output.mean(), expected_output_mean, atol=1e-4))
def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, config_file, pytorch_dump_path, base_model):
# Initialise PyTorch model
config = FunnelConfig.from_json_file(config_file)
print(f"Building PyTorch model from configuration: {config}")
model = FunnelBaseModel(config) if base_model else FunnelModel(config)
# Load weights from tf checkpoint
load_tf_weights_in_funnel(model, config, tf_checkpoint_path)
# Save pytorch-model
print(f"Save PyTorch model to {pytorch_dump_path}")
torch.save(model.state_dict(), pytorch_dump_path)
def create_and_check_model(
self,
config,
input_ids,
token_type_ids,
input_mask,
sequence_labels,
token_labels,
choice_labels,
fake_token_labels,
):
model = FunnelModel(config=config)
model.to(torch_device)
model.eval()
result = model(input_ids,
attention_mask=input_mask,
token_type_ids=token_type_ids)
result = model(input_ids, token_type_ids=token_type_ids)
result = model(input_ids)
self.parent.assertEqual(
result.last_hidden_state.shape,
(self.batch_size, self.seq_length, self.d_model))
model.config.truncate_seq = False
result = model(input_ids)
self.parent.assertEqual(
result.last_hidden_state.shape,
(self.batch_size, self.seq_length, self.d_model))
model.config.separate_cls = False
result = model(input_ids)
self.parent.assertEqual(
result.last_hidden_state.shape,
(self.batch_size, self.seq_length, self.d_model))
