def __init__(self, model_path: str = None) -> None:
super().__init__()
"Requires the BertTokenizer from pytorch_transformers"
# pip install pytorch_transformers
import os
import torch
from pytorch_transformers import BertTokenizer, cached_path
from training.transformer_utils.model import TransformerWithClfHeadAndAdapters
try:
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.config = torch.load(
cached_path(os.path.join(model_path,
"model_training_args.bin")))
self.model = TransformerWithClfHeadAndAdapters(
self.config["config"],
self.config["config_ft"]).to(self.device)
state_dict = torch.load(cached_path(
os.path.join(model_path, "model_weights.pth")),
map_location=self.device)
self.model.load_state_dict(state_dict)
self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased',
do_lower_case=False)
except:
raise Exception(
"Require a valid transformer model file ({0}/model_weights.pth) "
"and its config file ({0}/model_training_args.bin).".format(
model_path))
class TransformerExplainer:
"""Class to explain classification results of the causal transformer.
Assumes that we already have a trained transformer model with which to make predictions.
Code for training/evaluating the transformer is as per the NAACL transfer learning repository.
https://github.com/huggingface/naacl_transfer_learning_tutorial
"""
def __init__(self, model_file: str=None) -> None:
"Requires the BertTokenizer from pytorch_transformers"
# pip install pytorch_transformers
import os
import torch
from pytorch_transformers import BertTokenizer, cached_path
from training.transformer_utils.model import TransformerWithClfHeadAndAdapters
try:
self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
self.config = torch.load(cached_path(os.path.join(model_file, "model_training_args.bin")))
self.model = TransformerWithClfHeadAndAdapters(self.config["config"],
self.config["config_ft"]).to(self.device)
state_dict = torch.load(cached_path(os.path.join(model_file, "model_weights.pth")),
map_location=self.device)
self.model.load_state_dict(state_dict)
self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False)
except:
raise Exception("Require a valid transformer model file ({0}/model_weights.pth) "
"and its config file ({0}/model_training_args.bin)."
.format(model_file))
def encode(self, inputs):
return list(self.tokenizer.convert_tokens_to_ids(o) for o in inputs)
def predict(self, texts: List[str]) -> np.array([float, ...]):
"Return an integer value of predicted class from the transformer model."
import torch
import torch.nn.functional as F
self.model.eval() # Disable dropout
clf_token = self.tokenizer.vocab['[CLS]'] # classifier token
pad_token = self.tokenizer.vocab['[PAD]'] # pad token
max_length = self.config['config'].num_max_positions # Max length from trained model
probs = [] # Process each text and get softmax probabilities
for text in tqdm(texts):
inputs = self.tokenizer.tokenize(text)
if len(inputs) >= max_length:
inputs = inputs[:max_length - 1]
ids = self.encode(inputs) + [clf_token]
with torch.no_grad(): # Disable backprop
tensor = torch.tensor(ids, dtype=torch.long).to(self.device)
tensor = tensor.reshape(1, -1)
tensor_in = tensor.transpose(0, 1).contiguous() # to shape [seq length, 1]
logits = self.model(tensor_in,
clf_tokens_mask=(tensor_in == clf_token),
padding_mask=(tensor == pad_token))
val, _ = torch.max(logits, 0)
val = F.softmax(val, dim=0).detach().cpu().numpy()
probs.append(val)
return np.array(probs)
class TransformerSentiment(Base):
"""Predict sentiment scores using a causal transformer.
Code for training/evaluating the transformer is as per the NAACL transfer learning repository.
https://github.com/huggingface/naacl_transfer_learning_tutorial
"""
def __init__(self, model_path: str = None) -> None:
super().__init__()
"Requires the BertTokenizer from pytorch_transformers"
# pip install pytorch_transformers
import os
import torch
from pytorch_transformers import BertTokenizer, cached_path
from training.transformer_utils.model import TransformerWithClfHeadAndAdapters
try:
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.config = torch.load(
cached_path(os.path.join(model_path,
"model_training_args.bin")))
self.model = TransformerWithClfHeadAndAdapters(
self.config["config"],
self.config["config_ft"]).to(self.device)
state_dict = torch.load(cached_path(
os.path.join(model_path, "model_weights.pth")),
map_location=self.device)
self.model.load_state_dict(state_dict)
self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased',
do_lower_case=False)
except:
raise Exception(
"Require a valid transformer model file ({0}/model_weights.pth) "
"and its config file ({0}/model_training_args.bin).".format(
model_path))
def encode(self, inputs):
return list(self.tokenizer.convert_tokens_to_ids(o) for o in inputs)
def score(self, text: str) -> int:
"Return an integer value of predicted class from the transformer model."
import torch
import torch.nn.functional as F
self.model.eval() # Disable dropout
clf_token = self.tokenizer.vocab['[CLS]'] # classifier token
pad_token = self.tokenizer.vocab['[PAD]'] # pad token
max_length = self.config[
'config'].num_max_positions # Max length from trained model
inputs = self.tokenizer.tokenize(text)
if len(inputs) >= max_length:
inputs = inputs[:max_length - 1]
ids = self.encode(inputs) + [clf_token]
with torch.no_grad(): # Disable backprop
tensor = torch.tensor(ids, dtype=torch.long).to(self.device)
tensor = tensor.reshape(1, -1)
tensor_in = tensor.transpose(
0, 1).contiguous() # to shape [seq length, 1]
logits = self.model(tensor_in,
clf_tokens_mask=(tensor_in == clf_token),
padding_mask=(tensor == pad_token))
val, _ = torch.max(logits, 0)
val = F.softmax(val, dim=0).detach().cpu().numpy()
# To train the transformer in PyTorch we zero-indexed the labels.
# Now we increment the predicted label by 1 to match with those from other classifiers.
pred = int(val.argmax()) + 1
return pred
def predict(self, train_file: None, test_file: str,
lower_case: bool) -> pd.DataFrame:
"Use tqdm to display model prediction status bar"
# pip install tqdm
from tqdm import tqdm
tqdm.pandas()
df = self.read_data(test_file, lower_case)
df['pred'] = df['text'].progress_apply(self.score)
return df