def build_model(self):
'''
Builds BERT NER model layers
Arguments:
None
Returns:
None
'''
# set seeds if a seed was provided
if self.seed:
torch.manual_seed(self.seed)
torch.cuda.manual_seed(self.seed)
np.random.seed(self.seed)
# initialize BERT model from file
self.bert = BertModel(self.config).from_pretrained(self.model_file)
# dropout layer for bert output
self.dropout = nn.Dropout(self.config.hidden_dropout_prob)
# dense classification layer
self.classifier = nn.Linear(self.config.hidden_size, len(self.classes))
# CRF output layer
self.crf = CRF(classes=self.classes,
scheme=self.scheme,
batch_first=True)
# initialize CRF with seed
self.crf.initialize(self.seed)
def __init__(self, config, *model_args, **model_kargs):
super().__init__(config)
self.model_args = model_kargs["model_args"]
self.bert = BertModel(config)
if self.model_args.do_mlm:
self.lm_head = BertLMPredictionHead(config)
cl_init(self, config)
def __init__(self, config):
super(Bert_CRF, self).__init__(config)
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.crf = CRF(num_tags=config.num_labels, batch_first=True)
self.init_weights()
def __init__(self, config):
super(BertForMultiLable, self).__init__(config)
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.init_weights()
self.decoder = seq_model.rnn_decoder(config)
self.criterion = nn.CrossEntropyLoss(ignore_index=utils.PAD, reduction='none')
self.softmax = nn.Softmax(dim=2)
def __init__(self, config, weight=None):
super(BertForSequenceClassification, self).__init__(config)
self.num_labels = config.num_labels
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
self.weight = weight
self.init_weights()
def __init__(self, config, head2size):
super(MultiHeadModel, self).__init__(config, head2size)
config.num_labels = 1
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
module_dict = {}
for head_name, num_labels in head2size.items():
module_dict[head_name] = nn.Linear(config.hidden_size, num_labels)
self.heads = nn.ModuleDict(module_dict)
self.init_weights()
def __init__(self, config):
super().__init__(config)
self.num_labels = config.num_labels
self.config = config
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.pos_weight = torch.ones(self.config.num_labels,
requires_grad=False)
self.init_weights()
def init_data(self, use_cuda) -> None:
torch.set_grad_enabled(False)
torch.set_num_threads(4)
turbo_transformers.set_num_threads(4)
self.test_device = torch.device('cuda:0') if use_cuda else \
torch.device('cpu:0')
self.cfg = BertConfig()
self.torch_model = BertModel(self.cfg)
self.torch_model.eval()
if torch.cuda.is_available():
self.torch_model.to(self.test_device)
self.turbo_model = turbo_transformers.BertModel.from_torch(
self.torch_model, self.test_device, "turbo")
def setup_method(self):
self.monkeypatch = MonkeyPatch()
# monkeypatch the PretrainedBertModel to return the tiny test fixture model
config_path = FIXTURES_ROOT / "structured_prediction" / "srl" / "bert" / "config.json"
vocab_path = FIXTURES_ROOT / "structured_prediction" / "srl" / "bert" / "vocab.txt"
config = BertConfig.from_json_file(config_path)
self.monkeypatch.setattr(BertModel, "from_pretrained", lambda _: BertModel(config))
self.monkeypatch.setattr(
BertTokenizer, "from_pretrained", lambda _: BertTokenizer(vocab_path)
)
super().setup_method()
self.set_up_model(
FIXTURES_ROOT / "structured_prediction" / "srl" / "bert_srl.jsonnet",
FIXTURES_ROOT / "structured_prediction" / "srl" / "conll_2012",
)
def __init__(self, config):
super(Bert_BiLSTM_CRF, self).__init__(config)
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.linear = nn.Linear(config.hidden_size, config.num_labels)
self.bilstm = nn.LSTM(
config.hidden_size,
(config.hidden_size) // 2,
dropout=config.hidden_dropout_prob,
batch_first=True,
bidirectional=True,
num_layers=2,
)
self.crf = CRF(num_tags=config.num_labels, batch_first=True)
self.init_weights()
def init_bert_models(self, use_cuda: bool) -> None:
self.test_device = torch.device('cuda:0') if use_cuda else \
torch.device('cpu:0')
if not use_cuda:
torch.set_num_threads(1)
torch.set_grad_enabled(False)
self.cfg = BertConfig(attention_probs_dropout_prob=0.0,
hidden_dropout_prob=0.0)
self.torch_model = BertModel(self.cfg)
self.torch_model.eval()
if use_cuda:
self.torch_model.to(self.test_device)
self.hidden_size = self.cfg.hidden_size
self.turbo_model = turbo_transformers.BertModelSmartBatch.from_torch(
self.torch_model)
def __init__(
self,
vocab: Vocabulary,
embedding_dim: int,
feedforward_dim: int,
num_layers: int,
num_attention_heads: int,
position_embedding_dim: int,
tokenizer_path: str,
position_embedding_type: str = "absolute",
activation: str = "gelu",
hidden_dropout: float = 0.1,
) -> None:
super().__init__()
# TODO:
# - Need to apply corrections in pretrained_transformer_mismatched_embedder
tokenizer = BertTokenizer.from_pretrained(tokenizer_path)
vocab.add_transformer_vocab(tokenizer, "tokens")
# "tokens" is padded by default--undo that
del vocab._token_to_index["tokens"]["@@PADDING@@"]
del vocab._token_to_index["tokens"]["@@UNKNOWN@@"]
assert len(vocab._token_to_index["tokens"]) == len(vocab._index_to_token["tokens"])
cfg = BertConfig(
vocab_size=vocab.get_vocab_size("tokens"),
hidden_size=embedding_dim,
num_hidden_layers=num_layers,
num_attention_heads=num_attention_heads,
intermediate_size=feedforward_dim,
hidden_act=activation,
hidden_dropout_prob=hidden_dropout,
max_position_embeddings=position_embedding_dim,
position_embedding_type=position_embedding_type,
use_cache=True,
)
self.cfg = cfg
self._vocab = vocab
self._namespace = "tokens"
self.bert = BertModel(cfg)
self.masking_collator = DataCollatorForWholeWordMask(
tokenizer=tokenizer, mlm=True, mlm_probability=0.15
)
def __init__(self, config, args, intent_label_lst, slot_label_lst):
super(JointMBERT, self).__init__(config)
self.args = args
self.num_intent_labels = len(intent_label_lst)
self.num_slot_labels = len(slot_label_lst)
self.bert = BertModel(config=config) # Load pretrained bert
self.intent_classifier = IntentClassifier(config.hidden_size,
self.num_intent_labels,
args.dropout_rate)
self.slot_classifier = SlotClassifier(
config.hidden_size, self.num_intent_labels, self.num_slot_labels,
self.args.use_intent_context_concat,
self.args.use_intent_context_attention, self.args.max_seq_len,
self.args.intent_embedding_size,
self.args.attention_embedding_size, self.args.attention_type,
args.dropout_rate)
if args.use_crf:
self.crf = CRF(num_tags=self.num_slot_labels, batch_first=True)
def __init__(self, config):
"""
Init function that initializes the model.
Inputs:
config - Configuration of the model
"""
super().__init__(config)
self.num_labels = config.num_labels
self.num_labels_list = [config.num_labels]
self.hidden_size = config.hidden_size
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
# create a list of classifiers
self.classifier = nn.Linear(config.hidden_size, self.num_labels)
self.classifiers = [self.classifier]
self.init_weights()
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, Dict[str, Any], BertModel],
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
if isinstance(bert_model, str):
self.bert_model = BertModel.from_pretrained(bert_model)
elif isinstance(bert_model, dict):
warnings.warn(
"Initializing BertModel without pretrained weights. This is fine if you're loading "
"from an AllenNLP archive, but not if you're training.",
UserWarning,
)
bert_config = BertConfig.from_dict(bert_model)
self.bert_model = BertModel(bert_config)
else:
self.bert_model = bert_model
self.num_classes = self.vocab.get_vocab_size("labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.tag_projection_layer = Linear(self.bert_model.config.hidden_size,
self.num_classes)
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
initializer(self)
def __init__(self, config, weight=None):
super(BertForTextRepresentation, self).__init__(config)
self.bert = BertModel(config)
self.weight = weight
self.init_weights()