def forward(self, input_ids, attention_mask=None, token_type_ids=None,
position_ids=None, head_mask=None, labels=None):
outputs = self.bert(input_ids = input_ids,attention_mask=attention_mask,token_type_ids=token_type_ids)
sequence_output = outputs[0]
sequence_output = self.dropout(sequence_output)
logits = self.classifier(sequence_output)
outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here
if labels is not None:
assert self.loss_type in ['lsr', 'focal', 'ce']
if self.loss_type == 'lsr':
loss_fct = LabelSmoothingCrossEntropy(ignore_index=0)
elif self.loss_type == 'focal':
loss_fct = FocalLoss(ignore_index=0)
else:
loss_fct = CrossEntropyLoss(ignore_index=0)
# Only keep active parts of the loss
if attention_mask is not None:
active_loss = attention_mask.contiguous().view(-1) == 1
active_logits = logits.view(-1, self.num_labels)[active_loss]
active_labels = labels.contiguous().view(-1)[active_loss]
loss = loss_fct(active_logits, active_labels)
else:
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs = (loss,) + outputs
return outputs # (loss), scores, (hidden_states), (attentions)
def forward(self, input_ids,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
labels=None,
input_lens=None):
outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask,
token_type_ids=token_type_ids, position_ids=position_ids, head_mask=head_mask)
last_hidden_state = outputs[0]
sequence_output = self.dropout(last_hidden_state)
logits = self.classifier(sequence_output)
outputs = (logits,) + outputs[2:]
if labels is not None:
assert self.loss_type in ['ce', 'fl', 'lsc']
if self.loss_type == 'ce':
loss_fct = CrossEntropyLoss(ignore_index=0)
elif self.loss_type == 'fl':
loss_fct = FocalLoss(ignore_index=0)
elif self.loss_type == 'lsc':
loss_fct = LabelSmoothingCrossEntropy(ignore_index=0)
if attention_mask is not None:
active_loss = attention_mask.contiguous().view(-1) == 1
active_logits = logits.contiguous().view(-1, self.num_labels)[active_loss]
active_targets = labels.contiguous().view(-1)[active_loss]
loss = loss_fct(active_logits, active_targets)
else:
loss = loss_fct(logits.contiguous().view(-1, self.num_labels), labels.contiguous().view(-1))
outputs = (loss,) + outputs
return outputs
def forward(self,
input_ids,
token_type_ids=None,
attention_mask=None,
start_positions=None,
end_positions=None):
outputs = self.bert(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids)
sequence_output = outputs[0]
sequence_output = self.dropout(sequence_output)
start_logits = self.start_fc(sequence_output)
if start_positions is not None and self.training:
if self.soft_label:
batch_size = input_ids.size(0)
seq_len = input_ids.size(1)
label_logits = torch.FloatTensor(batch_size, seq_len,
self.num_labels)
label_logits.zero_()
label_logits = label_logits.to(input_ids.device)
label_logits.scatter_(2, start_positions.unsqueeze(2), 1)
else:
label_logits = start_positions.unsqueeze(2).float()
else:
label_logits = F.softmax(start_logits, -1)
if not self.soft_label:
label_logits = torch.argmax(label_logits,
-1).unsqueeze(2).float()
end_logits = self.end_fc(sequence_output, label_logits)
outputs = (
start_logits,
end_logits,
) + outputs[2:]
if start_positions is not None and end_positions is not None:
assert self.loss_type in ['lsr', 'focal', 'ce']
if self.loss_type == 'lsr':
loss_fct = LabelSmoothingCrossEntropy()
elif self.loss_type == 'focal':
loss_fct = FocalLoss()
else:
loss_fct = CrossEntropyLoss()
start_logits = start_logits.view(-1, self.num_labels)
end_logits = end_logits.view(-1, self.num_labels)
active_loss = attention_mask.view(-1) == 1
active_start_logits = start_logits[active_loss]
active_end_logits = end_logits[active_loss]
active_start_labels = start_positions.view(-1)[active_loss]
active_end_labels = end_positions.view(-1)[active_loss]
start_loss = loss_fct(active_start_logits, active_start_labels)
end_loss = loss_fct(active_end_logits, active_end_labels)
total_loss = (start_loss + end_loss) / 2
outputs = (total_loss, ) + outputs
return outputs
def forward(self,
input_ids,
attention_mask,
labels,
token_type_ids=None,
input_lens=None):
embs = self.embedding(input_ids)
embs = self.dropout(embs)
embs = embs * attention_mask.float().unsqueeze(2)
sequence_output, _ = self.bilstm(embs)
sequence_output = self.layer_norm(sequence_output)
logits = self.classifier(sequence_output)
outputs = (logits, )
if labels is not None:
if self.use_crf:
loss = self.crf(emissions=logits,
tags=labels,
mask=attention_mask)
outputs = (-1 * loss, ) + outputs
else:
assert self.loss_type in ['ce', 'fl', 'lsc']
if self.loss_type == 'ce':
loss_fct = CrossEntropyLoss(ignore_index=0)
elif self.loss_type == 'fl':
loss_fct = FocalLoss(ignore_index=0)
elif self.loss_type == 'lsc':
loss_fct = LabelSmoothingCrossEntropy(ignore_index=0)
if attention_mask is not None:
active_loss = attention_mask.contiguous().view(-1) == 1
active_logits = logits.contiguous().view(
-1, self.num_labels)[active_loss]
active_targets = labels.contiguous().view(-1)[active_loss]
loss = loss_fct(active_logits, active_targets)
else:
loss = loss_fct(
logits.contiguous().view(-1, self.num_labels),
labels.contiguous().view(-1))
outputs = (loss, ) + outputs
return outputs # (loss), scores
def forward(self, input_ids,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
labels=None,
input_lens=None):
outputs = self.bert(input_ids=input_ids, token_type_ids=token_type_ids,
position_ids=position_ids, head_mask=head_mask, attention_mask=attention_mask)
last_hidden_state = outputs[0] # (batch_size, sequence_length, hidden_size)
if self.use_lstm:
last_hidden_state, _ = self.bilstm(last_hidden_state)
sequence_output = self.dropout(last_hidden_state)
print(sequence_output.shape) # (batch_size, sequence_length, hidden_size)
logits = self.classifier(sequence_output) # (batch_size, seq_length, num_labels)
outputs = (logits,) + outputs
if labels is not None:
if self.use_crf:
loss = self.crf(emissions = logits, tags=labels, mask=attention_mask)
outputs =(-1*loss,)+outputs
else:
assert self.loss_type in ['ce', 'fl', 'lsc']
if self.loss_type == 'ce':
loss_fct = CrossEntropyLoss(ignore_index=0)
elif self.loss_type == 'fl':
loss_fct = FocalLoss(ignore_index=0)
elif self.loss_type == 'lsc':
loss_fct = LabelSmoothingCrossEntropy(ignore_index=0)
if attention_mask is not None:
active_loss = attention_mask.contiguous().view(-1) == 1
active_logits = logits.contiguous().view(-1, self.num_labels)[active_loss]
active_targets = labels.contiguous().view(-1)[active_loss]
loss = loss_fct(active_logits, active_targets)
else:
loss = loss_fct(logits.contiguous().view(-1, self.num_labels), labels.contiguous().view(-1))
outputs = (loss,) + outputs
return outputs