def forward(self, input_ids=None, attention_mask=None, labels=None): # BERT outputs = self.gpt2_layer(input_ids, attention_mask=attention_mask) # output 0 = batch size 6, tokens 512, each token dimension 768 [CLS] token # output 1 = batch size 6, each token dimension 768 # output 2 = layers 13, batch 6 (hidden states), tokens 512, each token dimension 768 sequence_output = outputs[2] # Last layer of each token prediction num_layer_sum = 4 summed_last_4_layers = torch.stack( sequence_output[:num_layer_sum]).mean(0) summed_last_4_layers = self.dropout( summed_last_4_layers) # newly added dropout # lstm with masks (same as attention masks) packed_input, perm_idx, seq_lengths = get_packed_padded_output( summed_last_4_layers, input_ids, attention_mask, self.tokenizer) packed_output, (ht, ct) = self.lstm_layer(packed_input) # Unpack and reorder the output output, input_sizes = pad_packed_sequence(packed_output, batch_first=True) _, unperm_idx = perm_idx.sort(0) lstm_output = output[ unperm_idx] # lstm_output.shape = shorter than the padded torch.Size([6, 388, 512]) seq_lengths_ordered = seq_lengths[unperm_idx] # shorten the labels as per the batchsize labels = labels[:, :lstm_output.shape[1]] # mask the unimportant tokens before log_reg mask = ((input_ids[:, :lstm_output.shape[1]] != 50256) & (input_ids[:, :lstm_output.shape[1]] != 50256) & (labels != 100)) # on the first time steps for eachIndex in range(mask.shape[0]): mask[eachIndex, 0] = True mask_expanded = mask.unsqueeze(-1).expand(lstm_output.size()) lstm_output *= mask_expanded.float() labels *= mask.long() # log reg probablities = self.hidden2tag(lstm_output) # CRF emissions loss = self.crf_layer(probablities, labels, reduction='token_mean') emissions_ = self.crf_layer.decode(probablities) emissions = [item for sublist in emissions_ for item in sublist] # flatten the nest list of emissions return loss, torch.Tensor(emissions_), labels, mask
def forward(self, input_ids=None, attention_mask=None, labels=None): # BERT outputs = self.bert_layer(input_ids, attention_mask=attention_mask) # output 0 = batch size 6, tokens 512, each token dimension 768 [CLS] token # output 1 = batch size 6, each token dimension 768 # output 2 = layers 13, batch 6 (hidden states), tokens 512, each token dimension 768 sequence_output = outputs[2] # Last layer of each token prediction num_layer_sum = 4 summed_last_4_layers = torch.stack( sequence_output[:num_layer_sum]).mean(0) # lstm with masks (same as attention masks) packed_input, perm_idx, seq_lengths = get_packed_padded_output( summed_last_4_layers, input_ids, attention_mask, self.tokenizer) packed_output, (ht, ct) = self.lstm_layer(packed_input) # Unpack and reorder the output output, input_sizes = pad_packed_sequence(packed_output, batch_first=True) _, unperm_idx = perm_idx.sort(0) lstm_output = output[ unperm_idx] # lstm_output.shape = shorter than the padded torch.Size([6, 388, 512]) seq_lengths_ordered = seq_lengths[unperm_idx] # shorten the labels as per the batchsize labels = labels[:, :lstm_output.shape[1]] # mask the unimportant tokens before log_reg mask = ( (input_ids[:, :lstm_output.shape[1]] != self.tokenizer.pad_token_id) & (input_ids[:, :lstm_output.shape[1]] != self.tokenizer.convert_tokens_to_ids(self.tokenizer.sep_token)) & (labels != 100)) mask_expanded = mask.unsqueeze(-1).expand(lstm_output.size()) lstm_output *= mask_expanded.float() labels *= mask.long() # linear for fine probablity = F.relu(self.hidden2tag_fine(lstm_output)) max_probs = torch.max(probablity, dim=2) logits = max_probs.indices.flatten() loss = self.loss_fct(probablity.view(-1, clf_P_fine_num_labels), labels.flatten()) return loss, logits.to('cpu').numpy(), labels, mask
def forward(self, args, input_ids=None, attention_mask=None, P_labels=None, P_f_labels=None): # BERT outputs = self.bert_layer(input_ids, attention_mask=attention_mask) # output 0 = batch size 6, tokens 512, each token dimension 768 [CLS] token # output 1 = batch size 6, each token dimension 768 # output 2 = layers 13, batch 6 (hidden states), tokens 512, each token dimension 768 sequence_output = outputs[2] # Last layer of each token prediction num_layer_sum = 4 summed_last_4_layers = torch.stack( sequence_output[:num_layer_sum]).mean(0) # lstm with masks (same as attention masks) packed_input, perm_idx, seq_lengths = get_packed_padded_output( summed_last_4_layers, input_ids, attention_mask, self.tokenizer) packed_output, (ht, ct) = self.lstm_layer(packed_input) # Unpack and reorder the output output, input_sizes = pad_packed_sequence(packed_output, batch_first=True) _, unperm_idx = perm_idx.sort(0) lstm_output = output[ unperm_idx] # lstm_output.shape = shorter than the padded torch.Size([6, 388, 512]) seq_lengths_ordered = seq_lengths[unperm_idx] # shorten the labels as per the batchsize P_labels = P_labels[:, :lstm_output.shape[1]] P_f_labels = P_f_labels[:, :lstm_output.shape[1]] # Apply mask before calculating the matmul of inputs and the K, Q, V weights attention_mask_ = attention_mask[:, :lstm_output.shape[1]] attention_mask_ = attention_mask_.bool() # Apply self-attention here attention_applied, attention_weights = self.self_attention( lstm_output, lstm_output, lstm_output, key_padding_mask=None, need_weights=True, attn_mask=None) # Multi-attention applied here multi_attention_applied, multi_attention_weights = self.self_attention2( lstm_output, lstm_output, lstm_output, key_padding_mask=None, need_weights=True, attn_mask=None) # mask the unimportant tokens before attention is applied mask = ( (input_ids[:, :attention_applied.shape[1]] != self.tokenizer.pad_token_id) & (input_ids[:, :attention_applied.shape[1]] != self.tokenizer.convert_tokens_to_ids(self.tokenizer.sep_token)) & (P_labels != 100)) mask_expanded = mask.unsqueeze(-1).expand(attention_applied.size()) attention_applied *= mask_expanded.float() P_labels *= mask.long() P_f_labels *= mask.long() # log reg (coarse) probablity = F.relu(self.hidden2tag(attention_applied)) max_probs = torch.max(probablity, dim=2) logits = max_probs.indices # log reg (fine) probablity_fine = F.relu(self.hidden2tag_fine(multi_attention_applied)) max_probs_fine = torch.max(probablity_fine, dim=2) logits_fine = max_probs_fine.indices # calculate coarse loss loss_coarse = self.loss_fct_coarse( probablity.view(-1, clf_P_num_labels), P_labels.flatten()) # CRF emissions (fine) loss_fine = self.crf_layer_fine(probablity_fine, P_f_labels, mask=mask, reduction='token_mean') emissions_fine = self.crf_layer_fine.decode(probablity_fine) emissions_f = [item for sublist in emissions_fine for item in sublist ] # flatten the nest list of emissions loss = abs(loss_coarse) + abs(loss_fine) return loss_coarse, logits, P_labels, loss_fine, torch.Tensor( emissions_fine), P_f_labels, mask, loss
def forward(self, args, input_ids=None, attention_mask=None, P_labels=None, P_f_labels=None): # BERT outputs = self.gpt2_layer( input_ids, attention_mask = attention_mask ) # output 0 = batch size 6, tokens 512, each token dimension 768 [CLS] token # output 1 = batch size 6, each token dimension 768 # output 2 = layers 13, batch 6 (hidden states), tokens 512, each token dimension 768 sequence_output = outputs[2] # Last layer of each token prediction num_layer_sum = 4 summed_last_4_layers = torch.stack(sequence_output[:num_layer_sum]).mean(0) # summed_last_4_layers = self.dropout(summed_last_4_layers) # lstm with masks (same as attention masks) packed_input, perm_idx, seq_lengths = get_packed_padded_output(summed_last_4_layers, input_ids, attention_mask, self.tokenizer) packed_output, (ht, ct) = self.lstm_layer(packed_input) # Unpack and reorder the output output, input_sizes = pad_packed_sequence(packed_output, batch_first=True) _, unperm_idx = perm_idx.sort(0) lstm_output = output[unperm_idx] # lstm_output.shape = shorter than the padded torch.Size([6, 388, 512]) seq_lengths_ordered = seq_lengths[unperm_idx] # shorten the labels as per the batchsize P_labels = P_labels[:, :lstm_output.shape[1]] P_f_labels = P_f_labels[:, :lstm_output.shape[1]] # mask the unimportant tokens before log_reg mask = ( #(input_ids[:, :lstm_output.shape[1]] != self.tokenizer.unk_token_id) (input_ids[:, :lstm_output.shape[1]] != 50256) # & (input_ids[:, :lstm_output.shape[1]] != self.tokenizer.convert_tokens_to_ids(self.tokenizer.eos_token)) & (input_ids[:, :lstm_output.shape[1]] != 50256) & (P_labels != 100) ) mask_expanded = mask.unsqueeze(-1).expand(lstm_output.size()) lstm_output *= mask_expanded.float() P_labels *= mask.long() P_f_labels *= mask.long() # log reg (coarse) probablity = F.relu ( self.hidden2tag( lstm_output ) ) # log reg (fine) probablity_fine = F.relu ( self.hidden2tag_fine( lstm_output ) ) # CRF emissions (coarse) loss_coarse = self.crf_layer(probablity, P_labels, reduction='token_mean') # print('Coarse-grained loss: ', loss_coarse) emissions_coarse = self.crf_layer.decode( probablity ) emissions_c = [item for sublist in emissions_coarse for item in sublist] # flatten the nest list of emissions # CRF emissions (fine) loss_fine = self.crf_layer_fine(probablity_fine, P_f_labels, reduction='token_mean') # print('Fine-grained loss: ', loss_fine) emissions_fine = self.crf_layer_fine.decode( probablity_fine ) emissions_f = [item for sublist in emissions_fine for item in sublist] # flatten the nest list of emissions loss = abs( loss_coarse ) + abs( loss_fine ) return loss_coarse, torch.Tensor(emissions_coarse), P_labels, loss_fine, torch.Tensor(emissions_fine), P_f_labels, mask, loss
def forward(self, input_ids=None, attention_mask=None, labels=None): # def forward(self, args, input_ids=None, attention_mask=None, labels=None): # BERT outputs = self.bert_layer( input_ids, attention_mask = attention_mask ) # output 0 = batch size 6, tokens 512, each token dimension 768 [CLS] token # output 1 = batch size 6, each token dimension 768 # output 2 = layers 13, batch 6 (hidden states), tokens 512, each token dimension 768 sequence_output = outputs[2] # Last layer of each token prediction num_layer_sum = 4 summed_last_4_layers = torch.stack(sequence_output[:num_layer_sum]).mean(0) # lstm with masks (same as attention masks) packed_input, perm_idx, seq_lengths = get_packed_padded_output(summed_last_4_layers, input_ids, attention_mask, self.tokenizer) packed_output, (ht, ct) = self.lstm_layer(packed_input) # Unpack and reorder the output output, input_sizes = pad_packed_sequence(packed_output, batch_first=True) _, unperm_idx = perm_idx.sort(0) lstm_output = output[unperm_idx] # lstm_output.shape = shorter than the padded torch.Size([6, 388, 512]) seq_lengths_ordered = seq_lengths[unperm_idx] # shorten the labels as per the batchsize labels = labels[:, :lstm_output.shape[1]] # Apply the initialized dropout layer # lstm_output = self.drop_layer(lstm_output) # Apply mask before calculating the matmul of inputs and the K, Q, V weights attention_mask_ = attention_mask[:, :lstm_output.shape[1]] attention_mask_ = attention_mask_.bool() # Apply attention here attention_applied, attention_weights = self.self_attention( lstm_output, lstm_output, lstm_output, key_padding_mask=None, need_weights=True, attn_mask=None ) # mask the unimportant tokens before attention is applied mask = ( (input_ids[:, :attention_applied.shape[1]] != self.tokenizer.pad_token_id) & (input_ids[:, :attention_applied.shape[1]] != self.tokenizer.convert_tokens_to_ids(self.tokenizer.sep_token)) & (labels != 100) ) mask_expanded = mask.unsqueeze(-1).expand(attention_applied.size()) attention_applied *= mask_expanded.float() labels *= mask.long() # log reg probablity = F.relu ( self.hidden2tag( attention_applied ) ) # CRF emissions (coarse) loss = self.crf_layer(probablity, labels, mask=mask, reduction='token_mean') emissions_ = self.crf_layer.decode( probablity ) emissions = [item for sublist in emissions_ for item in sublist] # flatten the nest list of emissions # # mask labels here according to masks # labels_masked = labels[mask] return loss, torch.Tensor(emissions_), labels, mask
def forward(self, args, input_ids=None, attention_mask=None, P_labels=None, P_f_labels=None): # BERT outputs = self.gpt2_layer(input_ids, attention_mask=attention_mask) # output 0 = batch size 6, tokens 512, each token dimension 768 [CLS] token # output 1 = batch size 6, each token dimension 768 # output 2 = layers 13, batch 6 (hidden states), tokens 512, each token dimension 768 sequence_output = outputs[2] # Last layer of each token prediction num_layer_sum = 4 summed_last_4_layers = torch.stack( sequence_output[:num_layer_sum]).mean(0) # lstm with masks (same as attention masks) packed_input, perm_idx, seq_lengths = get_packed_padded_output( summed_last_4_layers, input_ids, attention_mask, self.tokenizer) packed_output, (ht, ct) = self.lstm_layer(packed_input) # Unpack and reorder the output output, input_sizes = pad_packed_sequence(packed_output, batch_first=True) _, unperm_idx = perm_idx.sort(0) lstm_output = output[ unperm_idx] # lstm_output.shape = shorter than the padded torch.Size([6, 388, 512]) seq_lengths_ordered = seq_lengths[unperm_idx] # shorten the labels as per the batchsize P_labels = P_labels[:, :lstm_output.shape[1]] P_f_labels = P_f_labels[:, :lstm_output.shape[1]] # Apply mask before calculating the matmul of inputs and the K, Q, V weights attention_mask_ = attention_mask[:, :lstm_output.shape[1]] attention_mask_ = attention_mask_.bool() # apply more weight # Attention should be applied to the LSTM outputs here # attention_applied, attention_weights = self.attn(lstm_output, attention_mask.float()) # attention_applied, attention_weights = self.self_attention( lstm_output, lstm_output, lstm_output, key_padding_mask=attention_mask_.permute(1, 0), need_weights=True, attn_mask=None ) attention_applied, attention_weights = self.self_attention( lstm_output, lstm_output, lstm_output, key_padding_mask=None, need_weights=True, attn_mask=None) # mask the unimportant tokens after attention is applied mask = ( #(input_ids[:, :lstm_output.shape[1]] != self.tokenizer.unk_token_id) (input_ids[:, :lstm_output.shape[1]] != 50256) # & (input_ids[:, :lstm_output.shape[1]] != self.tokenizer.convert_tokens_to_ids(self.tokenizer.eos_token)) & (input_ids[:, :lstm_output.shape[1]] != 50256) & (P_labels != 100)) # on the first time steps for eachIndex in range(mask.shape[0]): mask[eachIndex, 0] = True mask_expanded = mask.unsqueeze(-1).expand(attention_applied.size()) attention_applied *= mask_expanded.float() P_labels *= mask.long() P_f_labels *= mask.long() # log reg (coarse) probablity = F.relu(self.hidden2tag(attention_applied)) # log reg (fine) probablity_fine = F.relu(self.hidden2tag_fine(attention_applied)) # CRF emissions (coarse) #loss_coarse = self.crf_layer(probablity, reduction='token_mean') loss_coarse = self.crf_layer(probablity, P_labels, mask=mask, reduction='token_mean') emissions_coarse = self.crf_layer.decode(probablity) emissions_c = [ item for sublist in emissions_coarse for item in sublist ] # flatten the nest list of emissions # CRF emissions (fine) #loss_fine = self.crf_layer_fine(probablity_fine, reduction='token_mean') loss_fine = self.crf_layer_fine(probablity_fine, P_f_labels, mask=mask, reduction='token_mean') emissions_fine = self.crf_layer_fine.decode(probablity_fine) emissions_f = [item for sublist in emissions_fine for item in sublist ] # flatten the nest list of emissions loss = abs(loss_coarse) + abs(loss_fine) return loss_coarse, torch.Tensor( emissions_coarse), P_labels, loss_fine, torch.Tensor( emissions_fine), P_f_labels, mask, loss