def collate(examples): p_ids, passages, token_type, segment_ids, label, pos_start_ids, pos_end_ids = zip( *examples) p_ids = torch.tensor([p_id for p_id in p_ids], dtype=torch.long) passages_tensor, _ = padding(passages, is_float=False, batch_first=batch_first) pos_start_tensor, _ = padding(pos_start_ids, is_float=False, batch_first=batch_first) pos_end_tensor, _ = padding(pos_end_ids, is_float=False, batch_first=batch_first) token_type_tensor, _ = padding(token_type, is_float=False, batch_first=batch_first) segment_tensor, _ = padding(segment_ids, is_float=False, batch_first=batch_first) o1_tensor, o2_tensor = mpn_padding(passages, label, class_num=self.attribute_num, is_float=True, use_bert=self.use_bert) return p_ids, passages_tensor, token_type_tensor, segment_tensor, pos_start_tensor, pos_end_tensor, o1_tensor, o2_tensor
def collate(examples): p_ids, passages, label_id = zip(*examples) p_ids = torch.tensor([p_id for p_id in p_ids], dtype=torch.long) passages_tensor, _ = padding(passages, is_float=False, batch_first=batch_first) label_tensor, _ = padding(label_id, is_float=False, batch_first=batch_first) return p_ids, passages_tensor, label_tensor
def collate(examples): p_ids, char_id, bichar_id, label_id = zip(*examples) p_ids = torch.tensor([p_id for p_id in p_ids], dtype=torch.long) char_tensor, _ = padding(char_id, is_float=False, batch_first=batch_first) bichar_tensor, _ = padding(bichar_id, is_float=False, batch_first=batch_first) label_tensor, _ = padding(label_id, is_float=False, batch_first=batch_first) return p_ids, char_tensor, bichar_tensor, label_tensor
def collate(examples): if self.is_train: p_ids, passages, segment_ids, token_type, pos_start_ids, pos_end_ids, s1, s2, label = zip( *examples) p_ids = torch.tensor([p_id for p_id in p_ids], dtype=torch.long) passages_tensor, _ = padding(passages, is_float=False, batch_first=batch_first) segment_tensor, _ = padding(segment_ids, is_float=False, batch_first=batch_first) token_type_tensor, _ = padding(token_type, is_float=False, batch_first=batch_first) pos_start_tensor, _ = padding(pos_start_ids, is_float=False, batch_first=batch_first) pos_end_tensor, _ = padding(pos_end_ids, is_float=False, batch_first=batch_first) s1_tensor, _ = padding(s1, is_float=True, batch_first=batch_first) s2_tensor, _ = padding(s2, is_float=True, batch_first=batch_first) po1_tensor, po2_tensor = spo_padding( passages, label, class_num=self.predict_num, is_float=True, use_bert=self.use_bert) return p_ids, passages_tensor, segment_tensor, token_type_tensor, s1_tensor, s2_tensor, po1_tensor, \ po2_tensor else: p_ids, passages, segment_ids = zip(*examples) p_ids = torch.tensor([p_id for p_id in p_ids], dtype=torch.long) passages_tensor, _ = padding(passages, is_float=False, batch_first=batch_first) segment_tensor, _ = padding(segment_ids, is_float=False, batch_first=batch_first) return p_ids, passages_tensor, segment_tensor