def collate(examples):
p_ids, passages, token_type, segment_ids, label, pos_start_ids, pos_end_ids = zip(
*examples)
p_ids = torch.tensor([p_id for p_id in p_ids], dtype=torch.long)
passages_tensor, _ = padding(passages,
is_float=False,
batch_first=batch_first)
pos_start_tensor, _ = padding(pos_start_ids,
is_float=False,
batch_first=batch_first)
pos_end_tensor, _ = padding(pos_end_ids,
is_float=False,
batch_first=batch_first)
token_type_tensor, _ = padding(token_type,
is_float=False,
batch_first=batch_first)
segment_tensor, _ = padding(segment_ids,
is_float=False,
batch_first=batch_first)
o1_tensor, o2_tensor = mpn_padding(passages,
label,
class_num=self.attribute_num,
is_float=True,
use_bert=self.use_bert)
return p_ids, passages_tensor, token_type_tensor, segment_tensor, pos_start_tensor, pos_end_tensor, o1_tensor, o2_tensor
def collate(examples):
p_ids, passages, label_id = zip(*examples)
p_ids = torch.tensor([p_id for p_id in p_ids], dtype=torch.long)
passages_tensor, _ = padding(passages,
is_float=False,
batch_first=batch_first)
label_tensor, _ = padding(label_id,
is_float=False,
batch_first=batch_first)
return p_ids, passages_tensor, label_tensor
def collate(examples):
p_ids, char_id, bichar_id, label_id = zip(*examples)
p_ids = torch.tensor([p_id for p_id in p_ids], dtype=torch.long)
char_tensor, _ = padding(char_id,
is_float=False,
batch_first=batch_first)
bichar_tensor, _ = padding(bichar_id,
is_float=False,
batch_first=batch_first)
label_tensor, _ = padding(label_id,
is_float=False,
batch_first=batch_first)
return p_ids, char_tensor, bichar_tensor, label_tensor
def collate(examples):
if self.is_train:
p_ids, passages, segment_ids, token_type, pos_start_ids, pos_end_ids, s1, s2, label = zip(
*examples)
p_ids = torch.tensor([p_id for p_id in p_ids],
dtype=torch.long)
passages_tensor, _ = padding(passages,
is_float=False,
batch_first=batch_first)
segment_tensor, _ = padding(segment_ids,
is_float=False,
batch_first=batch_first)
token_type_tensor, _ = padding(token_type,
is_float=False,
batch_first=batch_first)
pos_start_tensor, _ = padding(pos_start_ids,
is_float=False,
batch_first=batch_first)
pos_end_tensor, _ = padding(pos_end_ids,
is_float=False,
batch_first=batch_first)
s1_tensor, _ = padding(s1,
is_float=True,
batch_first=batch_first)
s2_tensor, _ = padding(s2,
is_float=True,
batch_first=batch_first)
po1_tensor, po2_tensor = spo_padding(
passages,
label,
class_num=self.predict_num,
is_float=True,
use_bert=self.use_bert)
return p_ids, passages_tensor, segment_tensor, token_type_tensor, s1_tensor, s2_tensor, po1_tensor, \
po2_tensor
else:
p_ids, passages, segment_ids = zip(*examples)
p_ids = torch.tensor([p_id for p_id in p_ids],
dtype=torch.long)
passages_tensor, _ = padding(passages,
is_float=False,
batch_first=batch_first)
segment_tensor, _ = padding(segment_ids,
is_float=False,
batch_first=batch_first)
return p_ids, passages_tensor, segment_tensor