示例#1
0
 def remove_eos(self, data, eos_token=EOS[1]):
     return operate_on_Narray(data, self._remove_eos, eos_token)
示例#2
0
 def add_eos(self, data, eos_token=EOS[1]):
     return operate_on_Narray(data, self._add_eos, eos_token)
示例#3
0
 def remove_go(self, data, go_token=GO[1]):
     return operate_on_Narray(data, self._remove_go, go_token)
示例#4
0
 def pad(self, data, max_len, pad_token=PAD[1], l_pad_len=0):
     return operate_on_Narray(data, self._pad, max_len, pad_token, l_pad_len)
示例#5
0
 def add_go(self, data, go_token=GO[1]):
     return operate_on_Narray(data, self._add_go, go_token)
示例#6
0
 def max_seq_len(self, batch_data):
     return operate_on_Narray(batch_data, max)
示例#7
0
 def seq_len(self, batch_data):
     return operate_on_Narray(batch_data, len)
示例#8
0
 def to_tokens(self, ids):
     return operate_on_Narray(ids, self.ids_to_tokens)
示例#9
0
 def combine_fn(self, tags):
     return operate_on_Narray(tags, self.combine_tag)
示例#10
0
 def modify_fn(self, tags):
     return operate_on_Narray(tags, self.modify_tag)