Пример #1
0
 def remove_eos(self, data, eos_token=EOS[1]):
     return operate_on_Narray(data, self._remove_eos, eos_token)
Пример #2
0
 def add_eos(self, data, eos_token=EOS[1]):
     return operate_on_Narray(data, self._add_eos, eos_token)
Пример #3
0
 def remove_go(self, data, go_token=GO[1]):
     return operate_on_Narray(data, self._remove_go, go_token)
Пример #4
0
 def pad(self, data, max_len, pad_token=PAD[1], l_pad_len=0):
     return operate_on_Narray(data, self._pad, max_len, pad_token, l_pad_len)
Пример #5
0
 def add_go(self, data, go_token=GO[1]):
     return operate_on_Narray(data, self._add_go, go_token)
Пример #6
0
 def max_seq_len(self, batch_data):
     return operate_on_Narray(batch_data, max)
Пример #7
0
 def seq_len(self, batch_data):
     return operate_on_Narray(batch_data, len)
Пример #8
0
 def to_tokens(self, ids):
     return operate_on_Narray(ids, self.ids_to_tokens)
Пример #9
0
 def combine_fn(self, tags):
     return operate_on_Narray(tags, self.combine_tag)
Пример #10
0
 def modify_fn(self, tags):
     return operate_on_Narray(tags, self.modify_tag)