示例#1
0
文件: models.py 项目: Bisk1/kongming
 def _break_and_wrap(text_to_tokenize_match):
     text_to_tokenize = text_to_tokenize_match.group(0)
     wrapped = ""
     for token in Languages.tokenize(Languages.chinese.value, text_to_tokenize):
         word_zh = WordZH.get_or_create_with_translator(word=token)[0]
         wrapped += '<span class="chinese-word"><span>' + word_zh.pinyin + '</span><span>' + token + '</span></span>'
     return wrapped
示例#2
0
文件: models.py 项目: Bisk1/kongming
 def auto_tokenize(self):
     """
     Tokenize the business text into words, create their objects
     if necessary and link the business text to them.
     Only applied to Chinese words.
     """
     word_model = to_word_model(self.language)
     if word_model == WordZH:
         tokens = Languages.tokenize(self.language, self.text)
         self.words_zh.clear()
         ordinal = 0
         for token in tokens:
             word_object = word_model.get_or_create_with_translator(word=token)[0]
             BusinessTextWordZH.objects.create(text=self, word=word_object, ordinal=ordinal).save()
             ordinal += 1