def test_tcc(self): self.assertEqual(tcc.segment(None), []) self.assertEqual(tcc.segment(""), []) self.assertEqual(tcc.segment("ประเทศไทย"), ["ป", "ระ", "เท", "ศ", "ไท", "ย"]) self.assertEqual(list(tcc.tcc("")), []) self.assertEqual(tcc.tcc_pos(""), set())
def test_tcc(self): self.assertEqual(tcc.segment(None), []) self.assertEqual(tcc.segment(""), []) self.assertEqual(tcc.segment("ประเทศไทย"), ["ป", "ระ", "เท", "ศ", "ไท", "ย"]) self.assertEqual(list(tcc.tcc("")), []) self.assertEqual(tcc.tcc_pos(""), set())
def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]: # main data structure: # - key is begin position (int) # - value is possible end positions (List[int]) # if key is not found, value is empty list graph = defaultdict(list) graph_size = 0 # keep track of graph size, if too big will force cutoff valid_poss = tcc_pos(text) # breaking positions that are TCC-valid len_text = len(text) pos_list = [0] # priority queue of possible breaking positions end_pos = 0 while pos_list[0] < len_text: begin_pos = heappop(pos_list) for word in custom_dict.prefixes(text[begin_pos:]): end_pos_candidate = begin_pos + len(word) if end_pos_candidate in valid_poss: graph[begin_pos].append(end_pos_candidate) graph_size = graph_size + 1 if end_pos_candidate not in pos_list: heappush(pos_list, end_pos_candidate) if graph_size > _MAX_GRAPH_SIZE: break len_pos_list = len(pos_list) if len_pos_list == 1: # one candidate, no longer ambiguous end_pos_candidates = next( _bfs_paths_graph(graph, end_pos, pos_list[0])) graph_size = 0 for pos in end_pos_candidates[1:]: yield text[end_pos:pos] end_pos = pos elif len_pos_list == 0: # no candidate, deal with non-dictionary word m = _PAT_NONTHAI.match(text[begin_pos:]) if m: # non-Thai token, skip to the end end_pos = begin_pos + m.end() else: # Thai token, find minimum skip for pos in range(begin_pos + 1, len_text): if pos in valid_poss: prefix = text[pos:] words = [ word for word in custom_dict.prefixes(prefix) if ((pos + len(word) in valid_poss) and not _PAT_THAI_TWOCHARS.match(word)) ] if words: # is a Thai token that longer than 2 chars end_pos = pos break # is a non-Thai token if _PAT_NONTHAI.match(prefix): end_pos = pos break else: end_pos = len_text graph[begin_pos].append(end_pos) graph_size = graph_size + 1 yield text[begin_pos:end_pos] heappush(pos_list, end_pos)
# -*- coding: utf-8 -*- from pythainlp.tokenize import tcc print(tcc.tcc("ประเทศไทย")) # ป/ระ/เท/ศ/ไท/ย print(tcc.tcc_pos("ประเทศไทย")) # {1, 3, 5, 6, 8, 9} for ch in tcc.tcc_gen("ประเทศไทย"): # ป-ระ-เท-ศ-ไท-ย- print(ch, end='-')