def test_is_control(self): self.assertTrue(tokenization._is_control(u"\u0005")) self.assertFalse(tokenization._is_control(u"A")) self.assertFalse(tokenization._is_control(u" ")) self.assertFalse(tokenization._is_control(u"\t")) self.assertFalse(tokenization._is_control(u"\r"))
def customize_tokenizer(text, do_lower_case=False): tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case) temp_x = "" text = tokenization.convert_to_unicode(text) for c in text: if tokenizer._is_chinese_char(ord(c)) or tokenization._is_punctuation( c) or tokenization._is_whitespace( c) or tokenization._is_control(c): temp_x += " " + c + " " else: temp_x += c if do_lower_case: temp_x = temp_x.lower() return temp_x.split()
def get_dirty_text_ind(text): """Performs invalid character removal and whitespace cleanup on text.""" text = [unicodedata.normalize("NFD", t) for t in text] output = [] for char_ind, char in enumerate(text): if len(char) > 1: output.append(char_ind) continue cp = ord(char) if cp == 0 or cp == 0xfffd or _is_control(char): output.append(char_ind) return output
def _is_control(self, char): return bert_tokenization._is_control(char) # pylint: disable=protected-access