def raw_data(self, ord_): """ Return similar Hanzi/Kanji etc characters using the MultiRadicals database """ char = w_unichr(ord_) if char in DBothChars: LRads = self.get_rads(char) LNumStrokes = self.char_data.raw_data('unihan.totalstrokes', ord_) or [] #print LNumStrokes L = [] SAdded = set() for rad in LRads: for i_ord in DBothRads[rad]: if char == w_unichr(i_ord) and True: continue elif i_ord in SAdded: continue SAdded.add(i_ord) L.append(self._get_cmp_value(LRads, LNumStrokes, i_ord)) L.sort() return [i[-1] for i in L] else: return None
def _get_cmp_value(self, LRads, LNumStrokes, ord_): i_LRads = self.get_rads(w_unichr(ord_)) # Get the number of multiradicals which are # different between the current character and radical num_same = len([i for i in i_LRads if i in LRads]) # Get the difference in number of strokes. If there are # multiple stroke counts for either character, then choose # the smallest difference stroke_diff = maxsize #print unichr(ord_), char_data.raw_data('unihan.totalstrokes', ord_) for num_strokes in self.char_data.raw_data('unihan.totalstrokes', ord_) or []: for i_num_strokes in LNumStrokes: x = abs(i_num_strokes - num_strokes) if x < stroke_diff: stroke_diff = x # TODO: Get the frequency depending on language! ========================================== LFreqs = [] for key in ('unihan.frequency', 'kanjidic.freq'): freq = self.char_data.raw_data(key, ord_) freq = freq[0] if freq else maxsize LFreqs.append(freq) return (-num_same, stroke_diff, LFreqs, ord_)
def raw_data(self, ord_): char = w_unichr(ord_) if char in DBothChars: return ' '.join(tuple([i for i in DBothChars[char] if i != '�'])) else: return None
def raw_data(self, ord_): char = w_unichr(ord_) try: return unicodedata.normalize(self.typ, char) except: return None
def get_L_ranges(self, L): """ """ LRtn = [] for typ, value in L: # FIXME: Don't allow searches by # fulltext unless it's a direct match for # name/conscript name/definitions etc! =============================== # (readings should allow partial matches I think) #print typ, value for i in self.char_indexes.search(typ, value): if isinstance(i, (list, tuple)): from_, to = i LRtn.append((w_unichr(from_), w_unichr(to))) else: LRtn.append(w_unichr(i)) return tuple(LRtn)
def raw_data(self, ord_): # Add encoding mappings char = w_unichr(ord_) try: return char.encode(self.encoding) except Exception: from traceback import print_exc #print_exc() return None
def get_simp_chars(char_data, Char, DTradChars=None): if type(Char) == int: Char = w_unichr(Char) LSubChars = [] if DTradChars: # convert the converted characters/radicals Traditional-Simplified LVariants = [ 'unihan.simplifiedvariant', 'unihan.semanticvariant', # CHECK ME! 'unihan.specializedsemanticvariant', # CHECK ME! 'unihan.zvariant' ] for Variant in LVariants: V = char_data.raw_data(Variant, w_ord(Char)) if V: LSubChars += [ w_unichr(i) for i in V ] print(('Char:', Char.encode('utf-8'), 'LSubChars:', ''.join(LSubChars).encode('utf-8'), 'V:', V)) # Use the transliteration system to convert from T-S, # only allowing if different and MultiRad data not # already available for that character from multi_translit.translit.my_engine.TranslitEngine import get_engine TradToSimp = get_engine('Chinese Traditional-Simplified') Conv = TradToSimp.convert(Char) LSubChars.append(Conv) # HACK: Make sure characters that are already # Traditional aren't converted so information isn't lost! # NOTE: This shouldn't be performed for RADICAL conversion, # just the CHARS as it can stuff up the radical lists :-( if DTradChars: LSubChars = [ i for i in LSubChars if not i in DTradChars ] return rem_dupes(LSubChars)
def conv_hex(key, s): #print 'KEY:', key if key == 'Block Subnames' or key == 'Name' or \ key == 'Unicode 1.0 Name' or \ key == 'Xiandai Hanyu Pinlu': return s # HACK! elif key == 'Names List' and s.startswith('# '): #print 'COMPAT:', s.encode('utf-8') try: L1 = [] L2 = [] for hex in s.split(): try: assert len(hex) == 4 ord_ = int(hex, 16) except: # Not hex, e.g. "<super>" in the trademark symbol? L1.append(hex) continue L1.append('U+%s' % hex) L2.append(w_unichr(ord_)) return '%s (%s)' % (' '.join(L1), ''.join(L2)) except: return s try: t_s = s.strip('() ') hex = t_s.split(' ')[-1].upper().replace('U+', '') assert len(hex) == 4 ord_ = int(hex, 16) t_s = ' '.join(t_s.split(' ')[:-1]).strip(' -') t_s = '%s U+%s (%s)' % (t_s, hex, w_unichr(ord_)) #print 'CONVHEX:', t_s.encode('utf-8') return t_s.strip() except: return s
def raw_data(self, ord_, data): char = w_unichr(ord_) if self.typ == 'lower': return char.lower() elif self.typ == 'upper': return char.upper() elif self.typ == 'title': return char.title() elif self.typ == 'capitalize': return char.capitalize() else: raise Exception("unknown casing type %s" % self.typ)
def get_D_latin(self, script='Latin'): """ This can also work for Latin-like scripts such as Cyrillic to an extent, but works best for Latin. It provides a map from e.g. "e" to "é" etc for allowing input popups (giving a choice of accented characters after a non-accented character is pressed) and grouping into like headers etc """ DLatin = {} LLatin = self.char_indexes.search('unicodedata.script', script) for L in LLatin: if not type(L) in (list, tuple): L = [L, L] from_, to = L for ord_ in range(from_, to + 1): char = w_unichr(ord_) # Find the relevant key name = [ _.lower() for _ in self.char_data.raw_data('name', ord_) ] letters = get_smallest_name(name) #print 'SMALLEST NAME:', name, letters, unichr(ord_) if not char in DLatin: DLatin[char] = [] DLatin[char].append(letters) if len(letters) > 1: DLatin[char].append(letters[0]) # TODO: What about "alternates" for characters which # look like e.g. an 'o' but sound like another letter? # Adjust case as appropriate if char.istitle(): DLatin[char] = [i.title() for i in DLatin[char]] elif char.isupper(): DLatin[char] = [i.upper() for i in DLatin[char]] elif char.islower(): DLatin[char] = [i.lower() for i in DLatin[char]] # Spanish HACKS! DLatin['!'] = ['¡'] DLatin['?'] = ['¿'] DLatin['¡'] = ['!'] DLatin['¿'] = ['?'] return DLatin
def get_D_latin_to_L_chars(self, script='Latin'): DRtn = {} LChars = self.char_indexes.search('unicodedata.script', script) for LRange in LChars: if type(LRange) != tuple: LRange = (LRange, LRange + 1) for ord_ in range(*LRange): name = self.char_data.raw_data('name', ord_) key = get_smallest_name(name).lower() #print key, unichr(ord_).encode('utf-8'), name if key != 'zz': key = key[0] # HACK! L = DRtn.setdefault(key, []) L.append(w_unichr(ord_)) return DRtn
def process_curly_braces(self, x, s): """ Characters inside curly braces/ brackets ({...}) make a string """ L = [] backslash_mode = False x += 1 while 1: # Get the current char try: c = s[x] except: break if backslash_mode and c == 'u': # A Unicode backslash L.append(w_unichr(int(s[x + 1:x + 5], 16))) backslash_mode = False x += 4 elif backslash_mode: L.append(c) backslash_mode = False elif c.strip(): if c == '}': break elif c == '\\': backslash_mode = True else: L.append(c) x += 1 return x, ''.join(L)
def _format_data(self, ord_, data): return ' '.join(w_unichr(i_ord) for i_ord in data)
def get_ranges(self, s): x = 0 LRtn = [] backslash_mode = False cur_operator = None neg = False while 1: # Get the current char try: c = s[x] except: break #print 'GET_RANGES:', c if backslash_mode: if c in 'pP': # perl-syntax! x, neg, LRanges = self.process_perl(x, s) LRtn.append((RANGES, neg, LRanges)) elif c == 'u': # Unicode backslash LRtn.append((STRING, w_unichr(int(s[x + 1:x + 5], 16)))) x += 4 elif c == '\\': # a literal backslash LRtn.append((STRING, '\\')) backslash_mode = False elif c == '\\': # Activate backslash mode backslash_mode = True elif c == '{': # Extract a string x, str_ = self.process_curly_braces(x, s) LRtn.append((STRING, str_)) elif s[x:x + 2] == '[:': # A POSIX range x, neg, LRanges = self.process_posix(x, s) LRtn.append((RANGES, neg, LRanges)) elif c == '[': # Embedded ranges # TODO: Fix PERL BACKSLASHES! ======================================= x, range_text = self.process_range(x, s) assert (range_text[0], range_text[-1]) == ('[', ']') LRtn.append(self.get_ranges(range_text[1:-1])) elif c == ']': # Should never be a closing bracket here! raise Exception("']' without '['!") elif c in '-&': # Operators # - is difference # & is intersect cur_operator = c x += 1 continue elif not x and c == '^': # Negative mode neg = True elif c == '$' and (x == len(s) - 1): pass #FIXME # FIXME! ================================================ elif c == '$' and self.DVars != None: # Process `$variableName` y, name = self.process_variable(x, s) value = self.DVars[name] if isinstance(value, (tuple, list)): value = ''.join(_[-1] for _ in value) # TYPE WARNING! ========== # Insert the variable, and reprocess #print 'BEFORE:', s, value s = s[:x] + value + s[y:] #print s, s[x] x -= 1 elif c.strip(): # Non-whitespace - add a single character as a string # OPEN ISSUE: Ignore whitespace??? ================================ LRtn.append((STRING, c)) else: # Whitespace - ignore it x += 1 continue if not backslash_mode and cur_operator and len(LRtn) < 2: # Allow initial "-" and "&" characters LRtn.append((STRING, cur_operator)) cur_operator = None elif not backslash_mode and cur_operator: if LRtn[-1][0] in (RANGES, OPERATOR): # A range difference etc, e.g. "[[a-e]-[c]]" assert LRtn[-2][0] in (RANGES, OPERATOR) LRtn.append((OPERATOR, (cur_operator, LRtn.pop()))) else: # an ordinary range, e.g. "[a-e]" item2, item1 = LRtn.pop(), LRtn.pop() assert cur_operator == '-' assert (item1[0], item2[0]) == (STRING, STRING), (item1, item2, self.s) assert (len(item1[1]), len(item2[1])) == (1, 1) # Note the "FALSE!" LRtn.append((RANGES, False, ((item1[1], item2[1]), ))) cur_operator = None x += 1 return (RANGES, neg, tuple(LRtn))