def transliterate(self, k, cat): ret = k gk = k c = Bsdconv("bsdconv:{0}".format(cat)) c.conv(p01(k)) if not c.counter("OERR"): return ret for layer in self.layers: gk = layer[0].rdata.get(gk) t = layer[1].get(gk).get(cat, None) if t: ret = t break else: gk = layer[0].rdata.get(gk, None) return ret
def _print_group1(self, g): c = Bsdconv("bsdconv:utf-8") print("###Layer 1 Group {0}\n".format(g)) print("####Attributes") a = self.layers[0][1].get(g) for cat in a: v = a[cat] print(" * {0}: {1} ({2})".format(cat, c.conv(p01(v)), v)) print("") print("####Member") d = self.layers[0][0].data.get(g) for e in d: print(" * {0} ({1})".format(c.conv(p01(e)), e)) print("")
def _print_group2(self, g2): c = Bsdconv("bsdconv:utf-8") print("#Layer 2 Group {0}\n".format(g2)) print("##Attributes") a = self.layers[1][1].get(g2) for cat in a: v = a[cat] print("* {0}: {1} ({2})".format(cat, c.conv(p01(v)), v)) print("##Member") for g in self.layers[1][0].data.get(g2): self._print_group1(g)
sep=re.compile(r"\s+") f=open(sys.argv[1]) for l in f: l = l.strip() if l == "": continue if l.startswith("#"): clist.write(l+"\n") plist.write(l+"\n") a = sep.split(l) p = a[0] ln = len(p.decode("utf-8")) if ln > 1: bonus = 6 p = bcv_zhtw.conv(p).rstrip(",") of = plist else: try: bonus = int(a[1]) except: bonus = 0 sc.counter_reset() sc.conv(p) score = sc.counter("SCORE") if score < 5*ln: bonus += 5*ln - score if bonus == 0: continue p = bcv.conv(p).rstrip(",") of = clist
["utf-8:count#for=lala&for=cjk:null", "123Б測a試bc", { "COUNT": 2 }], ] passed = True for c, i, o in iotest: p = Bsdconv(c) if not p: print(Bsdconv.error()) print("Test failed at %s" % repr([c, i, o])) del p passed = False continue r = p.conv(i) if o != r: print("Test failed at %s" % repr([c, i, o])) print("expected(%d): %s" % (len(o), repr(o))) print("result(%d): %s" % (len(r), repr(r))) passed = False del p for c, d, i in countertest: p = Bsdconv(c) if not p: print(Bsdconv.error()) print("Test failed at %s" % repr([c, i, o])) passed = False continue p.conv(d)
async def info_handler(request): cv = Bsdconv("utf-8:cns11643:bsdconv") us = list(request.GET["q"]) cs = [cv.conv(x).decode("utf-8") for x in us] r = [{"u":u,"p":c[2:4].lstrip('0'),"c":c[4:]} for u,c in zip(us, cs)] return web.json_response(r)
async def comp_handler(request): c = Bsdconv("bsdconv:zh-comp:utf-8") r = c.conv(request.GET["q"]).decode("utf-8") return web.json_response({"result":r})
async def decomp_handler(request): c = Bsdconv("utf-8:zh-decomp:split:bsdconv-keyword,bsdconv") a = c.conv(request.GET["q"]).decode("utf-8").strip(",").split(",") return web.json_response(a)
["utf-8:count:null","123Б測試",{"COUNT":6}], ["utf-8:count#blah:null","123Б測試",{"BLAH":6}], ["utf-8:count#for=lala&for=cjk:null","123Б測a試bc",{"COUNT":2}], ] passed=True for c, i, o in iotest: p=Bsdconv(c) if not p: print(Bsdconv.error()) print("Test failed at %s" % repr([c, i, o])) del p passed=False continue r=p.conv(i) if o != r: print("Test failed at %s" % repr([c, i, o])) print("expected(%d): %s" % (len(o), repr(o))) print("result(%d): %s" % (len(r), repr(r))) passed=False del p for c, d, i in countertest: p=Bsdconv(c) if not p: print(Bsdconv.error()) print("Test failed at %s" % repr([c, i, o])) passed=False continue p.conv(d)
# -*- coding: utf-8 -*- # python nfkc_gen.py '⁰¹²³'|sort|uniq import sys from bsdconv import Bsdconv nfkc = Bsdconv("utf-8:nfkc:utf-8") i = sys.argv[1].decode("utf-8") for c in i: c = c.encode("utf-8") d = nfkc.conv(c) if c == d: continue print("{}\t{}".format(d, c))
m = pq(m) if ftype not in fontdata: fontdata[ftype] = {} fontdata[ftype][m.attr("legacy")]=m.attr("unicode") def fwalk(ftype): ret = dict(fontdata[ftype]) if fonttype[ftype]["inherit"]: inh = fwalk(fonttype[ftype]["inherit"]) for k in inh: ret[k] = inh[k] return ret for ftype in fonttype: if fonttype[ftype]["hidden"]: continue fout = open("modules/inter/KHMER-"+ftype.upper()+".txt", "w") data = [] mapping = fwalk(ftype) for f in mapping: t = mapping[f] f = bcv1252.conv(b"".join([chr(eval(x)) for x in f.split(";")])).rstrip(",") t = bcv.conv(t.encode("utf-8")).rstrip(",") data.append((f, t)) data = sorted(data) for f, t in data: fout.write("%s\t%s\n" % (f, t)) fout.close()
#!/usr/bin/env python import sys from bsdconv import Bsdconv p=Bsdconv(sys.argv[1]) if not p: print(Bsdconv.error()) del p sys.exit() s=sys.stdin.read() print(p.conv(s)) print('====================================') print(p.counter()) del p
# -*- coding: utf-8 -*- # python nfkc_gen.py '⁰¹²³'|sort|uniq import sys from bsdconv import Bsdconv nfkc = Bsdconv("utf-8:nfkc:utf-8") i = sys.argv[1].decode("utf-8") for c in i: c = c.encode("utf-8") d = nfkc.conv(c) if c==d: continue print("{}\t{}".format(d, c))