def transliterate(self, k, cat):
ret = k
gk = k
c = Bsdconv("bsdconv:{0}".format(cat))
c.conv(p01(k))
if not c.counter("OERR"):
return ret
for layer in self.layers:
gk = layer[0].rdata.get(gk)
t = layer[1].get(gk).get(cat, None)
if t:
ret = t
break
else:
gk = layer[0].rdata.get(gk, None)
return ret
def _print_group1(self, g):
c = Bsdconv("bsdconv:utf-8")
print("###Layer 1 Group {0}\n".format(g))
print("####Attributes")
a = self.layers[0][1].get(g)
for cat in a:
v = a[cat]
print(" * {0}: {1} ({2})".format(cat, c.conv(p01(v)), v))
print("")
print("####Member")
d = self.layers[0][0].data.get(g)
for e in d:
print(" * {0} ({1})".format(c.conv(p01(e)), e))
print("")
def _print_group2(self, g2):
c = Bsdconv("bsdconv:utf-8")
print("#Layer 2 Group {0}\n".format(g2))
print("##Attributes")
a = self.layers[1][1].get(g2)
for cat in a:
v = a[cat]
print("* {0}: {1} ({2})".format(cat, c.conv(p01(v)), v))
print("##Member")
for g in self.layers[1][0].data.get(g2):
self._print_group1(g)
sep=re.compile(r"\s+")
f=open(sys.argv[1])
for l in f:
l = l.strip()
if l == "":
continue
if l.startswith("#"):
clist.write(l+"\n")
plist.write(l+"\n")
a = sep.split(l)
p = a[0]
ln = len(p.decode("utf-8"))
if ln > 1:
bonus = 6
p = bcv_zhtw.conv(p).rstrip(",")
of = plist
else:
try:
bonus = int(a[1])
except:
bonus = 0
sc.counter_reset()
sc.conv(p)
score = sc.counter("SCORE")
if score < 5*ln:
bonus += 5*ln - score
if bonus == 0:
continue
p = bcv.conv(p).rstrip(",")
of = clist
["utf-8:count#for=lala&for=cjk:null", "123Б測a試bc", {
"COUNT": 2
}],
]
passed = True
for c, i, o in iotest:
p = Bsdconv(c)
if not p:
print(Bsdconv.error())
print("Test failed at %s" % repr([c, i, o]))
del p
passed = False
continue
r = p.conv(i)
if o != r:
print("Test failed at %s" % repr([c, i, o]))
print("expected(%d): %s" % (len(o), repr(o)))
print("result(%d): %s" % (len(r), repr(r)))
passed = False
del p
for c, d, i in countertest:
p = Bsdconv(c)
if not p:
print(Bsdconv.error())
print("Test failed at %s" % repr([c, i, o]))
passed = False
continue
p.conv(d)
async def info_handler(request):
cv = Bsdconv("utf-8:cns11643:bsdconv")
us = list(request.GET["q"])
cs = [cv.conv(x).decode("utf-8") for x in us]
r = [{"u":u,"p":c[2:4].lstrip('0'),"c":c[4:]} for u,c in zip(us, cs)]
return web.json_response(r)
async def comp_handler(request):
c = Bsdconv("bsdconv:zh-comp:utf-8")
r = c.conv(request.GET["q"]).decode("utf-8")
return web.json_response({"result":r})
async def decomp_handler(request):
c = Bsdconv("utf-8:zh-decomp:split:bsdconv-keyword,bsdconv")
a = c.conv(request.GET["q"]).decode("utf-8").strip(",").split(",")
return web.json_response(a)
sep=re.compile(r"\s+")
f=open(sys.argv[1])
for l in f:
l = l.strip()
if l == "":
continue
if l.startswith("#"):
clist.write(l+"\n")
plist.write(l+"\n")
a = sep.split(l)
p = a[0]
ln = len(p.decode("utf-8"))
if ln > 1:
bonus = 6
p = bcv_zhtw.conv(p).rstrip(",")
of = plist
else:
try:
bonus = int(a[1])
except:
bonus = 0
sc.counter_reset()
sc.conv(p)
score = sc.counter("SCORE")
if score < 5*ln:
bonus += 5*ln - score
if bonus == 0:
continue
p = bcv.conv(p).rstrip(",")
of = clist
["utf-8:count:null","123Б測試",{"COUNT":6}],
["utf-8:count#blah:null","123Б測試",{"BLAH":6}],
["utf-8:count#for=lala&for=cjk:null","123Б測a試bc",{"COUNT":2}],
]
passed=True
for c, i, o in iotest:
p=Bsdconv(c)
if not p:
print(Bsdconv.error())
print("Test failed at %s" % repr([c, i, o]))
del p
passed=False
continue
r=p.conv(i)
if o != r:
print("Test failed at %s" % repr([c, i, o]))
print("expected(%d): %s" % (len(o), repr(o)))
print("result(%d): %s" % (len(r), repr(r)))
passed=False
del p
for c, d, i in countertest:
p=Bsdconv(c)
if not p:
print(Bsdconv.error())
print("Test failed at %s" % repr([c, i, o]))
passed=False
continue
p.conv(d)
# -*- coding: utf-8 -*-
# python nfkc_gen.py '⁰¹²³'|sort|uniq
import sys
from bsdconv import Bsdconv
nfkc = Bsdconv("utf-8:nfkc:utf-8")
i = sys.argv[1].decode("utf-8")
for c in i:
c = c.encode("utf-8")
d = nfkc.conv(c)
if c == d:
continue
print("{}\t{}".format(d, c))
m = pq(m)
if ftype not in fontdata:
fontdata[ftype] = {}
fontdata[ftype][m.attr("legacy")]=m.attr("unicode")
def fwalk(ftype):
ret = dict(fontdata[ftype])
if fonttype[ftype]["inherit"]:
inh = fwalk(fonttype[ftype]["inherit"])
for k in inh:
ret[k] = inh[k]
return ret
for ftype in fonttype:
if fonttype[ftype]["hidden"]:
continue
fout = open("modules/inter/KHMER-"+ftype.upper()+".txt", "w")
data = []
mapping = fwalk(ftype)
for f in mapping:
t = mapping[f]
f = bcv1252.conv(b"".join([chr(eval(x)) for x in f.split(";")])).rstrip(",")
t = bcv.conv(t.encode("utf-8")).rstrip(",")
data.append((f, t))
data = sorted(data)
for f, t in data:
fout.write("%s\t%s\n" % (f, t))
fout.close()
#!/usr/bin/env python
import sys
from bsdconv import Bsdconv
p=Bsdconv(sys.argv[1])
if not p:
print(Bsdconv.error())
del p
sys.exit()
s=sys.stdin.read()
print(p.conv(s))
print('====================================')
print(p.counter())
del p
# -*- coding: utf-8 -*-
# python nfkc_gen.py '⁰¹²³'|sort|uniq
import sys
from bsdconv import Bsdconv
nfkc = Bsdconv("utf-8:nfkc:utf-8")
i = sys.argv[1].decode("utf-8")
for c in i:
c = c.encode("utf-8")
d = nfkc.conv(c)
if c==d:
continue
print("{}\t{}".format(d, c))
