def _unmarshalInteger(s): # str -> int, use hex
#try: return int(s, 16) #if s.startswith('0x') else int(s)
try:
return int(s)
except ValueError:
dwarn("failed to marshal number %s" % s)
return 0
def translate(text, to='zhs', fr='ja'):
try:
query = text
salt = random.randint(32768, 65536)
sign = appid + query + str(salt) + secretKey
sign = hashlib.md5(sign.encode('utf-8')).hexdigest()
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
data = {
"from": "jp",
"to": "zh",
"q": query,
"salt": str(salt),
"sign": sign,
"appid": appid
}
url = 'https://api.fanyi.baidu.com/api/trans/vip/translate'
res = requests.post(url=url, headers=headers, data=data, timeout=4)
if res.ok:
result = json.loads(res.text)['trans_result'][0]['dst']
return result
else:
derror('error')
pass
except requests.ConnectionError, e:
dwarn("connection error", e.args)
def _writeCodecLine(f, tid, pattern, repl, regex, icase, context, host,
role):
"""
@param f file
@param tid long
@param pattern unicode
@param repl unicode
@param regex bool
@param icase bool
@param context str
@param host str
@param role str
@return unicode or None
"""
if '\n' in pattern or '\n' in repl or '\n' in role:
dwarn("skip new line in term: id = %s" % tid)
return
cat = make_categories(context=context, host=host)
features = [str(tid), str(cat)]
flags = ''
if icase:
flags += 'i'
if regex:
flags += 'r'
if flags:
features.append(flags)
feature = ' '.join(features)
if not repl:
role = '' # disable role if need to delete replacement
cols = [role, pattern, repl, feature]
line = ' ||| '.join(cols) + '\n'
f.write(line)
def speak(self, text, language=None, gender=''):
""""@reimp"""
if self.mutex.tryLock():
self.engine.speak(text)
self.mutex.unlock()
else:
dwarn("ignored due to thread contention")
def translate(text, to='en', fr='ja'):
"""Return translated text, which is NOT in unicode format
@param text unicode not None
@param fr unicode not None, must be valid language code
@param to unicode not None, must be valid language code
@return unicode or None
"""
try:
r = session.post(
LEC_API,
headers=GZIP_HEADERS, # disabled as qtrequests does not support it
data=_make_post(text, to, fr))
#print r.headers['Content-Type']
ret = r.content
# return error message if not r.ok
# example response: {"t":[{"text":"hello"}]}
if r.ok and len(ret) > 100:
ret = _parse(ret)
else:
dwarn("return content too short")
return ret.decode('utf8', errors='ignore')
#except socket.error, e:
# dwarn("socket error", e.args)
except requests.ConnectionError, e:
dwarn("connection error", e.args)
def dll(self):
if not self._dll:
try:
self._dll = ctypes.WinDLL(self.dllPath)
dprint("gts sdk dll is loaded")
except Exception, e:
dwarn("failed to load gts sdk dll", e)
def translate(text, to='zhs', fr='ja'):
"""Return translated text, which is NOT in unicode format
@param text unicode not None
@param fr unicode not None, must be valid language code
@param to unicode not None, must be valid language code
@return unicode or None
"""
try:
r = session.post(API,
data={
'q': text,
'type': '-'.join((
_lang(fr),
_lang(to),
))
})
ret = r.content # content is "err" when failed
if r.ok and len(ret) > 20 and ret[0] == '{' and ret[-1] == '}':
#ret = ret.decode('utf8')
js = json.loads(ret)
#print json.dumps(js, indent=2, ensure_ascii=False)
return js['retcopy']
#except socket.error, e:
# dwarn("socket error", e.args)
except requests.ConnectionError, e:
dwarn("connection error", e.args)
def listen(self, time):
if time < self.time: # aborted
return
q = self.q
r = self.recognizer
while self.enabled:
try:
from google import googlesr
with googlesr.Microphone(
device_index=self.deviceIndex) as source:
dprint("listen start")
r.stopped = False
audio = r.listen(source)
dprint("listen stop")
except Exception, e:
dwarn("audio device error", e)
q.recognitionFinished.emit()
return
if time < self.time or self.aborted or not self.enabled: # aborted
return
if audio and len(audio.data) < MAX_AUDIO_SIZE:
skthreads.runasync(partial(self.recognize, audio))
else:
q.recognitionFinished.emit()
if time < self.time or self.aborted or self.singleShot:
return
def editPost(self, data): # json ->
try:
post = json.loads(data)
post['id'] = long(post['id'])
self.manager.editPost(**post)
except Exception, e:
dwarn(e)
def dll(self):
if not self._dll:
try:
self._dll = ctypes.WinDLL(self.DLL_MODULE)
dprint("D_JK dll is loaded")
except Exception, e:
dwarn("failed to load D_JK", e)
def setType(self, v): # str ->
try:
index = dataman.Term.TYPES.index(v)
except ValueError:
dwarn("unknown term type: %s" % v)
index = 0
self.typeEdit.setCurrentIndex(index)
def parse(data, *args, **kwargs):
"""
@param data string
@param inenc string
@param outenc string
@param* callback function(unicode word, [unicode] xmls)
@return None or [(unicode word, [unicode] xmls]
"""
if data:
try:
filetype = data[:4] # string
assert filetype == '?LD2'
version = byteutil.toshort(data, 0x18)
assert version == 2
#fileId = byteutil.tolong(data, 0x1c) # not used
# Intro
dataOffset = byteutil.toint(data, 0x5c) + 0x60
introType = byteutil.toint(data, dataOffset)
introOffset = byteutil.toint(data,
dataOffset + 4) + dataOffset + 12
if introType == 3: # without additional information
return _parsedict(data, dataOffset, *args, **kwargs)
elif len(data) > introOffset - 0x1c:
return _parsedict(data, introOffset, *args,
**kwargs) # skip the intro
else:
dwarn("invalid or corrupted file")
except Exception, e:
dwarn(e)
def evalmacros(macros, limit=1000):
"""
@param macros {unicode name:unicode value}
@param* limit int maximum iteration count
@return unicode
"""
for count in xrange(1, limit):
dirty = False
for pattern, text in macros.iteritems(
): # not iteritems as I will modify ret
if text and '{{' in text:
dirty = True
ok = False
for m in _RE_MACRO.finditer(text):
macro = m.group(1)
repl = macros.get(macro)
if repl:
text = text.replace("{{%s}}" % macro, repl)
ok = True
else:
dwarn("missing macro", macro, text)
ok = False
break
if ok:
macros[pattern] = text
else:
macros[pattern] = None # delete this pattern
if not dirty:
break
if count == limit - 1:
dwarn("recursive macro definition")
def setenvrc(path): # unicode path to dic rcfile
try:
if os.name == 'nt':
path = path.replace('/', os.path.sep)
os.environ['MECABRC'] = path
except Exception, e:
dwarn(e)
def dll(self):
if not self._dll:
try:
self._dll = ctypes.CDLL(ENGINE_DLL)
dprint("nova engine dll is loaded")
except Exception, e:
dwarn("failed to load nova", e)
def main(argv):
"""
@param argv [unicode]
@return int
"""
dprint("enter")
ok = False
if not argv or len(argv) == 1 and argv[0] in ('-h', '--help'):
usage()
#elif len(argv) != 1:
# dwarn("invalid number of parameters")
# usage()
else:
family = argv[-1]
quiet = '-q' in argv or '--quiet' in argv
try:
msg(family)
init()
ok = run(family)
if ok and not quiet:
from sakurakit import skos
font = FONTS[family]
path = os.path.join(FONT_DIR, font['path'])
if font['type'] == 'file':
path = os.path.dirname(path)
skos.open_location(os.path.abspath(path))
except Exception, e:
dwarn(e)
def _iterparsebrands(self, h):
"""
@param h unicode html
@yield {kw}
"""
try:
m = self._rx_brands.search(h)
if m:
line = m.group(1)
for hh in line.split(u'、'):
id = int(self._rx_brands_id.search(hh).group(1))
name = unescapehtml(
self._rx_brands_name.search(hh).group(1))
yield {
'id':
id, # int
'name':
name, # unicode
'img':
"http://media.erogetrailers.com/img/brand/%i.png" %
id, # str
#'url': "http://erogetrailers.com/brand/%i" % id, # not used
}
except Exception, e:
dwarn(e)
def quit(self):
if self.hasQuit:
return
self.hasQuit = True
dprint("schedule to kill all python instances in a few seconds")
skevents.runlater(lambda:
os.system("tskill pythonw & tskill python"),
config.QT_QUIT_TIMEOUT + config.QT_THREAD_TIMEOUT)
# wait for for done or kill all threads
from PySide.QtCore import QThreadPool
if QThreadPool.globalInstance().activeThreadCount():
dwarn("warning: wait for active threads")
QThreadPool.globalInstance().waitForDone(config.QT_THREAD_TIMEOUT)
dprint("leave qthread pool")
dprint("send quit signal to qApp")
qApp = QCoreApplication.instance()
# Make sure settings.sync is the last signal conneced with aboutToQuit
#qApp.aboutToQuit.connect(self.settings.sync)
skevents.runlater(qApp.quit)
def db2csv(csvpath, dbpath):
"""
@param csvpath unicode output
@param dbpath unicode dbpath
@return bool
"""
import sqlite3
from dictdb import dictdb
from dictp import edictp
try:
with sqlite3.connect(dbpath) as conn:
cur = conn.cursor()
q = dictdb.iterentries(cur)
with open(csvpath, 'w') as f:
for i, (word, content) in enumerate(q):
id = i + 1
entries = edictp.parseword(word)
roles = edictp.parsetransrole(content).split(',')
trans = edictp.parsetransdef(content).replace(
',', ' ') # ',' not allowed
lines = mdcompile.assemble(entries,
roles=roles,
id=id,
type='edict',
trans=trans,
surfacefilter=surfacefilter)
f.writelines(lines)
return True
except Exception, e:
dwarn(e)
return False
def translate(self, t, to='auto', fr='auto'):
"""
@param t unicode
@param* to str
@param* fr str
@return unicode or None
"""
try:
r = self.session.post(self.api,
headers=self.headers,
data={
'hl': googledef.lang2locale(to),
'sl': googledef.lang2locale(fr),
'q': t,
})
h = r.content
if h:
start = h.find(self._TEXT_BEGIN)
if start > 0:
start += len(self._TEXT_BEGIN)
stop = h.find(self._TEXT_END, start)
if stop > 0:
h = h[start:stop]
return unescapehtml(h)
#except socket.error, e:
# dwarn("socket error", e.args)
except requests.ConnectionError, e:
dwarn("connection error", e.args)
def lookup(self, *args, **kwargs):
try:
with sqlite3.connect(self.path) as conn:
for it in queryentries(conn.cursor(), *args, **kwargs):
yield it
except Exception, e:
dwarn(e)
def makedb(lang): # str -> bool
dprint("enter: lang = %s" % lang)
dic = DICS[lang]
ldpath = LD_DIR + '/' + lang + LD_SUFFIX
dbpath = DB_DIR + '/' + lang + DB_SUFFIX
tmppath = TMP_DIR + '/' + lang + DB_SUFFIX
from sakurakit import skfileio
for it in dbpath, tmppath:
if os.path.exists(it):
skfileio.removefile(it)
if not os.path.exists(ldpath):
dwarn("leave: ld does not exist: %s" % ldpath)
return False
ok = False
from lingoes.lingoesdb import LingoesDb
with SkProfiler("extract"):
if LingoesDb(tmppath).create(ldpath, dic['inenc'], dic['outenc']):
os.renames(tmppath, dbpath) # renames to create DB_DIR
ok = True
elif os.path.exists(tmppath):
skfileio.removefile(tmppath)
if os.path.exists(ldpath):
skfileio.removefile(ldpath)
dprint("leave: ok = %s" % ok)
return ok
def stop(self):
""""@reimp"""
if self.mutex.tryLock():
self.engine.stop()
self.mutex.unlock()
else:
dwarn("ignored due to thread contention")
def quit(self):
if self.hasQuit:
return
self.hasQuit = True
import curtheme
curtheme.unload()
for w in self.widgets:
if w.isVisible():
w.hide()
# wait for for done or kill all threads
from PySide.QtCore import QThreadPool
if QThreadPool.globalInstance().activeThreadCount():
dwarn("warning: wait for active threads")
QThreadPool.globalInstance().waitForDone(config.QT_THREAD_TIMEOUT)
dprint("leave qthread pool")
dprint("send quit signal to qApp")
qApp = QCoreApplication.instance()
# Make sure settings.sync is the last signal conneced with aboutToQuit
#qApp.aboutToQuit.connect(self.settings.sync)
skevents.runlater(qApp.quit)
def updateAndRefresh(self):
if self._locked:
dwarn("locked")
return
gameId = self.gameId
if not gameId:
#growl.notify(my.tr("Unknown game. Please try updating the database."))
return
if netman.manager().isOnline():
if self.itemId and self.itemId < defs.MIN_NORMAL_GAME_ITEM_ID:
dprint("found non-normal game, ignore refs")
else:
self._locked = True # lock before online access
#growl.msg(my.tr("Updating game information"))
ok = dataman.manager().updateReferences(gameId)
#dm.touchGames()
if not ok:
growl.notify(
my.
tr("The game title it not specified. You can click the Edit button to add one."
))
#else:
#growl.msg(my.tr("Found game information"))
self._locked = False # always unlock no matter what happened
self.refresh()
def __enter__(self):
self.locked = READ_MUTEX.tryLock()
if self.locked:
return self
else:
dwarn("failed to lock mutex due to contention")
return
def _writeTransformLine(f, tid, pattern, repl, regex, icase, context,
host):
"""
@param f file
@param tid long
@param pattern unicode
@param repl unicode
@param regex bool
@param icase bool
@param context str
@param host str
@return unicode or None
"""
if '\n' in pattern or '\n' in repl or '\t' in pattern or '\t' in repl:
dwarn("skip tab or new line in term: id = %s" % tid)
return
cat = make_categories(context=context, host=host)
cols = [str(tid), str(cat), pattern]
if repl:
cols.append(repl)
line = '\t'.join(cols)
line = "\t%s\n" % line # add leading/trailing spaces
if icase:
line = 'i' + line
if regex:
line = 'r' + line
f.write(line)
def compile(dic, csv, exe='mecab-dict-index', dicdir='', call=subprocess.call):
"""csv2dic. This process would take several seconds
@param dic unicode path
@param csv unicode path
@param* exe unicode path
@param* dicdir unicode
@param* call launcher function
@return bool
"""
# MeCab would crash for empty sized csv
if skfileio.filesize(csv) < MIN_CSV_SIZE:
dwarn("insufficient input csv size", csv)
return False
args = [
exe,
'-f',
'utf8', # from utf8
'-t',
'utf8', # to utf8
'-u',
dic,
csv,
]
if dicdir:
args.extend(('-d', dicdir))
return call(args) in (0, True) and os.path.exists(dic)
def _doSpeakTask(self):
if self._speakTask:
try:
apply(self._speakTask)
except Exception, e:
dwarn(e)
self._speakTask = None
def setLocation(self, v):
self.location = v
if v and not os.path.exists(v):
try: os.makedirs(v)
except OSError:
dwarn("warning: failed to create directory: %s" % v)
self.quit()
