class _ShioriBean: def __init__(self): self.enabled = True self._renderMutex = QMutex() def renderKorean(self, text): """ @param text unicode @return unicode """ ret = "" if self._renderMutex.tryLock(): ret = dictman.manager().renderKorean(text) self._renderMutex.unlock() else: dwarn("ignore thread contention") return ret def renderJapanese(self, text): """ @param text unicode @return unicode """ ret = "" feature = GrimoireBean.instance.lookupFeature(text) if self._renderMutex.tryLock(): ret = skthreads.runsync(partial( dictman.manager().renderJapanese, text, exact=True,feature=feature)) # do exact match for speed self._renderMutex.unlock() else: dwarn("ignore thread contention") return ret def renderJson(self, data): """ @param data unicode json @return unicode """ ret = "" if self._renderMutex.tryLock(): try: data = json.loads(data) ret = self._renderJson(**data) or '' except Exception, e: dwarn(e) self._renderMutex.unlock() else:
class _TermManager: instance = None # _TermManager needed for updateTime def __init__(self, q): _TermManager.instance = self #self.convertsChinese = False self.enabled = True # bool self.hentai = False # bool self.marked = False # bool self.rubyEnabled = False # bool self.chineseRubyEnabled = False # bool self.koreanRubyEnabled = False # bool self.vietnameseRubyEnabled = False # bool #self.syntax = False # bool # For saving terms self.updateTime = 0 # float self.targetLanguage = 'ja' # str targetLanguage self.saveMutex = QMutex() self.scripts = { } # {(str type, str fr, str to):TranslationScriptPerformer or ScriptCoder} self.scriptLocks = {} # {(str lang, str fr, str to):bool} self.scriptTimes = {} # [(str lang, str fr, str to):float time] self.proxies = {} # {(str fr, str to):TranslationProxy} #self.rbmt = {} # {str language:rbmt.api.Translator} #self.rbmtTimes = {} # [str language:float time] t = self.saveTimer = QTimer(q) t.setSingleShot(True) t.setInterval(2000) # wait for 2 seconds for rebuilding t.timeout.connect(self.saveTerms) q.invalidateCacheRequested.connect(t.start, Qt.QueuedConnection) q.cacheChangedRequested.connect(q.cacheChanged, Qt.QueuedConnection) def rebuildCacheLater(self, queued=False): if queued: self.q.invalidateCacheRequested.emit() else: self.saveTimer.start() def saveTerms(self): if not self.scriptTimes: return if not self.saveMutex.tryLock(): dwarn("retry later due to thread contention") self.rebuildCacheLater(queued=True) return saveTime = time() skthreads.runsync(partial(self._saveTerms, saveTime)) self.saveMutex.unlock() def _saveTerms(self, createTime): """Invoked async @param createTime float """ #for lang,ts in self.targetLanguages.iteritems(): scriptTimes = self.scriptTimes #rbmtTimes = self.rbmtTimes if not scriptTimes or createTime < self.updateTime: return dprint("enter") if scriptTimes and createTime >= self.updateTime: self._saveScriptTerms(createTime=createTime, times=scriptTimes) if createTime >= self.updateTime: dprint("cache changed") self.q.cacheChangedRequested.emit() dprint("leave") def _saveScriptTerms(self, createTime, times): """ @param createTime float @param times {str key:float time} """ dprint("enter") dm = dataman.manager() gameIds = dm.currentGameIds() dprint("current series gameIds = %s" % gameIds) if gameIds: gameIds = set(gameIds) # in case it is changed during iteration l = [ t.d for t in dm.terms() if not t.d.disabled and not t.d.deleted and t.d.pattern ] # filtered _termman.sort_terms(l) w = _termman.TermWriter( termData=l, gameIds=gameIds, hentai=self.hentai, createTime=createTime, parent=self, rubyEnabled=self.rubyEnabled, chineseRubyEnabled=self.rubyEnabled and self.chineseRubyEnabled, koreanRubyEnabled=self.rubyEnabled and self.koreanRubyEnabled, vietnameseRubyEnabled=self.rubyEnabled and self.vietnameseRubyEnabled, ) #for scriptKey,ts in times.iteritems(): for scriptKey, ts in times.items(): # back up items if ts < self.updateTime: # skip language that does not out of date type, to, fr = scriptKey macros = w.queryMacros(to, fr) if w.isOutdated(): dwarn("leave: cancel saving out-of-date terms") return path = rc.term_path(type, to=to, fr=fr) # unicode dir = os.path.dirname(path) # unicode path if not os.path.exists(dir): skfileio.makedirs(dir) scriptKey = type, to, fr if self.scriptLocks.get(scriptKey): raise Exception("cancel saving locked terms") self.scriptLocks[scriptKey] = True man = self.scripts.get(scriptKey) if not man: man = self.scripts[scriptKey] = _make_script_interpreter( type) #elif not man.isEmpty(): # man.clear() else: #man.mutex.lock() if not man.isEmpty(): man.clear() #man.mutex.unlock() try: if type == 'trans': proxies = self.proxies.get((to, fr)) if proxies: del proxies[:] if w.saveTerms(path, type, to, fr, macros): #man.mutex.lock() ok = man.loadScript(path) #man.mutex.unlock() if ok and type == 'trans': self.proxies[(to, fr)] = w.queryProxies(to, fr) dprint("type = %s, to = %s, fr = %s, count = %s" % (type, to, fr, man.size())) except: self.scriptLocks[scriptKey] = False raise self.scriptLocks[scriptKey] = False times[scriptKey] = createTime dprint("leave") def applyTerms(self, text, type, to, fr, context='', host='', mark=False, ignoreIfNotReady=False): """ @param text unicode @param type str @param to str language @param* fr str language @param* context str @param* host str @param* mark bool or None @param* ignoreIfNotReady bool """ if mark is None: mark = self.marked if type in ('encode', 'decode'): key = 'trans', to, fr # share the same manager else: key = type, to, fr man = self.scripts.get(key) if man is None: self.scriptTimes[key] = 0 self.rebuildCacheLater() if ignoreIfNotReady: dwarn("ignore text while processing shared dictionary") growl.notify(my.tr("Processing Shared Dictionary") + "...") return '' if self.scriptLocks.get(key): dwarn("skip applying locked script") if ignoreIfNotReady: dwarn("ignore text while processing shared dictionary") growl.notify(my.tr("Processing Shared Dictionary") + "...") return '' return text if not man: #or man.isEmpty(): return text #if not man.mutex.tryLock(TERM_TRYLOCK_INTERVAL): # dwarn("try lock timeout") # return text ret = text if not man.isEmpty(): category = _termman.make_category(context=context, host=host) if type == 'encode': ret = man.encode(text, category) elif type == 'decode': ret = man.decode(text, category, mark) else: ret = man.transform(text, category, mark) #man.mutex.unlock() return ret def warmupTerms(self, type, to, fr): """ @param type str @param to str language @param* fr str language """ if type in ('encode', 'decode'): key = 'trans', to, fr # share the same manager else: key = type, to, fr if key not in self.scripts: self.scriptTimes[key] = 0 self.rebuildCacheLater() _rx_delegate = re.compile(r"{{" r"([a-zA-Z0-9,_$]+)" # TR_RE_TOKEN r"<[-0-9<>]+>" r"}}") def delegateTranslation(self, text, to, fr, context, host, proxies, proxyDigit): """ @param text unicode @param to str language @param fr str language @param context str @param host str @param proxies {unicode input:unicode output} @param proxyDigit proxy using letters or digits @return unicode """ if '{{' not in text or '}}' not in text: return text if to in DELEGATE_DISABLED_LANGUAGES: unused_proxies = None # disabled else: unused_proxies = self.proxies.get((to, fr)) # [TranslationProxy] used_proxy_input = set() # [unicode] used_proxy_output = set() # [unicode] category = _termman.make_category(context=context, host=host) def fn(m): # re.match -> unicode matched_text = m.group() role = m.group(1) ret = None if unused_proxies: for proxy in unused_proxies: if (proxy.match_category(category) and proxy.match_role(role) and proxy.input not in used_proxy_input and proxy.output not in used_proxy_output and proxy.input not in text and proxy.output not in text): used_proxy_input.add(proxy.input) used_proxy_output.add(proxy.output) proxies[proxy.output] = matched_text return proxy.input index = len(proxies) ret = defs.term_role_proxy(role, index, proxyDigit=proxyDigit) proxies[ret] = matched_text return ret return self._rx_delegate.sub(fn, text) _rx_undelegate_latin = re.compile(r"Z[A-Y]+Z") _rx_undelegate_digit = re.compile(r"9[0-8]+9") def undelegateTranslation(self, text, to, fr, proxies, proxyDigit=False): """ @param text unicode @param to str language @param fr str language @param* proxies {unicode input:unicode output} @param* proxyDigit proxy using letters or digits @return unicode """ if not proxies: return text rx = None if proxyDigit: if '9' in text: rx = self._rx_undelegate_digit else: if 'Z' in text: rx = self._rx_undelegate_latin if rx: def fn(m): # re.match -> unicode matched_text = m.group() try: return proxies.pop(matched_text) except KeyError: return matched_text text = rx.sub(fn, text) if proxies: #to_latin = config.is_latin_language(to) for k, v in proxies.iteritems(): text = text.replace(k, v) return text
class _TtsManager(object): def __init__(self, q): self.defaultEngineKey = '' # str self._online = True # bool self._speakTask = None # partial function object self._zunkoEngine = None # _ttsman.ZunkoEngine self._zunkoMutex = QMutex() self._yukariEngine = None # _ttsman.YukariEngine self._sapiEngines = {} # {str key:_ttsman.SapiEngine} self._onlineEngines = {} # {str key:_ttsman.OnlineEngine} self._voiceroidEngines = {} # {str key:_ttsman.VoiceroidEngine} self._voicetextEngines = {} # {str key:_ttsman.VoiceTextEngine} #self.defaultEngineKey = 'wrong engine' #self.defaultEngineKey = 'VW Misaki' #self.defaultEngineKey = 'VW Show' #self.defaultEngineKey = 'zunko' #self.defaultEngineKey = 'yukari' #self.defaultEngineKey = 'google' t = self._speakTimer = QTimer(q) t.setSingleShot(True) t.timeout.connect(self._doSpeakTask) @staticmethod def _repairText(text, key=''): """ @param text unicode @param* key str engine key @return unicode """ ret = text.replace(u'…', '.') # てんてんてん #if key == 'zunko.offline': # ret = textutil.repair_zunko_text(ret) return ret def iterActiveEngines(self): """ @yield engine """ if self._online: for it in self._onlineEngines.itervalues(): if it.isValid(): yield it for it in self._voiceroidEngines.itervalues(): if it.isValid(): yield it for it in self._voicetextEngines.itervalues(): if it.isValid(): yield it for it in self._zunkoEngine, self._yukariEngine: if it and it.isValid(): yield it for it in self._sapiEngines.itervalues(): if it.isValid(): yield it def stop(self): for it in self.iterActiveEngines(): #if it.type != 'sapi': # do not disable sapi TTS to make it faster it.stop() def speakLater(self, text, interval, **kwargs): # unicode, long -> self._speakTask = partial(self.speak, text, **kwargs) self._speakTimer.start(interval) def speak(self, text, engine='', termEnabled=False, language='', gender='', verbose=True): """ @param text unicode @param* engine str @param* termEnabled bool whether apply game-specific terms @param* language str @param* gender str @param* verbose bool whether warn on error """ #if not features.TEXT_TO_SPEECH: if not text or jpchars.allpunct(text): #self.stop() # stop is not needed for games return eng = self.getEngine(engine) if not eng and self.defaultEngineKey and self.defaultEngineKey != engine: eng = self.getEngine(self.defaultEngineKey) if not eng: if verbose: growl.warn(my.tr("TTS is not available in Preferences")) dprint("missing engine: %s" % (engine or self.defaultEngineKey)) return if not eng.isValid(): #if verbose: # Always warn growl.warn('<br/>'.join(( my.tr("TTS is not available"), eng.name, ))) dprint("invalid engine: %s" % eng.key) return if eng.online and not self._online: dprint("ignore when offline: %s" % eng.key) return if language == '?': # language = unilang.guess_language(text) or 'ja' if language and eng.language and eng.language != '*' and language[:2] != eng.language[:2]: #if verbose: # growl.notify("<br/>".join(( # my.tr("TTS languages mismatch"), # "%s: %s != %s" % (eng.key, language, eng.language)))) dprint("language mismatch: %s != %s" % (language, eng.language)) eng = self.getAvailableEngine(language) if not eng: return if termEnabled: #and (not language or language == 'ja'): text = termman.manager().applyTtsTerms(text, language) # Even if text is empty, trigger stop tts #if not text: # return text = self._repairText(text, eng.key) # always apply no matter terms are enabled or not #if eng.type == 'sapi': # eng.speak(text, async=True) #else: #with SkProfiler(): if text: eng.speak(text, language, gender) # 0.007 ~ 0.009 seconds for SAPI else: eng.stop() #skevents.runlater(partial(eng.speak, text)) def getAvailableEngine(self, language): """ @param language str @return _ttsman.VoiceEngine or None """ if not self.online: dprint("ignore when offline") return None key = 'naver' if language == 'ja' else 'baidu' if language.startswith('zh') else 'google' return self.getEngine(key) @property def online(self): return self._online @online.setter def online(self, v): if v != self._online: self._online = v # Voiceroid @property def yukariEngine(self): if not self._yukariEngine: ss = settings.global_() eng = self._yukariEngine = _ttsman.YukariEngine(path=ss.yukariLocation()) ss.yukariLocationChanged.connect(eng.setPath) growl.msg(' '.join(( my.tr("Load TTS"), eng.name, ))) return self._yukariEngine @property def zunkoEngine(self): if not self._zunkoEngine: if self._zunkoMutex.tryLock(): eng = self._zunkoEngine = _ttsman.ZunkoEngine( mutex = self._zunkoMutex, volume=self.getVolume(_ttsman.ZunkoEngine.key)) settings.global_().zunkoLocationChanged.connect(eng.setPath) #growl.msg(' '.join(( # my.tr("Load TTS"), # eng.name, #))) self._zunkoMutex.unlock() else: dwarn("ignored due to thread contention") return self._zunkoEngine # Online def getOnlineEngine(self, key): ret = self._onlineEngines.get(key) if not ret and key in _ttsman.ONLINE_ENGINES: ret = _ttsman.OnlineEngine.create(key) if ret: ret.speed = self.getSpeed(key) ret.pitch = self.getPitch(key) ret.volume = self.getVolume(key) ret.gender = self.getGender(key) if ret and ret.isValid(): self._onlineEngines[key] = ret growl.msg(' '.join(( my.tr("Load TTS"), ret.name, ))) else: ret = None growl.warn(' '.join(( my.tr("Failed to load TTS"), key, ))) return ret # SAPI def getSapiEngine(self, key): ret = self._sapiEngines.get(key) if not ret: ret = _ttsman.SapiEngine(key=key, speed=self.getSpeed(key), pitch=self.getPitch(key), volume=self.getVolume(key), #gender=self.getGender(key), ) if ret.isValid(): self._sapiEngines[key] = ret growl.msg(' '.join(( my.tr("Load TTS"), ret.name, ))) else: ret = None growl.warn(' '.join(( my.tr("Failed to load TTS"), key, ))) return ret def getPitch(self, key): """ @param key str @return int """ try: return int(settings.global_().ttsPitches().get(key) or 0) except (ValueError, TypeError): return 0 def setPitch(self, key, v): """ @param key str @param v int """ ss = settings.global_() m = ss.ttsPitches() if v != m.get(key): m[key] = v ss.setTtsPitches(m) eng = self.getCreatedEngine(key) if eng: eng.setPitch(v) def getVolume(self, key): """ @param key str @return int """ try: return int(settings.global_().ttsVolumes().get(key) or 100) except (ValueError, TypeError): return 100 def setVolume(self, key, v): """ @param key str @param v int """ ss = settings.global_() m = ss.ttsVolumes() if v != m.get(key): m[key] = v ss.setTtsVolumes(m) eng = self.getCreatedEngine(key) if eng: eng.setVolume(v) def getSpeed(self, key): """ @param key str @return int """ try: return int(settings.global_().ttsSpeeds().get(key) or 0) except (ValueError, TypeError): return 0 def setSpeed(self, key, v): """ @param key str @param v int """ ss = settings.global_() m = ss.ttsSpeeds() if v != m.get(key): m[key] = v ss.setTtsSpeeds(m) eng = self.getCreatedEngine(key) if eng: eng.setSpeed(v) def getGender(self, key): """ @param key str @return str """ try: return settings.global_().ttsGenders().get(key) or 'f' except (ValueError, TypeError): return 'f' def setGender(self, key, v): """ @param key str @param v str """ ss = settings.global_() m = ss.ttsGenders() if v != m.get(key): m[key] = v ss.setTtsGenders(m) eng = self.getCreatedEngine(key) if eng: eng.setGender(v) # Actions def getEngine(self, key): """ @return _ttsman.VoiceEngine or None """ if not key: return None if key == 'zunkooffline': return self.zunkoEngine if key == 'yukarioffline': return self.yukariEngine return self.getOnlineEngine(key) or self.getSapiEngine(key) def getCreatedEngine(self, key): """ @return _ttsman.VoiceEngine or None """ if not key: return None if key == 'zunkooffline': return self._zunkoEngine if key == 'yukarioffline': return self._yukariEngine return self._onlineEngines.get(key) or self._voiceroidEngines.get(key) or self._voicetextEngines.get(key) or self._sapiEngines.get(key) #@memoizedproperty #def speakTimer(self): # ret = QTimer(self.q) # ret.setSingleShot(True) # ret.timeout.connect(self._doSpeakTask) # return ret def _doSpeakTask(self): if self._speakTask: try: self._speakTask() except Exception, e: dwarn(e) self._speakTask = None