def get(lang): # str -> bool filename = DICS[lang] url = DIC_URL % filename minsize = MIN_DIC_SIZE path = TMP_DIR + '/' + filename targetpath = TARGET_DIR + '/' + filename dprint("enter: url = %s, minsize = %s" % (url, minsize)) #from sakurakit import skfileio #if os.path.exists(path) and skfileio.filesize(path) == size: # dprint("leave: already downloaded") # return True ok = False from sakurakit import skfileio, sknetio with SkProfiler("fetch"): # gzip=True to automatically extract gzip # flush=false to use more memory to reduce disk access if sknetio.getfile(url, path, flush=False, gzip=True): ok = skfileio.filesize(path) > minsize if ok: os.renames(path, targetpath) elif os.path.exists(path): skfileio.removefile(path) dprint("leave: ok = %s" % ok) return ok
def dump(path, gaiji={}, inenc='euc_jp', outenc='utf8', output='eb.db'): """ @param path unicode @param* gaiji {str:str} @param* inenc str @param* outenc str """ dprint("enter") from sakurakit import skfileio skfileio.removefile(output) headings = set() # [unicode] dictdb.createdb(output) with sqlite3.connect(output) as conn: cur = conn.cursor() shiori = EBShiori(path, gaiji=gaiji) kana = jpchars.s_hira + jpchars.s_kata for i, ch in enumerate(kana): s = ch.encode(inenc) hits = shiori.search_word(s) dprint(ch, i, len(hits)) for hit in hits: h = hit.heading().decode(outenc) if h not in headings: headings.add(h) t = hit.text().decode(outenc) dictdb.insertentry(cur, (h, t)) conn.commit() dprint("leave")
def _getdata(url, path, tmppath=None, touchbad=False, mimefilter=None, **kwargs): """ @param token unicode @param path unicode @param* tmppath unicode @param* touchbad bool @param* mimefilter str @return bool """ if not tmppath: tmppath = path + TMP_SUFFIX data = sknetio.getdata(url, mimefilter=mimefilter, **kwargs) #if not data: # url = _fixtwimg(url) # if url: # return _getdata(url, path, tmppath=tmppath, mimefilter=mimefilter, **kwargs) ok = bool(data) and skfileio.writedata(tmppath, data) if ok and skfileio.rename(tmppath, path): return True else: skfileio.removefile(tmppath) if touchbad: badpath = path + BAD_SUFFIX skfileio.touchfile(badpath) return False
def compile(): # -> bool dprint("enter") dbpath = TARGET_DIR + '/' + DB_FILENAME dicpath = TARGET_DIR + '/' + DIC_FILENAME csvpath = TARGET_DIR + '/' + CSV_FILENAME tmpdicpath = dicpath + '.tmp' from mecabdic import mdedict with SkProfiler("assemble csv"): ok = mdedict.db2csv(csvpath, dbpath) if ok: with SkProfiler("compile dic"): ok = mdedict.csv2dic(tmpdicpath, csvpath, exe=MECAB_EXEPATH, dicdir=UNIDIC_DICPATH) from sakurakit import skfileio if ok: if os.path.exists(dicpath): skfileio.remove(dicpath) os.rename(tmpdicpath, dicpath) elif os.path.exists(tmpdicpath): skfileio.removefile(tmpdicpath) if os.path.exists(csvpath): skfileio.removefile(csvpath) dprint("leave: ok = %s" % ok) return ok
def get(): # -> bool targetpath = INST_DIR + '/' + APPLOC_FILENAME tmppath = TMP_DIR + '/' + APPLOC_FILENAME url = APPLOC_URL size = APPLOC_FILESIZE dprint("enter: size = %s, url = %s" % (size, url)) from sakurakit import skfileio if os.path.exists(targetpath) and skfileio.filesize(targetpath) == size: dprint("leave: already downloaded") return True from sakurakit import sknetio ok = False with SkProfiler("fetch"): if sknetio.getfile(url, tmppath, flush=False): # flush=false to use more memory to reduce disk access ok = skfileio.filesize(tmppath) == size if ok: os.rename(tmppath, targetpath) if not ok and os.path.exists(tmppath): skfileio.removefile(tmppath) dprint("leave: ok = %s" % ok) return ok
def _continue(self): if self._cvid and self._cpath and not _is_good_video(self._cpath): if os.path.exists(self._cpath): growl.warn("[main] %s: %s" % (my.tr("remove failed file"), self._cpath)) skfileio.removefile(self._cpath) self._retry() else: self._next()
def clearTemporaryFiles(): from sakurakit import skfileio for tmpdir in rc.DIR_CACHE_DATA, : try: for root, dirs, files in os.walk(tmpdir): for f in files: if f.endswith(TMP_SUFFIX): p = os.path.join(root, f) skfileio.removefile(p) except Exception, e: dwarn(e)
def saveClosedUrls(self): path = rc.CLOSE_HISTORY_LOCATION if self.closedUrls: l = self.closedUrls if len(l) > config.CLOSE_HISTORY_SIZE: l = l[len(l) - config.CLOSE_HISTORY_SIZE:] data = '\n'.join(l) skfileio.writefile(path, data) else: skfileio.removefile(path) dprint("pass")
def _savePixmap(self, pm): """ @param pm QPixmap or QImage @param path unicode @return bool """ path = self._randomPath() ret = save_pixmap(pm, path) if ret: skfileio.removefile(self.path) self.setPath(path) return ret
def extract(dic): # str -> bool dprint("enter: dic = %s" % dic) srcpath = TMP_DIR + '/' + FILENAME_TPL % dic tmppath = TMP_DIR + '/cabocha.' + dic targetpath = DIC_DIR + '/' + dic import shutil from sakurakit import skfileio with SkProfiler("extract"): ok = skfileio.extracttarbz2(srcpath, tmppath) if ok: dickey = 'ipa' if dic == 'ipadic' else dic for it in MODELS: model = "%s.%s.model" % (it, dickey) if not os.path.exists(tmppath + '/' + model): ok = False dwarn("missing model: %s" % model) if os.path.exists(targetpath): # alway remove old path even if not ok shutil.rmtree(targetpath) if ok: os.renames(tmppath, targetpath) #child = skfileio.getfirstchilddir(tmppath) #modeldir = child + '/model' #if not os.path.exists(modeldir): # dwarn("model dir does not exist") #else: # from cabochajlp import cabochamodel # if os.path.exists(targetpath): # shutil.rmtree(targetpath) # os.makedirs(targetpath) # dickey = 'ipa' if dic == 'ipadic' else dic # for model in MODELS: # with SkProfiler(): # dprint("process model: %s" % model) # input = "%s.%s.txt" % (model , dickey) # output = "%s.%s.model" % (model , dickey) # inputpath = os.path.abspath(modeldir + '/' + input) # outputpath = os.path.abspath(modeldir + '/' + output) # if cabochamodel.modelindex(outputpath, inputpath): # os.renames(outputpath, targetpath + '/' + output) # else: # dwarn("failed to process model %s" % model) if os.path.exists(tmppath): shutil.rmtree(tmppath) skfileio.removefile(srcpath) dprint("leave: ok = %s" % ok) return ok
def saveTabs(self): # -> bool ret = False urls = [] # [unicode url] for w in self._iterTabWidgets(): url = w.url().toString() if url != BLANK_URL: urls.append(url) path = rc.TABS_LOCATION if urls: data = '\n'.join(urls) ret = skfileio.writefile(path, data) else: skfileio.removefile(path) dprint("pass: ret = %s" % ret) return ret
def ld2db(ldpath, dbpath=None, inenc=None, outenc=None): """ @param ldpath unicode @param* dbpath unicode @param* inenc str @param* outenc str """ if not inenc: inenc = 'utf8' if not outenc: outenc = inenc dprint("enter: inenc = %s, outenc = %s" % (inenc, outenc)) if not dbpath or os.path.isdir(dbpath): basename = os.path.basename(ldpath) if ldpath.endswith(LD_SUFFIX): path = os.path.splitext(basename)[0] + DB_SUFFIX else: path = basename if dbpath: dbpath = os.path.join(dbpath, path) else: dbpath = path if not os.path.exists(ldpath): dwarn("ld file does not exist: %s" % ldpath) return False if os.path.exists(dbpath): dwarn("db file already exist: %s" % dbpath) return False tmppath = dbpath + TMP_SUFFIX from sakurakit import skfileio if os.path.exists(tmppath): skfileio.removefile(tmppath) ok = False from lingoes.lingoesdb import LingoesDb with SkProfiler(): if LingoesDb(tmppath).create(ldpath, inenc, outenc): os.renames(tmppath, dbpath) # renames to create DB_DIR ok = True elif os.path.exists(tmppath): skfileio.removefile(tmppath) dprint("leave: ok = %s" % ok) return ok
def get(dic): # str -> bool url = DIC_URL + DICS[dic]['file'] path = TMP_DIR + '/' + DICS[dic]['file'] dprint("enter: dic = %s, url = %s" % (dic, url)) #from sakurakit import skfileio #if os.path.exists(path) and skfileio.filesize(path) == size: # dprint("leave: already downloaded") # return True ok = False from sakurakit import skfileio, sknetio with SkProfiler("fetch"): if sknetio.getfile(url, path, flush=False): # flush=false to use more memory to reduce disk access ok = skfileio.filesize(path) == DICS[dic]['size'] if not ok and os.path.exists(path): skfileio.removefile(path) dprint("leave: ok = %s" % ok) return ok
def updateAvatar(self, token): """ @param token unicode @return bool whether succeeded """ path = rc.avatar_image_path(token) if os.path.exists(path): return True if not self.__d.enabled: return False path_tmp = path + TMP_SUFFIX if os.path.exists(path_tmp): return False skfileio.touchfile(path_tmp) ok = skthreads.runsync(partial(_saveavatar, token, path_tmp)) if ok and skfileio.rename(path_tmp, path): return True else: skfileio.removefile(path_tmp) return False
def get(): # return bool url = UNIDIC_URL path = TMP_DIR + '/' + UNIDIC_FILENAME + UNIDIC_SUFFIX size = UNIDIC_FILESIZE dprint("enter: size = %s, url = %s" % (size, url)) from sakurakit import skfileio if os.path.exists(path) and skfileio.filesize(path) == size: dprint("leave: already downloaded") return True from sakurakit import sknetio ok = False with SkProfiler("fetch"): if sknetio.getfile(url, path, flush=False): # flush=false to use more memory to reduce disk access ok = skfileio.filesize(path) == size if not ok and os.path.exists(path): skfileio.removefile(path) dprint("leave: ok = %s" % ok) return ok
def get(family): # str -> bool font = FONTS[family] url = font['dl'] path = TMP_DIR + '/font-%s.%s' % (family, font['format']) dprint("enter: family = %s, url = %s" % (family, url)) #from sakurakit import skfileio #if os.path.exists(path) and skfileio.filesize(path) == size: # dprint("leave: already downloaded") # return True ok = False from sakurakit import skfileio, sknetio with SkProfiler("fetch"): if sknetio.getfile(url, path, flush=False): # flush=false to use more memory to reduce disk access ok = skfileio.filesize(path) == font['size'] if not ok and os.path.exists(path): skfileio.removefile(path) dprint("leave: ok = %s" % ok) return ok
def extract(): dprint("enter") srcpath = TMP_DIR + '/' + WADOKU_FILENAME + WADOKU_SUFFIX tmppath = TMP_DIR + '/' + WADOKU_FILENAME targetpath = TARGET_DIR import shutil from sakurakit import skfileio with SkProfiler("extract"): ok = skfileio.extractzip(srcpath, tmppath) if ok: if os.path.exists(targetpath): shutil.rmtree(targetpath) os.renames(tmppath, targetpath) if os.path.exists(tmppath): shutil.rmtree(tmppath) skfileio.removefile(srcpath) dprint("leave: ok = %s" % ok) return ok
def makedb(): # -> bool dprint("enter") tmpdic = TMP_DIR + '/' + DIC_FILENAME tmpdb = TMP_DIR + '/' + DB_FILENAME targetdic = TARGET_DIR + '/' + DIC_FILENAME targetdb = TARGET_DIR + '/' + DB_FILENAME from dictdb import edictdb with SkProfiler("create db"): ok = edictdb.makedb(tmpdb, tmpdic) if ok: with SkProfiler("create index"): ok = edictdb.makesurface(tmpdb) from sakurakit import skfileio if ok: skfileio.removefile(targetdb) skfileio.removefile(targetdic) os.rename(tmpdb, targetdb) os.rename(tmpdic, targetdic) else: for it in tmpdb, tmpdic: if os.path.exists(it): skfileio.removefile(it) dprint("leave: ok = %s" % ok) return ok
def makedb(lang): # str -> bool dprint("enter: lang = %s" % lang) dic = DICS[lang] ldpath = LD_DIR + '/' + lang + LD_SUFFIX dbpath = DB_DIR + '/' + lang + DB_SUFFIX tmppath = TMP_DIR + '/' + lang + DB_SUFFIX from sakurakit import skfileio for it in dbpath, tmppath: if os.path.exists(it): skfileio.removefile(it) if not os.path.exists(ldpath): dwarn("leave: ld does not exist: %s" % ldpath) return False ok = False from lingoes.lingoesdb import LingoesDb with SkProfiler("extract"): if LingoesDb(tmppath).create(ldpath, dic['inenc'], dic['outenc']): os.renames(tmppath, dbpath) # renames to create DB_DIR ok = True elif os.path.exists(tmppath): skfileio.removefile(tmppath) if os.path.exists(ldpath): skfileio.removefile(ldpath) dprint("leave: ok = %s" % ok) return ok
def get(): # -> bool url = DIC_URL minsize = MIN_DIC_SIZE path = TMP_DIR + '/' + DIC_FILENAME path_compressed = path + '.gz' dprint("enter: url = %s, minsize = %s" % (url, minsize)) #from sakurakit import skfileio #if os.path.exists(path) and skfileio.filesize(path) == size: # dprint("leave: already downloaded") # return True ok = False import gzip from sakurakit import skfileio, sknetio with SkProfiler("fetch"): # gzip=True to automatically extract gzip # flush=false to use more memory to reduce disk access if sknetio.getfile(url, path_compressed, flush=False, gzip=False): # Note: gzip=True does not extract gzip, it decompresses the header ... probs? >_< with gzip.open(path_compressed, 'rb') as f_in, open(path, 'wb') as f_out: f_content = f_in.read() f_out.write(f_content) ok = skfileio.filesize(path) > minsize if ok: skfileio.removefile(path_compressed) elif os.path.exists(path): skfileio.removefile(path) skfileio.removefile(path_compressed) # if not ok and os.path.exists(path): # skfileio.removefile(path) dprint("leave: ok = %s" % ok) return ok
def extract(lang): # str -> bool dprint("enter: lang = %s" % lang) srcpath = TMP_DIR + '/' + DICS[lang]['file'] tmppath = TMP_DIR + '/ovdp-' + lang targetpath = DIC_DIR + '/' + DICS[lang]['path'] import shutil from sakurakit import skfileio with SkProfiler("extract"): ok = skfileio.extractzip(srcpath, tmppath) if ok: if os.path.exists(targetpath): shutil.rmtree(targetpath) child = skfileio.getfirstchilddir(tmppath) os.renames(child, targetpath) if os.path.exists(tmppath): shutil.rmtree(tmppath) skfileio.removefile(srcpath) dprint("leave: ok = %s" % ok) return ok
def extract(dic): # str -> bool dprint("enter: dic = %s" % dic) srcpath = TMP_DIR + '/' + DICS[dic]['file'] tmppath = TMP_DIR + '/stardic-' + dic targetpath = DIC_DIR + '/' + DICS[dic]['path'] import shutil from sakurakit import skfileio with SkProfiler("extract"): ok = skfileio.extracttarbz2(srcpath, tmppath) if ok: if os.path.exists(targetpath): shutil.rmtree(targetpath) child = skfileio.getfirstchilddir(tmppath) os.renames(child, targetpath) if os.path.exists(tmppath): shutil.rmtree(tmppath) skfileio.removefile(srcpath) dprint("leave: ok = %s" % ok) return ok
def extract(lang): # str -> bool dprint("enter: lang = %s" % lang) srcpath = TMP_DIR + '/' + FILENAME_TPL % lang tmppath = TMP_DIR + '/JMDict-' + lang targetpath = DIC_DIR + '/%s.fpw' % lang import shutil from sakurakit import skfileio with SkProfiler("extract"): ok = skfileio.extracttar(srcpath, tmppath) if ok: if os.path.exists(targetpath): shutil.rmtree(targetpath) child = skfileio.getfirstchilddir(tmppath) os.renames(child, targetpath) if os.path.exists(tmppath): shutil.rmtree(tmppath) skfileio.removefile(srcpath) dprint("leave: ok = %s" % ok) return ok
def extract(): dprint("enter") srcpath = TMP_DIR + '/' + UNIDIC_FILENAME + UNIDIC_SUFFIX tmppath = TMP_DIR + '/' + UNIDIC_FILENAME targetpath = TARGET_DIR import shutil from sakurakit import skfileio with SkProfiler("extract"): ok = skfileio.extracttarbz2(srcpath, tmppath) if ok: if os.path.exists(targetpath): shutil.rmtree(targetpath) child = skfileio.getfirstchilddir(tmppath) os.renames(child, targetpath) if os.path.exists(tmppath): shutil.rmtree(tmppath) skfileio.removefile(srcpath) dprint("leave: ok = %s" % ok) return ok
def get(dic): # str -> bool url = URL_TPL % dic minsize = MIN_FILESIZE path = TMP_DIR + '/' + FILENAME_TPL % dic dprint("enter: url = %s, minsize = %s" % (url, minsize)) #from sakurakit import skfileio #if os.path.exists(path) and skfileio.filesize(path) == size: # dprint("leave: already downloaded") # return True ok = False from sakurakit import skfileio, sknetio with SkProfiler("fetch"): if sknetio.getfile( url, path, flush=False ): # flush=false to use more memory to reduce disk access ok = skfileio.filesize(path) > minsize if not ok and os.path.exists(path): skfileio.removefile(path) dprint("leave: ok = %s" % ok) return ok
def get(lang): # str -> bool dprint("enter: lang = %s" % lang) url = URL + DICS[lang]['file'] path = TMP_DIR + '/' + FILENAME_TPL % lang dprint("enter: url = %s" % url) #from sakurakit import skfileio #if os.path.exists(path) and skfileio.filesize(path) == size: # dprint("leave: already downloaded") # return True ok = False from sakurakit import skfileio, sknetio with SkProfiler("fetch"): if sknetio.getfile( url, path, flush=False ): # flush=false to use more memory to reduce disk access ok = skfileio.filesize(path) > MIN_FILESIZE if not ok and os.path.exists(path): skfileio.removefile(path) dprint("leave: ok = %s" % ok) return ok
def extract(family): # str -> bool dprint("enter: family = %s" % family) font = FONTS[family] fmt = font['format'] tmppath = TMP_DIR + '/font-%s' % family srcpath = tmppath + '.' + fmt targetpath = FONT_DIR + '/' + font['path'] import shutil from sakurakit import skfileio with SkProfiler("extract"): ok = skfileio.extractarchive(srcpath, tmppath, type=fmt) if ok: if os.path.exists(targetpath): shutil.rmtree(targetpath) os.renames(tmppath, targetpath) if os.path.exists(tmppath): shutil.rmtree(tmppath) skfileio.removefile(srcpath) dprint("leave: ok = %s" % ok) return ok
def get(): # -> bool url = DIC_URL minsize = MIN_DIC_SIZE path = TMP_DIR + '/' + DIC_FILENAME dprint("enter: url = %s, minsize = %s" % (url, minsize)) #from sakurakit import skfileio #if os.path.exists(path) and skfileio.filesize(path) == size: # dprint("leave: already downloaded") # return True ok = False from sakurakit import skfileio, sknetio with SkProfiler("fetch"): # gzip=True to automatically extract gzip # flush=false to use more memory to reduce disk access if sknetio.getfile(url, path, flush=False, gzip=True): ok = skfileio.filesize(path) > minsize if not ok and os.path.exists(path): skfileio.removefile(path) dprint("leave: ok = %s" % ok) return ok
def extract(): dprint("enter") srcpath = TMP_DIR + '/' + IPADIC_FILENAME + IPADIC_SUFFIX tmppath = TMP_DIR + '/' + IPADIC_FILENAME targetpath = TARGET_DIR import shutil from sakurakit import skfileio with SkProfiler("extract"): ok = skfileio.extracttarbz2(srcpath, tmppath) if ok: if os.path.exists(targetpath): shutil.rmtree(targetpath) #child = skfileio.getfirstchilddir(tmppath) #child = os.path.join(tmppath, '/opt/local/lib/mecab/dic/ipadic-utf8') child = tmppath + IPADIC_RELPATH os.renames(child, targetpath) if os.path.exists(tmppath): shutil.rmtree(tmppath) skfileio.removefile(srcpath) dprint("leave: ok = %s" % ok) return ok
def getld(lang): # str -> bool url = DICS[lang].get( 'url') or "http://%s/pub/lingoes/%s.ld2" % (initdefs.DOMAIN_ORG, lang) size = DICS[lang]['size'] path = LD_DIR + '/' + lang + LD_SUFFIX dprint("enter: lang = %s, size = %s" % (lang, size)) from sakurakit import skfileio if os.path.exists(path) and skfileio.filesize(path) == size: dprint("leave: already downloaded") return True ok = False from sakurakit import sknetio with SkProfiler("fetch"): if sknetio.getfile( url, path, flush=False ): # flush=false to use more memory to reduce disk access ok = skfileio.filesize(path) == size if not ok and os.path.exists(path): skfileio.removefile(path) dprint("leave: ok = %s" % ok) return ok