def get(lang): # str -> bool
filename = DICS[lang]
url = DIC_URL % filename
minsize = MIN_DIC_SIZE
path = TMP_DIR + '/' + filename
targetpath = TARGET_DIR + '/' + filename
dprint("enter: url = %s, minsize = %s" % (url, minsize))
#from sakurakit import skfileio
#if os.path.exists(path) and skfileio.filesize(path) == size:
# dprint("leave: already downloaded")
# return True
ok = False
from sakurakit import skfileio, sknetio
with SkProfiler("fetch"):
# gzip=True to automatically extract gzip
# flush=false to use more memory to reduce disk access
if sknetio.getfile(url, path, flush=False, gzip=True):
ok = skfileio.filesize(path) > minsize
if ok:
os.renames(path, targetpath)
elif os.path.exists(path):
skfileio.removefile(path)
dprint("leave: ok = %s" % ok)
return ok
def dump(path, gaiji={}, inenc='euc_jp', outenc='utf8', output='eb.db'):
"""
@param path unicode
@param* gaiji {str:str}
@param* inenc str
@param* outenc str
"""
dprint("enter")
from sakurakit import skfileio
skfileio.removefile(output)
headings = set() # [unicode]
dictdb.createdb(output)
with sqlite3.connect(output) as conn:
cur = conn.cursor()
shiori = EBShiori(path, gaiji=gaiji)
kana = jpchars.s_hira + jpchars.s_kata
for i, ch in enumerate(kana):
s = ch.encode(inenc)
hits = shiori.search_word(s)
dprint(ch, i, len(hits))
for hit in hits:
h = hit.heading().decode(outenc)
if h not in headings:
headings.add(h)
t = hit.text().decode(outenc)
dictdb.insertentry(cur, (h, t))
conn.commit()
dprint("leave")
def _getdata(url,
path,
tmppath=None,
touchbad=False,
mimefilter=None,
**kwargs):
"""
@param token unicode
@param path unicode
@param* tmppath unicode
@param* touchbad bool
@param* mimefilter str
@return bool
"""
if not tmppath:
tmppath = path + TMP_SUFFIX
data = sknetio.getdata(url, mimefilter=mimefilter, **kwargs)
#if not data:
# url = _fixtwimg(url)
# if url:
# return _getdata(url, path, tmppath=tmppath, mimefilter=mimefilter, **kwargs)
ok = bool(data) and skfileio.writedata(tmppath, data)
if ok and skfileio.rename(tmppath, path):
return True
else:
skfileio.removefile(tmppath)
if touchbad:
badpath = path + BAD_SUFFIX
skfileio.touchfile(badpath)
return False
def compile(): # -> bool
dprint("enter")
dbpath = TARGET_DIR + '/' + DB_FILENAME
dicpath = TARGET_DIR + '/' + DIC_FILENAME
csvpath = TARGET_DIR + '/' + CSV_FILENAME
tmpdicpath = dicpath + '.tmp'
from mecabdic import mdedict
with SkProfiler("assemble csv"):
ok = mdedict.db2csv(csvpath, dbpath)
if ok:
with SkProfiler("compile dic"):
ok = mdedict.csv2dic(tmpdicpath,
csvpath,
exe=MECAB_EXEPATH,
dicdir=UNIDIC_DICPATH)
from sakurakit import skfileio
if ok:
if os.path.exists(dicpath):
skfileio.remove(dicpath)
os.rename(tmpdicpath, dicpath)
elif os.path.exists(tmpdicpath):
skfileio.removefile(tmpdicpath)
if os.path.exists(csvpath):
skfileio.removefile(csvpath)
dprint("leave: ok = %s" % ok)
return ok
def get(): # -> bool
targetpath = INST_DIR + '/' + APPLOC_FILENAME
tmppath = TMP_DIR + '/' + APPLOC_FILENAME
url = APPLOC_URL
size = APPLOC_FILESIZE
dprint("enter: size = %s, url = %s" % (size, url))
from sakurakit import skfileio
if os.path.exists(targetpath) and skfileio.filesize(targetpath) == size:
dprint("leave: already downloaded")
return True
from sakurakit import sknetio
ok = False
with SkProfiler("fetch"):
if sknetio.getfile(url, tmppath, flush=False): # flush=false to use more memory to reduce disk access
ok = skfileio.filesize(tmppath) == size
if ok:
os.rename(tmppath, targetpath)
if not ok and os.path.exists(tmppath):
skfileio.removefile(tmppath)
dprint("leave: ok = %s" % ok)
return ok
def _continue(self):
if self._cvid and self._cpath and not _is_good_video(self._cpath):
if os.path.exists(self._cpath):
growl.warn("[main] %s: %s" % (my.tr("remove failed file"), self._cpath))
skfileio.removefile(self._cpath)
self._retry()
else:
self._next()
def clearTemporaryFiles():
from sakurakit import skfileio
for tmpdir in rc.DIR_CACHE_DATA, :
try:
for root, dirs, files in os.walk(tmpdir):
for f in files:
if f.endswith(TMP_SUFFIX):
p = os.path.join(root, f)
skfileio.removefile(p)
except Exception, e:
dwarn(e)
def saveClosedUrls(self):
path = rc.CLOSE_HISTORY_LOCATION
if self.closedUrls:
l = self.closedUrls
if len(l) > config.CLOSE_HISTORY_SIZE:
l = l[len(l) - config.CLOSE_HISTORY_SIZE:]
data = '\n'.join(l)
skfileio.writefile(path, data)
else:
skfileio.removefile(path)
dprint("pass")
def _savePixmap(self, pm):
"""
@param pm QPixmap or QImage
@param path unicode
@return bool
"""
path = self._randomPath()
ret = save_pixmap(pm, path)
if ret:
skfileio.removefile(self.path)
self.setPath(path)
return ret
def extract(dic): # str -> bool
dprint("enter: dic = %s" % dic)
srcpath = TMP_DIR + '/' + FILENAME_TPL % dic
tmppath = TMP_DIR + '/cabocha.' + dic
targetpath = DIC_DIR + '/' + dic
import shutil
from sakurakit import skfileio
with SkProfiler("extract"):
ok = skfileio.extracttarbz2(srcpath, tmppath)
if ok:
dickey = 'ipa' if dic == 'ipadic' else dic
for it in MODELS:
model = "%s.%s.model" % (it, dickey)
if not os.path.exists(tmppath + '/' + model):
ok = False
dwarn("missing model: %s" % model)
if os.path.exists(targetpath): # alway remove old path even if not ok
shutil.rmtree(targetpath)
if ok:
os.renames(tmppath, targetpath)
#child = skfileio.getfirstchilddir(tmppath)
#modeldir = child + '/model'
#if not os.path.exists(modeldir):
# dwarn("model dir does not exist")
#else:
# from cabochajlp import cabochamodel
# if os.path.exists(targetpath):
# shutil.rmtree(targetpath)
# os.makedirs(targetpath)
# dickey = 'ipa' if dic == 'ipadic' else dic
# for model in MODELS:
# with SkProfiler():
# dprint("process model: %s" % model)
# input = "%s.%s.txt" % (model , dickey)
# output = "%s.%s.model" % (model , dickey)
# inputpath = os.path.abspath(modeldir + '/' + input)
# outputpath = os.path.abspath(modeldir + '/' + output)
# if cabochamodel.modelindex(outputpath, inputpath):
# os.renames(outputpath, targetpath + '/' + output)
# else:
# dwarn("failed to process model %s" % model)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
skfileio.removefile(srcpath)
dprint("leave: ok = %s" % ok)
return ok
def saveTabs(self): # -> bool
ret = False
urls = [] # [unicode url]
for w in self._iterTabWidgets():
url = w.url().toString()
if url != BLANK_URL:
urls.append(url)
path = rc.TABS_LOCATION
if urls:
data = '\n'.join(urls)
ret = skfileio.writefile(path, data)
else:
skfileio.removefile(path)
dprint("pass: ret = %s" % ret)
return ret
def ld2db(ldpath, dbpath=None, inenc=None, outenc=None):
"""
@param ldpath unicode
@param* dbpath unicode
@param* inenc str
@param* outenc str
"""
if not inenc:
inenc = 'utf8'
if not outenc:
outenc = inenc
dprint("enter: inenc = %s, outenc = %s" % (inenc, outenc))
if not dbpath or os.path.isdir(dbpath):
basename = os.path.basename(ldpath)
if ldpath.endswith(LD_SUFFIX):
path = os.path.splitext(basename)[0] + DB_SUFFIX
else:
path = basename
if dbpath:
dbpath = os.path.join(dbpath, path)
else:
dbpath = path
if not os.path.exists(ldpath):
dwarn("ld file does not exist: %s" % ldpath)
return False
if os.path.exists(dbpath):
dwarn("db file already exist: %s" % dbpath)
return False
tmppath = dbpath + TMP_SUFFIX
from sakurakit import skfileio
if os.path.exists(tmppath):
skfileio.removefile(tmppath)
ok = False
from lingoes.lingoesdb import LingoesDb
with SkProfiler():
if LingoesDb(tmppath).create(ldpath, inenc, outenc):
os.renames(tmppath, dbpath) # renames to create DB_DIR
ok = True
elif os.path.exists(tmppath):
skfileio.removefile(tmppath)
dprint("leave: ok = %s" % ok)
return ok
def get(dic): # str -> bool
url = DIC_URL + DICS[dic]['file']
path = TMP_DIR + '/' + DICS[dic]['file']
dprint("enter: dic = %s, url = %s" % (dic, url))
#from sakurakit import skfileio
#if os.path.exists(path) and skfileio.filesize(path) == size:
# dprint("leave: already downloaded")
# return True
ok = False
from sakurakit import skfileio, sknetio
with SkProfiler("fetch"):
if sknetio.getfile(url, path, flush=False): # flush=false to use more memory to reduce disk access
ok = skfileio.filesize(path) == DICS[dic]['size']
if not ok and os.path.exists(path):
skfileio.removefile(path)
dprint("leave: ok = %s" % ok)
return ok
def updateAvatar(self, token):
"""
@param token unicode
@return bool whether succeeded
"""
path = rc.avatar_image_path(token)
if os.path.exists(path):
return True
if not self.__d.enabled:
return False
path_tmp = path + TMP_SUFFIX
if os.path.exists(path_tmp):
return False
skfileio.touchfile(path_tmp)
ok = skthreads.runsync(partial(_saveavatar, token, path_tmp))
if ok and skfileio.rename(path_tmp, path):
return True
else:
skfileio.removefile(path_tmp)
return False
def get(): # return bool
url = UNIDIC_URL
path = TMP_DIR + '/' + UNIDIC_FILENAME + UNIDIC_SUFFIX
size = UNIDIC_FILESIZE
dprint("enter: size = %s, url = %s" % (size, url))
from sakurakit import skfileio
if os.path.exists(path) and skfileio.filesize(path) == size:
dprint("leave: already downloaded")
return True
from sakurakit import sknetio
ok = False
with SkProfiler("fetch"):
if sknetio.getfile(url, path, flush=False): # flush=false to use more memory to reduce disk access
ok = skfileio.filesize(path) == size
if not ok and os.path.exists(path):
skfileio.removefile(path)
dprint("leave: ok = %s" % ok)
return ok
def get(family): # str -> bool
font = FONTS[family]
url = font['dl']
path = TMP_DIR + '/font-%s.%s' % (family, font['format'])
dprint("enter: family = %s, url = %s" % (family, url))
#from sakurakit import skfileio
#if os.path.exists(path) and skfileio.filesize(path) == size:
# dprint("leave: already downloaded")
# return True
ok = False
from sakurakit import skfileio, sknetio
with SkProfiler("fetch"):
if sknetio.getfile(url, path, flush=False): # flush=false to use more memory to reduce disk access
ok = skfileio.filesize(path) == font['size']
if not ok and os.path.exists(path):
skfileio.removefile(path)
dprint("leave: ok = %s" % ok)
return ok
def extract():
dprint("enter")
srcpath = TMP_DIR + '/' + WADOKU_FILENAME + WADOKU_SUFFIX
tmppath = TMP_DIR + '/' + WADOKU_FILENAME
targetpath = TARGET_DIR
import shutil
from sakurakit import skfileio
with SkProfiler("extract"):
ok = skfileio.extractzip(srcpath, tmppath)
if ok:
if os.path.exists(targetpath):
shutil.rmtree(targetpath)
os.renames(tmppath, targetpath)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
skfileio.removefile(srcpath)
dprint("leave: ok = %s" % ok)
return ok
def makedb(): # -> bool
dprint("enter")
tmpdic = TMP_DIR + '/' + DIC_FILENAME
tmpdb = TMP_DIR + '/' + DB_FILENAME
targetdic = TARGET_DIR + '/' + DIC_FILENAME
targetdb = TARGET_DIR + '/' + DB_FILENAME
from dictdb import edictdb
with SkProfiler("create db"):
ok = edictdb.makedb(tmpdb, tmpdic)
if ok:
with SkProfiler("create index"):
ok = edictdb.makesurface(tmpdb)
from sakurakit import skfileio
if ok:
skfileio.removefile(targetdb)
skfileio.removefile(targetdic)
os.rename(tmpdb, targetdb)
os.rename(tmpdic, targetdic)
else:
for it in tmpdb, tmpdic:
if os.path.exists(it):
skfileio.removefile(it)
dprint("leave: ok = %s" % ok)
return ok
def makedb(lang): # str -> bool
dprint("enter: lang = %s" % lang)
dic = DICS[lang]
ldpath = LD_DIR + '/' + lang + LD_SUFFIX
dbpath = DB_DIR + '/' + lang + DB_SUFFIX
tmppath = TMP_DIR + '/' + lang + DB_SUFFIX
from sakurakit import skfileio
for it in dbpath, tmppath:
if os.path.exists(it):
skfileio.removefile(it)
if not os.path.exists(ldpath):
dwarn("leave: ld does not exist: %s" % ldpath)
return False
ok = False
from lingoes.lingoesdb import LingoesDb
with SkProfiler("extract"):
if LingoesDb(tmppath).create(ldpath, dic['inenc'], dic['outenc']):
os.renames(tmppath, dbpath) # renames to create DB_DIR
ok = True
elif os.path.exists(tmppath):
skfileio.removefile(tmppath)
if os.path.exists(ldpath):
skfileio.removefile(ldpath)
dprint("leave: ok = %s" % ok)
return ok
def get(): # -> bool
url = DIC_URL
minsize = MIN_DIC_SIZE
path = TMP_DIR + '/' + DIC_FILENAME
path_compressed = path + '.gz'
dprint("enter: url = %s, minsize = %s" % (url, minsize))
#from sakurakit import skfileio
#if os.path.exists(path) and skfileio.filesize(path) == size:
# dprint("leave: already downloaded")
# return True
ok = False
import gzip
from sakurakit import skfileio, sknetio
with SkProfiler("fetch"):
# gzip=True to automatically extract gzip
# flush=false to use more memory to reduce disk access
if sknetio.getfile(url, path_compressed, flush=False, gzip=False):
# Note: gzip=True does not extract gzip, it decompresses the header ... probs? >_<
with gzip.open(path_compressed, 'rb') as f_in, open(path, 'wb') as f_out:
f_content = f_in.read()
f_out.write(f_content)
ok = skfileio.filesize(path) > minsize
if ok:
skfileio.removefile(path_compressed)
elif os.path.exists(path):
skfileio.removefile(path)
skfileio.removefile(path_compressed)
# if not ok and os.path.exists(path):
# skfileio.removefile(path)
dprint("leave: ok = %s" % ok)
return ok
def extract(lang): # str -> bool
dprint("enter: lang = %s" % lang)
srcpath = TMP_DIR + '/' + DICS[lang]['file']
tmppath = TMP_DIR + '/ovdp-' + lang
targetpath = DIC_DIR + '/' + DICS[lang]['path']
import shutil
from sakurakit import skfileio
with SkProfiler("extract"):
ok = skfileio.extractzip(srcpath, tmppath)
if ok:
if os.path.exists(targetpath):
shutil.rmtree(targetpath)
child = skfileio.getfirstchilddir(tmppath)
os.renames(child, targetpath)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
skfileio.removefile(srcpath)
dprint("leave: ok = %s" % ok)
return ok
def extract(dic): # str -> bool
dprint("enter: dic = %s" % dic)
srcpath = TMP_DIR + '/' + DICS[dic]['file']
tmppath = TMP_DIR + '/stardic-' + dic
targetpath = DIC_DIR + '/' + DICS[dic]['path']
import shutil
from sakurakit import skfileio
with SkProfiler("extract"):
ok = skfileio.extracttarbz2(srcpath, tmppath)
if ok:
if os.path.exists(targetpath):
shutil.rmtree(targetpath)
child = skfileio.getfirstchilddir(tmppath)
os.renames(child, targetpath)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
skfileio.removefile(srcpath)
dprint("leave: ok = %s" % ok)
return ok
def extract(lang): # str -> bool
dprint("enter: lang = %s" % lang)
srcpath = TMP_DIR + '/' + FILENAME_TPL % lang
tmppath = TMP_DIR + '/JMDict-' + lang
targetpath = DIC_DIR + '/%s.fpw' % lang
import shutil
from sakurakit import skfileio
with SkProfiler("extract"):
ok = skfileio.extracttar(srcpath, tmppath)
if ok:
if os.path.exists(targetpath):
shutil.rmtree(targetpath)
child = skfileio.getfirstchilddir(tmppath)
os.renames(child, targetpath)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
skfileio.removefile(srcpath)
dprint("leave: ok = %s" % ok)
return ok
def extract():
dprint("enter")
srcpath = TMP_DIR + '/' + UNIDIC_FILENAME + UNIDIC_SUFFIX
tmppath = TMP_DIR + '/' + UNIDIC_FILENAME
targetpath = TARGET_DIR
import shutil
from sakurakit import skfileio
with SkProfiler("extract"):
ok = skfileio.extracttarbz2(srcpath, tmppath)
if ok:
if os.path.exists(targetpath):
shutil.rmtree(targetpath)
child = skfileio.getfirstchilddir(tmppath)
os.renames(child, targetpath)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
skfileio.removefile(srcpath)
dprint("leave: ok = %s" % ok)
return ok
def get(dic): # str -> bool
url = URL_TPL % dic
minsize = MIN_FILESIZE
path = TMP_DIR + '/' + FILENAME_TPL % dic
dprint("enter: url = %s, minsize = %s" % (url, minsize))
#from sakurakit import skfileio
#if os.path.exists(path) and skfileio.filesize(path) == size:
# dprint("leave: already downloaded")
# return True
ok = False
from sakurakit import skfileio, sknetio
with SkProfiler("fetch"):
if sknetio.getfile(
url, path, flush=False
): # flush=false to use more memory to reduce disk access
ok = skfileio.filesize(path) > minsize
if not ok and os.path.exists(path):
skfileio.removefile(path)
dprint("leave: ok = %s" % ok)
return ok
def get(lang): # str -> bool
dprint("enter: lang = %s" % lang)
url = URL + DICS[lang]['file']
path = TMP_DIR + '/' + FILENAME_TPL % lang
dprint("enter: url = %s" % url)
#from sakurakit import skfileio
#if os.path.exists(path) and skfileio.filesize(path) == size:
# dprint("leave: already downloaded")
# return True
ok = False
from sakurakit import skfileio, sknetio
with SkProfiler("fetch"):
if sknetio.getfile(
url, path, flush=False
): # flush=false to use more memory to reduce disk access
ok = skfileio.filesize(path) > MIN_FILESIZE
if not ok and os.path.exists(path):
skfileio.removefile(path)
dprint("leave: ok = %s" % ok)
return ok
def extract(family): # str -> bool
dprint("enter: family = %s" % family)
font = FONTS[family]
fmt = font['format']
tmppath = TMP_DIR + '/font-%s' % family
srcpath = tmppath + '.' + fmt
targetpath = FONT_DIR + '/' + font['path']
import shutil
from sakurakit import skfileio
with SkProfiler("extract"):
ok = skfileio.extractarchive(srcpath, tmppath, type=fmt)
if ok:
if os.path.exists(targetpath):
shutil.rmtree(targetpath)
os.renames(tmppath, targetpath)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
skfileio.removefile(srcpath)
dprint("leave: ok = %s" % ok)
return ok
def get(): # -> bool
url = DIC_URL
minsize = MIN_DIC_SIZE
path = TMP_DIR + '/' + DIC_FILENAME
dprint("enter: url = %s, minsize = %s" % (url, minsize))
#from sakurakit import skfileio
#if os.path.exists(path) and skfileio.filesize(path) == size:
# dprint("leave: already downloaded")
# return True
ok = False
from sakurakit import skfileio, sknetio
with SkProfiler("fetch"):
# gzip=True to automatically extract gzip
# flush=false to use more memory to reduce disk access
if sknetio.getfile(url, path, flush=False, gzip=True):
ok = skfileio.filesize(path) > minsize
if not ok and os.path.exists(path):
skfileio.removefile(path)
dprint("leave: ok = %s" % ok)
return ok
def extract():
dprint("enter")
srcpath = TMP_DIR + '/' + IPADIC_FILENAME + IPADIC_SUFFIX
tmppath = TMP_DIR + '/' + IPADIC_FILENAME
targetpath = TARGET_DIR
import shutil
from sakurakit import skfileio
with SkProfiler("extract"):
ok = skfileio.extracttarbz2(srcpath, tmppath)
if ok:
if os.path.exists(targetpath):
shutil.rmtree(targetpath)
#child = skfileio.getfirstchilddir(tmppath)
#child = os.path.join(tmppath, '/opt/local/lib/mecab/dic/ipadic-utf8')
child = tmppath + IPADIC_RELPATH
os.renames(child, targetpath)
if os.path.exists(tmppath):
shutil.rmtree(tmppath)
skfileio.removefile(srcpath)
dprint("leave: ok = %s" % ok)
return ok
def getld(lang): # str -> bool
url = DICS[lang].get(
'url') or "http://%s/pub/lingoes/%s.ld2" % (initdefs.DOMAIN_ORG, lang)
size = DICS[lang]['size']
path = LD_DIR + '/' + lang + LD_SUFFIX
dprint("enter: lang = %s, size = %s" % (lang, size))
from sakurakit import skfileio
if os.path.exists(path) and skfileio.filesize(path) == size:
dprint("leave: already downloaded")
return True
ok = False
from sakurakit import sknetio
with SkProfiler("fetch"):
if sknetio.getfile(
url, path, flush=False
): # flush=false to use more memory to reduce disk access
ok = skfileio.filesize(path) == size
if not ok and os.path.exists(path):
skfileio.removefile(path)
dprint("leave: ok = %s" % ok)
return ok
