Beispiel #1
0
    def test():
        global session

        #s = u"オープニングやエンディングのアニメーションは単純に主人公を入れ替えた程度の物ではなく、タイトルロゴはもちろん金時や定春の行動や表情、登場する道具(万事屋の面々が乗る車のデザインなど)やクレジット文字など、細部に渡って変更がなされた。更に、坂田金時が『銀魂'』を最終回に追い込み新しいアニメ『まんたま』を始めようとした時にはエンディングや提供表示の煽りコメントが最終回を思わせる演出となり、『まんたま』でも専用のタイトルロゴとオープニングアニメーション(スタッフクレジット付き)が新造され、偽物の提供クレジットまで表示されるなど随所に至るまで徹底的な演出が行われた。また、テレビ欄では金魂篇終了回は『金魂'』最終回として、その翌週は新番組「銀魂'」として案内された。"
        s = "test"
        fr = "ja"
        to = "zhs"

        #s = u"What are you doing?"
        #fr = "en"

        from sakurakit.skprof import SkProfiler

        from qtrequests import qtrequests
        from PySide.QtNetwork import QNetworkAccessManager
        session = qtrequests.Session(QNetworkAccessManager())
        with SkProfiler():
            for i in range(1):
                t = translate(s, to=to, fr=fr)
        print t

        session = requests.Session()
        with SkProfiler():
            for i in range(1):
                t = translate(s, to=to, fr=fr)
        print t
        print type(t)

        #session = requests
        #with SkProfiler():
        #  for i in range(10):
        #    t = translate(s, to=to, fr=fr)
        #print t

        app.quit()
Beispiel #2
0
    def test_create():
        path = 'edict2u'
        try:
            for i, (k, v) in enumerate(iterparse(path)):
                print '|%s|' % i
                print '|%s|' % k
                print '|%s|' % v
                break
        except:
            pass

        import os
        if os.path.exists(dbpath):
            os.remove(dbpath)

        from sakurakit.skprof import SkProfiler

        from dictdb import dictdb

        with SkProfiler("create db"):
            print dictdb.createdb(dbpath)

        with SkProfiler("insert db"):
            try:
                with sqlite3.connect(dbpath) as conn:
                    q = parsefile(path)
                    dictdb.insertentries(conn.cursor(), q, ignore_errors=True)
                    conn.commit()
            except Exception, e:
                dwarn(e)
Beispiel #3
0
def makedb(): # -> bool
  dprint("enter")
  tmpdic = TMP_DIR + '/' + DIC_FILENAME
  tmpdb = TMP_DIR + '/' + DB_FILENAME

  targetdic = TARGET_DIR + '/' + DIC_FILENAME
  targetdb = TARGET_DIR + '/' + DB_FILENAME

  from dictdb import edictdb
  with SkProfiler("create db"):
    ok = edictdb.makedb(tmpdb, tmpdic)
  if ok:
    with SkProfiler("create index"):
      ok = edictdb.makesurface(tmpdb)

  from sakurakit import skfileio
  if ok:
    skfileio.removefile(targetdb)
    skfileio.removefile(targetdic)
    os.rename(tmpdb, targetdb)
    os.rename(tmpdic, targetdic)
  else:
    for it in tmpdb, tmpdic:
      if os.path.exists(it):
        skfileio.removefile(it)
  dprint("leave: ok = %s" % ok)
  return ok
Beispiel #4
0
def compile():  # -> bool
    dprint("enter")
    dbpath = TARGET_DIR + '/' + DB_FILENAME
    dicpath = TARGET_DIR + '/' + DIC_FILENAME
    csvpath = TARGET_DIR + '/' + CSV_FILENAME
    tmpdicpath = dicpath + '.tmp'

    from mecabdic import mdedict
    with SkProfiler("assemble csv"):
        ok = mdedict.db2csv(csvpath, dbpath)
    if ok:
        with SkProfiler("compile dic"):
            ok = mdedict.csv2dic(tmpdicpath,
                                 csvpath,
                                 exe=MECAB_EXEPATH,
                                 dicdir=UNIDIC_DICPATH)
    from sakurakit import skfileio
    if ok:
        if os.path.exists(dicpath):
            skfileio.remove(dicpath)
        os.rename(tmpdicpath, dicpath)
    elif os.path.exists(tmpdicpath):
        skfileio.removefile(tmpdicpath)
    if os.path.exists(csvpath):
        skfileio.removefile(csvpath)
    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #5
0
def get(): # -> bool

  targetpath = INST_DIR + '/' + APPLOC_FILENAME
  tmppath = TMP_DIR + '/' + APPLOC_FILENAME
  url = APPLOC_URL
  size = APPLOC_FILESIZE

  dprint("enter: size = %s, url = %s" % (size, url))

  from sakurakit import skfileio
  if os.path.exists(targetpath) and skfileio.filesize(targetpath) == size:
    dprint("leave: already downloaded")
    return True

  from sakurakit import sknetio
  ok = False
  with SkProfiler("fetch"):
    if sknetio.getfile(url, tmppath, flush=False): # flush=false to use more memory to reduce disk access
      ok = skfileio.filesize(tmppath) == size
      if ok:
        os.rename(tmppath, targetpath)
  if not ok and os.path.exists(tmppath):
    skfileio.removefile(tmppath)
  dprint("leave: ok = %s" % ok)
  return ok
Beispiel #6
0
def get(lang):  # str -> bool
    filename = DICS[lang]
    url = DIC_URL % filename
    minsize = MIN_DIC_SIZE
    path = TMP_DIR + '/' + filename
    targetpath = TARGET_DIR + '/' + filename

    dprint("enter: url = %s, minsize = %s" % (url, minsize))

    #from sakurakit import skfileio
    #if os.path.exists(path) and skfileio.filesize(path) == size:
    #  dprint("leave: already downloaded")
    #  return True

    ok = False
    from sakurakit import skfileio, sknetio
    with SkProfiler("fetch"):
        # gzip=True to automatically extract gzip
        # flush=false to use more memory to reduce disk access
        if sknetio.getfile(url, path, flush=False, gzip=True):
            ok = skfileio.filesize(path) > minsize
    if ok:
        os.renames(path, targetpath)
    elif os.path.exists(path):
        skfileio.removefile(path)
    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #7
0
def get(): # -> bool
  url = DIC_URL
  minsize = MIN_DIC_SIZE
  path = TMP_DIR + '/' + DIC_FILENAME
  path_compressed = path + '.gz'

  dprint("enter: url = %s, minsize = %s" % (url, minsize))

  #from sakurakit import skfileio
  #if os.path.exists(path) and skfileio.filesize(path) == size:
  #  dprint("leave: already downloaded")
  #  return True

  ok = False
  import gzip
  from sakurakit import skfileio, sknetio
  with SkProfiler("fetch"):
    # gzip=True to automatically extract gzip
    # flush=false to use more memory to reduce disk access
    if sknetio.getfile(url, path_compressed, flush=False, gzip=False):
      # Note: gzip=True does not extract gzip, it decompresses the header ... probs? >_<
      with gzip.open(path_compressed, 'rb') as f_in, open(path, 'wb') as f_out:
        f_content = f_in.read()
        f_out.write(f_content)
      ok = skfileio.filesize(path) > minsize
  if ok:
    skfileio.removefile(path_compressed)
  elif os.path.exists(path):
    skfileio.removefile(path)
    skfileio.removefile(path_compressed)
#  if not ok and os.path.exists(path):
#    skfileio.removefile(path)
  dprint("leave: ok = %s" % ok)
  return ok
Beispiel #8
0
def makedb(lang):  # str -> bool
    dprint("enter: lang = %s" % lang)
    dic = DICS[lang]

    ldpath = LD_DIR + '/' + lang + LD_SUFFIX
    dbpath = DB_DIR + '/' + lang + DB_SUFFIX
    tmppath = TMP_DIR + '/' + lang + DB_SUFFIX

    from sakurakit import skfileio
    for it in dbpath, tmppath:
        if os.path.exists(it):
            skfileio.removefile(it)

    if not os.path.exists(ldpath):
        dwarn("leave: ld does not exist: %s" % ldpath)
        return False

    ok = False
    from lingoes.lingoesdb import LingoesDb
    with SkProfiler("extract"):
        if LingoesDb(tmppath).create(ldpath, dic['inenc'], dic['outenc']):
            os.renames(tmppath, dbpath)  # renames to create DB_DIR
            ok = True
        elif os.path.exists(tmppath):
            skfileio.removefile(tmppath)

    if os.path.exists(ldpath):
        skfileio.removefile(ldpath)
    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #9
0
    def test():
        global session

        #s = u"オープニングやエンディングのアニメーションは単純に主人公を入れ替えた程度の物ではなく、タイトルロゴはもちろん金時や定春の行動や表情、登場する道具(万事屋の面々が乗る車のデザインなど)やクレジット文字など、細部に渡って変更がなされた。更に、坂田金時が『銀魂'』を最終回に追い込み新しいアニメ『まんたま』を始めようとした時にはエンディングや提供表示の煽りコメントが最終回を思わせる演出となり、『まんたま』でも専用のタイトルロゴとオープニングアニメーション(スタッフクレジット付き)が新造され、偽物の提供クレジットまで表示されるなど随所に至るまで徹底的な演出が行われた。また、テレビ欄では金魂篇終了回は『金魂'』最終回として、その翌週は新番組「銀魂'」として案内された。"
        s = u"お花の匂い"
        s = '"<html>&abcde"'
        s = u"그렇습니다"
        s = u"hello"
        s = u"悠真くんを攻略すれば210円か。なるほどなぁ…"
        s = u'るみちゃん、めでたい結婚を機にさ、名前変えたら'

        fr = "ja"
        fr = "zhs"
        fr = "es"
        fr = "th"

        fr = "en"
        fr = "es"
        to = "en"

        fr = "ja"
        to = "ko"

        s = "hello world"
        fr = "en"

        s = u"What are you doing for today?"
        fr = "en"

        from sakurakit.skprof import SkProfiler

        #from qtrequests import qtrequests
        #from PySide.QtNetwork import QNetworkAccessManager
        #session = qtrequests.Session(QNetworkAccessManager())
        #with SkProfiler():
        #  for i in range(1):
        #    t = translate(s, to=to, fr=fr)
        #print t

        m = []

        session = requests.Session()
        with SkProfiler():
            for i in range(1):
                t = translate(s, to=to, fr=fr, align=m)
        print s
        print t

        print json.dumps(m, indent=2, ensure_ascii=False)

        #session = requests
        #with SkProfiler():
        #  for i in range(10):
        #    t = translate(s, to=to, fr=fr)
        #print t

        app.quit()
Beispiel #10
0
def extract(dic):  # str -> bool
    dprint("enter: dic = %s" % dic)

    srcpath = TMP_DIR + '/' + FILENAME_TPL % dic
    tmppath = TMP_DIR + '/cabocha.' + dic
    targetpath = DIC_DIR + '/' + dic

    import shutil
    from sakurakit import skfileio
    with SkProfiler("extract"):
        ok = skfileio.extracttarbz2(srcpath, tmppath)
    if ok:
        dickey = 'ipa' if dic == 'ipadic' else dic
        for it in MODELS:
            model = "%s.%s.model" % (it, dickey)
            if not os.path.exists(tmppath + '/' + model):
                ok = False
                dwarn("missing model: %s" % model)
        if os.path.exists(targetpath):  # alway remove old path even if not ok
            shutil.rmtree(targetpath)
        if ok:
            os.renames(tmppath, targetpath)
        #child = skfileio.getfirstchilddir(tmppath)
        #modeldir = child + '/model'
        #if not os.path.exists(modeldir):
        #  dwarn("model dir does not exist")
        #else:
        #  from cabochajlp import cabochamodel
        #  if os.path.exists(targetpath):
        #    shutil.rmtree(targetpath)
        #  os.makedirs(targetpath)
        #  dickey = 'ipa' if dic == 'ipadic' else dic
        #  for model in MODELS:
        #    with SkProfiler():
        #      dprint("process model: %s" % model)
        #      input = "%s.%s.txt" % (model , dickey)
        #      output = "%s.%s.model" % (model , dickey)
        #      inputpath = os.path.abspath(modeldir + '/' + input)
        #      outputpath = os.path.abspath(modeldir + '/' + output)
        #      if cabochamodel.modelindex(outputpath, inputpath):
        #        os.renames(outputpath, targetpath + '/' + output)
        #      else:
        #        dwarn("failed to process model %s" % model)
    if os.path.exists(tmppath):
        shutil.rmtree(tmppath)
    skfileio.removefile(srcpath)

    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #11
0
def ld2db(ldpath, dbpath=None, inenc=None, outenc=None):
    """
  @param  ldpath  unicode
  @param* dbpath  unicode
  @param* inenc  str
  @param* outenc  str
  """
    if not inenc:
        inenc = 'utf8'
    if not outenc:
        outenc = inenc
    dprint("enter: inenc = %s, outenc = %s" % (inenc, outenc))

    if not dbpath or os.path.isdir(dbpath):
        basename = os.path.basename(ldpath)
        if ldpath.endswith(LD_SUFFIX):
            path = os.path.splitext(basename)[0] + DB_SUFFIX
        else:
            path = basename
        if dbpath:
            dbpath = os.path.join(dbpath, path)
        else:
            dbpath = path

    if not os.path.exists(ldpath):
        dwarn("ld file does not exist: %s" % ldpath)
        return False

    if os.path.exists(dbpath):
        dwarn("db file already exist: %s" % dbpath)
        return False

    tmppath = dbpath + TMP_SUFFIX
    from sakurakit import skfileio
    if os.path.exists(tmppath):
        skfileio.removefile(tmppath)

    ok = False
    from lingoes.lingoesdb import LingoesDb
    with SkProfiler():
        if LingoesDb(tmppath).create(ldpath, inenc, outenc):
            os.renames(tmppath, dbpath)  # renames to create DB_DIR
            ok = True
        elif os.path.exists(tmppath):
            skfileio.removefile(tmppath)

    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #12
0
    def test():
        global session

        #s = u"オープニングやエンディングのアニメーションは単純に主人公を入れ替えた程度の物ではなく、タイトルロゴはもちろん金時や定春の行動や表情、登場する道具(万事屋の面々が乗る車のデザインなど)やクレジット文字など、細部に渡って変更がなされた。更に、坂田金時が『銀魂'』を最終回に追い込み新しいアニメ『まんたま』を始めようとした時にはエンディングや提供表示の煽りコメントが最終回を思わせる演出となり、『まんたま』でも専用のタイトルロゴとオープニングアニメーション(スタッフクレジット付き)が新造され、偽物の提供クレジットまで表示されるなど随所に至るまで徹底的な演出が行われた。また、テレビ欄では金魂篇終了回は『金魂'』最終回として、その翌週は新番組「銀魂'」として案内された。"
        #s = "test"
        #s = u"悠真くんを攻略すれば210円か。\nなるほどなぁ…\""
        #s = u"なるほどなぁ…"
        #s = '"<html>&abcde"'
        s = u"それで、人まで殺した。"

        fr = 'ja'
        to = 'en'
        #to = 'zhs'

        #s = u"What are you doing?"
        #fr = "en"

        from sakurakit.skprof import SkProfiler

        #from qtrequests import qtrequests
        #from PySide.QtNetwork import QNetworkAccessManager
        #session = qtrequests.Session(QNetworkAccessManager())
        #with SkProfiler():
        #  for i in range(1):
        #    t = translate(appid, s, to=to, fr=fr)
        #print t

        m = []
        session = requests.Session()
        with SkProfiler():
            for i in range(1):
                t = translate(appid, s, to=to, fr=fr, align=m)
        print t
        print type(t)
        print json.dumps(m, indent=2, ensure_ascii=False)

        #session = requests
        #e = create_engine()
        #with SkProfiler():
        #  for i in range(10):
        #    t = e.translate(s, to=to, fr=fr)
        #print t

        app.quit()
Beispiel #13
0
def get(dic): # str -> bool
  url = DIC_URL + DICS[dic]['file']
  path = TMP_DIR + '/' + DICS[dic]['file']

  dprint("enter: dic = %s, url = %s" % (dic, url))

  #from sakurakit import skfileio
  #if os.path.exists(path) and skfileio.filesize(path) == size:
  #  dprint("leave: already downloaded")
  #  return True

  ok = False
  from sakurakit import skfileio, sknetio
  with SkProfiler("fetch"):
    if sknetio.getfile(url, path, flush=False): # flush=false to use more memory to reduce disk access
      ok = skfileio.filesize(path) == DICS[dic]['size']
  if not ok and os.path.exists(path):
    skfileio.removefile(path)
  dprint("leave: ok = %s" % ok)
  return ok
Beispiel #14
0
def get(family): # str -> bool
  font = FONTS[family]
  url = font['dl']
  path = TMP_DIR + '/font-%s.%s' % (family, font['format'])

  dprint("enter: family = %s, url = %s" % (family, url))

  #from sakurakit import skfileio
  #if os.path.exists(path) and skfileio.filesize(path) == size:
  #  dprint("leave: already downloaded")
  #  return True

  ok = False
  from sakurakit import skfileio, sknetio
  with SkProfiler("fetch"):
    if sknetio.getfile(url, path, flush=False): # flush=false to use more memory to reduce disk access
      ok = skfileio.filesize(path) == font['size']
  if not ok and os.path.exists(path):
    skfileio.removefile(path)
  dprint("leave: ok = %s" % ok)
  return ok
Beispiel #15
0
def get(): # return bool
  url = UNIDIC_URL
  path = TMP_DIR + '/' + UNIDIC_FILENAME + UNIDIC_SUFFIX
  size = UNIDIC_FILESIZE

  dprint("enter: size = %s, url = %s" % (size, url))

  from sakurakit import skfileio
  if os.path.exists(path) and skfileio.filesize(path) == size:
    dprint("leave: already downloaded")
    return True

  from sakurakit import sknetio
  ok = False
  with SkProfiler("fetch"):
    if sknetio.getfile(url, path, flush=False): # flush=false to use more memory to reduce disk access
      ok = skfileio.filesize(path) == size
  if not ok and os.path.exists(path):
    skfileio.removefile(path)
  dprint("leave: ok = %s" % ok)
  return ok
Beispiel #16
0
def extract():
    dprint("enter")

    srcpath = TMP_DIR + '/' + WADOKU_FILENAME + WADOKU_SUFFIX
    tmppath = TMP_DIR + '/' + WADOKU_FILENAME
    targetpath = TARGET_DIR

    import shutil
    from sakurakit import skfileio
    with SkProfiler("extract"):
        ok = skfileio.extractzip(srcpath, tmppath)
    if ok:
        if os.path.exists(targetpath):
            shutil.rmtree(targetpath)
        os.renames(tmppath, targetpath)
    if os.path.exists(tmppath):
        shutil.rmtree(tmppath)
    skfileio.removefile(srcpath)

    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #17
0
def extract(lang):  # str -> bool
    dprint("enter: lang = %s" % lang)

    srcpath = TMP_DIR + '/' + DICS[lang]['file']
    tmppath = TMP_DIR + '/ovdp-' + lang
    targetpath = DIC_DIR + '/' + DICS[lang]['path']

    import shutil
    from sakurakit import skfileio
    with SkProfiler("extract"):
        ok = skfileio.extractzip(srcpath, tmppath)
    if ok:
        if os.path.exists(targetpath):
            shutil.rmtree(targetpath)
        child = skfileio.getfirstchilddir(tmppath)
        os.renames(child, targetpath)
    if os.path.exists(tmppath):
        shutil.rmtree(tmppath)
    skfileio.removefile(srcpath)

    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #18
0
def extract(lang):  # str -> bool
    dprint("enter: lang = %s" % lang)

    srcpath = TMP_DIR + '/' + FILENAME_TPL % lang
    tmppath = TMP_DIR + '/JMDict-' + lang
    targetpath = DIC_DIR + '/%s.fpw' % lang

    import shutil
    from sakurakit import skfileio
    with SkProfiler("extract"):
        ok = skfileio.extracttar(srcpath, tmppath)
    if ok:
        if os.path.exists(targetpath):
            shutil.rmtree(targetpath)
        child = skfileio.getfirstchilddir(tmppath)
        os.renames(child, targetpath)
    if os.path.exists(tmppath):
        shutil.rmtree(tmppath)
    skfileio.removefile(srcpath)

    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #19
0
def extract(dic): # str -> bool
  dprint("enter: dic = %s" % dic)

  srcpath = TMP_DIR + '/' + DICS[dic]['file']
  tmppath = TMP_DIR + '/stardic-' + dic
  targetpath = DIC_DIR + '/' + DICS[dic]['path']

  import shutil
  from sakurakit import skfileio
  with SkProfiler("extract"):
    ok = skfileio.extracttarbz2(srcpath, tmppath)
  if ok:
    if os.path.exists(targetpath):
      shutil.rmtree(targetpath)
    child = skfileio.getfirstchilddir(tmppath)
    os.renames(child, targetpath)
  if os.path.exists(tmppath):
    shutil.rmtree(tmppath)
  skfileio.removefile(srcpath)

  dprint("leave: ok = %s" % ok)
  return ok
Beispiel #20
0
def extract():
    dprint("enter")

    srcpath = TMP_DIR + '/' + UNIDIC_FILENAME + UNIDIC_SUFFIX
    tmppath = TMP_DIR + '/' + UNIDIC_FILENAME
    targetpath = TARGET_DIR

    import shutil
    from sakurakit import skfileio
    with SkProfiler("extract"):
        ok = skfileio.extracttarbz2(srcpath, tmppath)
    if ok:
        if os.path.exists(targetpath):
            shutil.rmtree(targetpath)
        child = skfileio.getfirstchilddir(tmppath)
        os.renames(child, targetpath)
    if os.path.exists(tmppath):
        shutil.rmtree(tmppath)
    skfileio.removefile(srcpath)

    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #21
0
def extract(family): # str -> bool
  dprint("enter: family = %s" % family)
  font = FONTS[family]

  fmt = font['format']
  tmppath = TMP_DIR + '/font-%s' % family
  srcpath = tmppath + '.' + fmt
  targetpath = FONT_DIR + '/' + font['path']

  import shutil
  from sakurakit import skfileio
  with SkProfiler("extract"):
    ok = skfileio.extractarchive(srcpath, tmppath, type=fmt)
  if ok:
    if os.path.exists(targetpath):
      shutil.rmtree(targetpath)
    os.renames(tmppath, targetpath)
  if os.path.exists(tmppath):
    shutil.rmtree(tmppath)
  skfileio.removefile(srcpath)

  dprint("leave: ok = %s" % ok)
  return ok
Beispiel #22
0
def get(dic):  # str -> bool
    url = URL_TPL % dic
    minsize = MIN_FILESIZE
    path = TMP_DIR + '/' + FILENAME_TPL % dic

    dprint("enter: url = %s, minsize = %s" % (url, minsize))

    #from sakurakit import skfileio
    #if os.path.exists(path) and skfileio.filesize(path) == size:
    #  dprint("leave: already downloaded")
    #  return True

    ok = False
    from sakurakit import skfileio, sknetio
    with SkProfiler("fetch"):
        if sknetio.getfile(
                url, path, flush=False
        ):  # flush=false to use more memory to reduce disk access
            ok = skfileio.filesize(path) > minsize
    if not ok and os.path.exists(path):
        skfileio.removefile(path)
    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #23
0
def get(lang):  # str -> bool
    dprint("enter: lang = %s" % lang)
    url = URL + DICS[lang]['file']
    path = TMP_DIR + '/' + FILENAME_TPL % lang

    dprint("enter: url = %s" % url)

    #from sakurakit import skfileio
    #if os.path.exists(path) and skfileio.filesize(path) == size:
    #  dprint("leave: already downloaded")
    #  return True

    ok = False
    from sakurakit import skfileio, sknetio
    with SkProfiler("fetch"):
        if sknetio.getfile(
                url, path, flush=False
        ):  # flush=false to use more memory to reduce disk access
            ok = skfileio.filesize(path) > MIN_FILESIZE
    if not ok and os.path.exists(path):
        skfileio.removefile(path)
    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #24
0
def get():  # -> bool
    url = DIC_URL
    minsize = MIN_DIC_SIZE
    path = TMP_DIR + '/' + DIC_FILENAME

    dprint("enter: url = %s, minsize = %s" % (url, minsize))

    #from sakurakit import skfileio
    #if os.path.exists(path) and skfileio.filesize(path) == size:
    #  dprint("leave: already downloaded")
    #  return True

    ok = False
    from sakurakit import skfileio, sknetio
    with SkProfiler("fetch"):
        # gzip=True to automatically extract gzip
        # flush=false to use more memory to reduce disk access
        if sknetio.getfile(url, path, flush=False, gzip=True):
            ok = skfileio.filesize(path) > minsize
    if not ok and os.path.exists(path):
        skfileio.removefile(path)
    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #25
0
def extract():
    dprint("enter")

    srcpath = TMP_DIR + '/' + IPADIC_FILENAME + IPADIC_SUFFIX
    tmppath = TMP_DIR + '/' + IPADIC_FILENAME
    targetpath = TARGET_DIR

    import shutil
    from sakurakit import skfileio
    with SkProfiler("extract"):
        ok = skfileio.extracttarbz2(srcpath, tmppath)
    if ok:
        if os.path.exists(targetpath):
            shutil.rmtree(targetpath)
        #child = skfileio.getfirstchilddir(tmppath)
        #child = os.path.join(tmppath, '/opt/local/lib/mecab/dic/ipadic-utf8')
        child = tmppath + IPADIC_RELPATH
        os.renames(child, targetpath)
    if os.path.exists(tmppath):
        shutil.rmtree(tmppath)
    skfileio.removefile(srcpath)

    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #26
0
def getld(lang):  # str -> bool
    url = DICS[lang].get(
        'url') or "http://%s/pub/lingoes/%s.ld2" % (initdefs.DOMAIN_ORG, lang)
    size = DICS[lang]['size']
    path = LD_DIR + '/' + lang + LD_SUFFIX

    dprint("enter: lang = %s, size = %s" % (lang, size))

    from sakurakit import skfileio
    if os.path.exists(path) and skfileio.filesize(path) == size:
        dprint("leave: already downloaded")
        return True

    ok = False
    from sakurakit import sknetio
    with SkProfiler("fetch"):
        if sknetio.getfile(
                url, path, flush=False
        ):  # flush=false to use more memory to reduce disk access
            ok = skfileio.filesize(path) == size
    if not ok and os.path.exists(path):
        skfileio.removefile(path)
    dprint("leave: ok = %s" % ok)
    return ok
Beispiel #27
0
 def test_create():
     with SkProfiler("make db"):
         print makedb(dbpath, dictpath)
     with SkProfiler("make surface"):
         print makesurface(dbpath)
Beispiel #28
0
        # [1] to get xmls, [0] to get the first xml
        try:
            return searchutil.lookup(t, self.data)[1][0]
        except:
            pass


if __name__ == '__main__':
    import os
    from sakurakit.skprof import SkProfiler
    location = '/Users/jichi/Dropbox/Developer/Dictionaries/LD2/'
    if os.name == 'nt':
        location = 'C:' + location

    dic = 'GBK Japanese-Chinese Dictionary.ld2'
    with SkProfiler():
        ld = LingoesDict(location + dic, 'utf8')
    print '-' * 4

    #dic = 'Monash Romanized Japanese-English Dictionary.ld2'
    #with SkProfiler():
    #  ld = LingoesDict(location + dic, 'utf8')
    #print '-' * 4

    #dic = 'New Japanese-Chinese Dictionary.ld2'
    #with SkProfiler():
    #  ld = LingoesDict(location + dic, 'utf16')
    #print '-' * 4

    #dic = 'Naver Japanese-Korean Dictionary.ld2'
    #ld = LingoesDict(location + dic, 'utf8')
Beispiel #29
0
 def test_compile():
     with SkProfiler("compile"):
         dicdir = "../../../../../../Caches/Dictionaries/UniDic"
         #dicdir = "/opt/local/lib/mecab/dic/unidic-utf8"
         print compile('edict.dic', csvpath, dicdir=dicdir)
Beispiel #30
0
 def test_assemble():
     with SkProfiler("assemble"):
         #dbpath = '../dictp/edict.db'
         dbpath = '/Users/jichi/stream/Caches/Dictionaries/EDICT/edict.db'
         import mdedict
         mdedict.db2csv(csvpath, dbpath)