def process(self):
terms, definition, format = self._curr_row["data"]
if definition == "DATA":
res_dirname = os.path.join(self.plugin.output_directory, "res")
mkdir_p(res_dirname)
with open(os.path.join(res_dirname, terms), "wb") as f:
f.write(format.getData())
else:
if isinstance(terms, str): terms = [terms]
term = terms[0]
alts = terms[1:]
definition = self.do_bgl_definition(definition, term)
self.append(term, definition, alts)
def run(self):
conn = sqlite3.connect(self.plugin.output_db)
c = conn.cursor()
zdirname = os.path.join(self.plugin.output_directory, "zip")
uzdirname = os.path.join(self.plugin.output_directory, "raw")
resdirname = os.path.join(self.plugin.output_directory, "res")
read_cursor = conn.cursor()
for rawid, zfile, flag in read_cursor.execute(
'''
SELECT id, data, flag FROM raw
WHERE flag & :flag == :flag
AND flag & :nonflag == 0
''', {
"flag": FLAGS["ZIP_FETCHER"] | FLAGS["FETCHED"],
"nonflag": FLAGS["PROCESSED"]
}):
with zipfile.ZipFile(zfile) as z:
for n in filter(self.zfile_filter, z.namelist()):
dest = os.path.join(uzdirname, n)
destdir = os.path.dirname(dest)
mkdir_p(destdir)
if not os.path.isdir(dest):
with open(dest, 'wb') as f:
f.write(z.read(n))
c.execute(
'''
INSERT INTO raw (uri, flag)
VALUES (?,?)
''', (dest, FLAGS["FILE"]))
for n in filter(self.zfile_resfilter, z.namelist()):
dest = os.path.join(resdirname, os.path.basename(n))
with open(dest, 'wb') as f:
f.write(z.read(n))
c.execute(
'''
UPDATE raw
SET flag=?
WHERE id=?
''', (flag | FLAGS["PROCESSED"], rawid))
if self._canceled: break
conn.commit()
conn.close()
def reset(self):
uzdirname = os.path.join(self.plugin.output_directory, "raw")
resdirname = os.path.join(self.plugin.output_directory, "res")
for dn in [uzdirname, resdirname]:
if os.path.exists(dn):
shutil.rmtree(dn)
mkdir_p(dn)
conn = sqlite3.connect(self.plugin.output_db)
c = conn.cursor()
c.execute(
'''
DELETE FROM raw WHERE flag & ? > 0
''', (FLAGS["FILE"], ))
c.execute(
'''
UPDATE raw SET flag = flag & ~:nonflag
WHERE flag & :flag == :flag
''', {
"flag": FLAGS["ZIP_FETCHER"] | FLAGS["FETCHED"],
"nonflag": FLAGS["PROCESSED"]
})
conn.commit()
conn.close()
def setup(self):
mkdir_p(os.path.join(self.output_directory, "raw"))
mkdir_p(os.path.join(self.output_directory, "zip"))
mkdir_p(os.path.join(self.output_directory, "res"))
if not os.path.exists(self.output_db):
conn = sqlite3.connect(self.output_db)
c = conn.cursor()
c.execute('''
CREATE TABLE raw (
id INTEGER PRIMARY KEY,
uri TEXT,
data TEXT,
flag INTEGER
)
''')
c.execute('''
CREATE INDEX raw_uri_idx ON raw (uri)
''')
c.execute('''
CREATE INDEX raw_data_idx ON raw (data)
''')
c.execute('''
CREATE TABLE dict (
id INTEGER PRIMARY KEY,
word TEXT,
def TEXT,
rawid INTEGER
)
''')
c.execute('''
CREATE TABLE synonyms (
id INTEGER PRIMARY KEY,
wid INTEGER,
syn TEXT
)
''')
c.execute('''
CREATE INDEX synonym_wid_idx ON synonyms (wid)
''')
c.execute('''
CREATE TABLE info (
id INTEGER PRIMARY KEY,
key TEXT,
value TEXT
)
''')
self.post_setup(c)
c.execute(
'''
INSERT INTO info(key,value) VALUES (?,?)
''', ("bookname", self.dictname))
conn.commit()
conn.close()