def process(self): terms, definition, format = self._curr_row["data"] if definition == "DATA": res_dirname = os.path.join(self.plugin.output_directory, "res") mkdir_p(res_dirname) with open(os.path.join(res_dirname, terms), "wb") as f: f.write(format.getData()) else: if isinstance(terms, str): terms = [terms] term = terms[0] alts = terms[1:] definition = self.do_bgl_definition(definition, term) self.append(term, definition, alts)
def run(self): conn = sqlite3.connect(self.plugin.output_db) c = conn.cursor() zdirname = os.path.join(self.plugin.output_directory, "zip") uzdirname = os.path.join(self.plugin.output_directory, "raw") resdirname = os.path.join(self.plugin.output_directory, "res") read_cursor = conn.cursor() for rawid, zfile, flag in read_cursor.execute( ''' SELECT id, data, flag FROM raw WHERE flag & :flag == :flag AND flag & :nonflag == 0 ''', { "flag": FLAGS["ZIP_FETCHER"] | FLAGS["FETCHED"], "nonflag": FLAGS["PROCESSED"] }): with zipfile.ZipFile(zfile) as z: for n in filter(self.zfile_filter, z.namelist()): dest = os.path.join(uzdirname, n) destdir = os.path.dirname(dest) mkdir_p(destdir) if not os.path.isdir(dest): with open(dest, 'wb') as f: f.write(z.read(n)) c.execute( ''' INSERT INTO raw (uri, flag) VALUES (?,?) ''', (dest, FLAGS["FILE"])) for n in filter(self.zfile_resfilter, z.namelist()): dest = os.path.join(resdirname, os.path.basename(n)) with open(dest, 'wb') as f: f.write(z.read(n)) c.execute( ''' UPDATE raw SET flag=? WHERE id=? ''', (flag | FLAGS["PROCESSED"], rawid)) if self._canceled: break conn.commit() conn.close()
def reset(self): uzdirname = os.path.join(self.plugin.output_directory, "raw") resdirname = os.path.join(self.plugin.output_directory, "res") for dn in [uzdirname, resdirname]: if os.path.exists(dn): shutil.rmtree(dn) mkdir_p(dn) conn = sqlite3.connect(self.plugin.output_db) c = conn.cursor() c.execute( ''' DELETE FROM raw WHERE flag & ? > 0 ''', (FLAGS["FILE"], )) c.execute( ''' UPDATE raw SET flag = flag & ~:nonflag WHERE flag & :flag == :flag ''', { "flag": FLAGS["ZIP_FETCHER"] | FLAGS["FETCHED"], "nonflag": FLAGS["PROCESSED"] }) conn.commit() conn.close()
def setup(self): mkdir_p(os.path.join(self.output_directory, "raw")) mkdir_p(os.path.join(self.output_directory, "zip")) mkdir_p(os.path.join(self.output_directory, "res")) if not os.path.exists(self.output_db): conn = sqlite3.connect(self.output_db) c = conn.cursor() c.execute(''' CREATE TABLE raw ( id INTEGER PRIMARY KEY, uri TEXT, data TEXT, flag INTEGER ) ''') c.execute(''' CREATE INDEX raw_uri_idx ON raw (uri) ''') c.execute(''' CREATE INDEX raw_data_idx ON raw (data) ''') c.execute(''' CREATE TABLE dict ( id INTEGER PRIMARY KEY, word TEXT, def TEXT, rawid INTEGER ) ''') c.execute(''' CREATE TABLE synonyms ( id INTEGER PRIMARY KEY, wid INTEGER, syn TEXT ) ''') c.execute(''' CREATE INDEX synonym_wid_idx ON synonyms (wid) ''') c.execute(''' CREATE TABLE info ( id INTEGER PRIMARY KEY, key TEXT, value TEXT ) ''') self.post_setup(c) c.execute( ''' INSERT INTO info(key,value) VALUES (?,?) ''', ("bookname", self.dictname)) conn.commit() conn.close()