def help_about(self): year = datetime.date.today().year year = "2020-{}".format(str(year)[-2:]) if year != 2020 else "2020" TEMPLATE = ('Python {}.{}.{} <br> {} <br> {}' '<br>APSW {} <br> SQLite {}<br>{}<br>{}') BINDING = f'PySide2 {PySide2.__version__}' QT = f'Qt {PySide2.QtCore.qVersion()}' info = TEMPLATE.format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro, BINDING, QT, apsw.apswversion(), apsw.sqlitelibversion(), QSysInfo.prettyProductName(), platform.platform()) QMessageBox.about(self, f'About — {APPNAME}', f'''<p> <font color=navy><b>{APPNAME} {VERSION}</b></font></p> <p> <font color=navy>{APPNAME} is an easy to learn and use GUI application for viewing, creating, editing, and updating SQLite and {APPNAME} databases. </font> </p> <p><a href="https://github.com/mark-summerfield/songbird">Source Code</a> </p> <p>Copyright © {year} Mark Summerfield.<br> All Rights Reserved.</p> <p> This software is Free Open Source Software (FOSS) licensed under the GNU Public License version 3 (GPLv3). </p> <hr> <p><font color=teal>{info}</font></p> '''.format(year=year, info=info)) # noqa
def construct_compiler(self): """Construct compiler.""" # Parse version string version_info = tuple( [int(value) for value in apsw.sqlitelibversion().split('.')]) # Construct compiler return self.plugins.get_compiler('sqlite')(self, version=version_info)
def first_time_setup(): """Show a page with some configuration data and some simple stats.""" dbinfo = {'note count': 0, 'source count': 0, 'sqlite version': apsw.sqlitelibversion()} from sys import version dbinfo['python version'] = version cfg = {'dbinfo': dbinfo, 'cfg file': config} return template('templates/config', cfg=cfg)
def show_config_page(): """Show a page with some configuration data and some simple stats.""" dbinfo = { 'note count': dbq("SELECT COUNT(id) FROM NOTES WHERE source_id IS NULL")[0][ "COUNT(id)"], 'source count': dbq("SELECT COUNT(id) FROM NOTES WHERE source_id IS NOT NULL")[0][ "COUNT(id)"], 'sqlite version': apsw.sqlitelibversion()} from sys import version dbinfo['python version'] = version cfg = {'dbinfo': dbinfo, 'cfg file': config} return template('templates/config', cfg=cfg)
def show_config_page(): """Show a page with some configuration data and some simple stats.""" dbinfo = { 'note count': dbq("SELECT COUNT(id) FROM NOTES WHERE source_id IS NULL")[0] ["COUNT(id)"], 'source count': dbq("SELECT COUNT(id) FROM NOTES WHERE source_id IS NOT NULL")[0] ["COUNT(id)"], 'sqlite version': apsw.sqlitelibversion() } from sys import version dbinfo['python version'] = version cfg = {'dbinfo': dbinfo, 'cfg file': config} return template('templates/config', cfg=cfg)
def apsw_is_available(): """Checks lock-ability for SQLite on the current file system""" try: import apsw #another python sqlite wrapper (maybe supports URIs) except ImportError: return False # if you got here, apsw is available, check we have matching versions w.r.t # the sqlit3 module import sqlite3 if apsw.sqlitelibversion() != sqlite3.sqlite_version: return False # if you get to this point, all seems OK return True
def parse_args(cls, argv): parser = argparse.ArgumentParser( description='clquery command line tool.') parser.add_argument('--aws-profile', default=None, required=False, help='AWS profile name of the credentials to use.') parser.add_argument( '-V', '--version', action='version', version=', '.join([ 'clquery ' + pkg_resources.require('clquery')[0].version, 'apsw ' + apsw.apswversion(), 'sqlite ' + apsw.sqlitelibversion() ])) args = parser.parse_args() cls.set_all(args)
def inext(v): # next value from iterator return next(v) if py3 else v.next() ### ### Check we have the expected version of apsw and sqlite ### print(" Using APSW file", apsw.__file__) # from the extension module print(" APSW version", apsw.apswversion()) # from the extension module print(" SQLite lib version", apsw.sqlitelibversion()) # from the sqlite library code print("SQLite header version", apsw.SQLITE_VERSION_NUMBER) # from the sqlite header file at compile time #%% ### ### Opening/creating database ### connection = apsw.Connection("dbfile") cursor = connection.cursor() #%% ### ### update hook ### def myupdatehook(type, databasename, tablename, rowid): print("Updated: %s database %s, table %s, row %d" % ( apsw.mapping_authorizer_function[type], databasename, tablename, rowid)) cursor.execute("create table foo(x,y,z)") connection.setupdatehook(myupdatehook) cursor.execute("insert into foo values(?,?,?)", (1, 1.1, None)) # integer, float/real, Null cursor.execute("insert into foo(x) values(?)", ("abc", )) # string (note trailing comma to ensure tuple!) cursor.execute("insert into foo(x) values(?)", (b"abc\xff\xfe",)) # a blob (binary data) connection.setupdatehook(None)
def __init__(self): try: if debuglevel > 1: print "\nUsing APSW file", apsw.__file__ # from the extension module print "APSW version", apsw.apswversion( ) # from the extension module print "SQLite lib version", apsw.sqlitelibversion( ) # from the sqlite library code print "SQLite header version", apsw.SQLITE_VERSION_NUMBER # from the sqlite header file at compile time self.dbcon = apsw.Connection(settings.BAKER_DB) self.lastrowid = 0 self.seconds = 6 # time to sleep before final close of DB pub.subscribe(self.close, 'ShutdownDB') pub.subscribe(self.bmRecSave, 'RecSaveDB') pub.subscribe(self.bmSendSave, 'SendSaveDB') pub.subscribe(self.BakerCmdInsertRunner, 'BakerCmdInsertRunner') pub.subscribe(self.BakerCmdReport1, 'BakerCmdReport1') except apsw.Error, e: print "APSW error - all args:", e.args self.dbcon = False print "Error - clsBakerDB %s:" % e.args[0] if self.dbcon: self.dbcon.close()
def __init__(self, db_filename): self._logger = logging.getLogger(__name__) self._logger.debug('APSW version ' + apsw.apswversion()) self._logger.debug('SQLite version ' + apsw.sqlitelibversion()) self._conn = None self._connect_to_db(db_filename)
sqlandmtermstatements=['select ', 'create ', 'where ', 'table ', 'group by ', 'drop ', 'order by ', 'index ', 'from ', 'alter ', 'limit ', 'delete ', '..', "attach database '", 'detach database ', 'distinct', 'exists '] dotcompletions=['.help ', '.colnums', '.schema ', '.functions ', '.tables', '.quote', '.explain ', '.vacuum', '.quit'] allfuncs=functions.functions['vtable'].keys()+functions.functions['row'].keys()+functions.functions['aggregate'].keys() alltables=[] alltablescompl=[] updated_tables=set() update_tablelist() lastcols=[] newcols=[] colscompl=[] #Intro Message if not pipedinput: print mtermdetails print "running on Python: "+'.'.join([str(x) for x in sys.version_info[0:3]])+', APSW: '+apsw.apswversion()+', SQLite: '+apsw.sqlitelibversion(), try: sys.stdout.write(", madIS: "+functions.VERSION+'\n') except: print print intromessage number_of_kb_exceptions=0 while True: statement = raw_input_no_history("mterm> ") if statement==None: number_of_kb_exceptions+=1 print if number_of_kb_exceptions<2: continue else:
import os, sys, time import apsw print "using APSW file: ", apsw.__file__ print "APSW version: ", apsw.apswversion() print "SQLite version: ", apsw.sqlitelibversion() print "SQLite header version: ", apsw.SQLITE_VERSION_NUMBER connection = apsw.Connection("yogeshmk.db") cursor = connection.cursor() cursor.execute("create table tt (a, b, c)") cursor.execute("insert into tt values (?, ?, ?)", (1, 1.1, None)) # int, float, NULL cursor.execute("insert into tt (a) values(?)", ("abc",)) # single element tuple cursor.execute("insert into tt (a) values(?)", (buffer("abc\xff\xfe"), )) # multiple statements cursor.execute("delete from tt; insert into tt values(1,2,3); create table bar(a,b,c) ; insert into tt values(4, 'five', 6.0)") # iterator for a, b, c in cursor.execute("select a, b, c from tt"): print cursor.getdescription() # show column names and datatypes print a, b, c cursor.execute("insert into tt (:aplha, :beta, :gamma)", {'beta': 2, 'aplha':1, 'gamma':3})
import os, sys, time import apsw ### ### Check we have the expected version of apsw and sqlite ### #@@CAPTURE print " Using APSW file",apsw.__file__ # from the extension module print " APSW version",apsw.apswversion() # from the extension module print " SQLite lib version",apsw.sqlitelibversion() # from the sqlite library code print "SQLite header version",apsw.SQLITE_VERSION_NUMBER # from the sqlite header file at compile time #@@ENDCAPTURE ### ### Opening/creating database ### connection=apsw.Connection("dbfile") cursor=connection.cursor() ### ### simple statement @@ example-cursor ### cursor.execute("create table foo(x,y,z)") ### ### using different types ###
def sanitycheck(): "Check all dependencies are present and at the correct version" print "=== Sanity check ===" print "svn location", if not "$HeadURL$".split( ":", 1)[1].strip().startswith("https://bitpim.svn.sourceforge.net"): raise Exception( "Needs to be checked out from https://bitpim.svn.sourceforge.net") print " OK" print "python version", if sys.version_info[:2] != (2, 5): raise Exception("Should be Python 2.5 - this is " + sys.version) print " OK" print "wxPython version", import wx if wx.VERSION[:4] != (2, 8, 8, 1): raise Exception("Should be wxPython 2.8.8.1. This is " + ` wx.VERSION `) print " OK" print "wxPython is unicode build", if not wx.USE_UNICODE: raise Exception("You need a unicode build of wxPython") print " OK" if sys.platform != 'win32': print "native.usb", import native.usb print " OK" print "pycrypto version", expect = '2.0.1' import Crypto if Crypto.__version__ != expect: raise Exception("Should be %s version of pycrypto - you have %s" % (expect, Crypto.__version__)) print " OK" print "paramiko version", expect = '1.7.4 (Desmond)' import paramiko if paramiko.__version__ != expect: raise Exception("Should be %s version of paramiko - you have %s" % (expect, paramiko.__version__)) print " OK" print "bitfling", import bitfling print " OK" print "pyserial", import serial print " OK" print "apsw", import apsw ver = "3.5.9-r2" if apsw.apswversion() != ver: raise Exception("Should be apsw version %s - you have %s" % (ver, apsw.apswversion())) print " OK" print "sqlite", ver = "3.6.1" if apsw.sqlitelibversion() != ver: raise Exception("Should be sqlite version %s - you have %s" % (ver, apsw.sqlitelibversion())) print " OK" print "jaro/winkler string matcher", import native.strings.jarow print " OK" # bsddb (Linux only, for evolution) if sys.platform == "linux2": print "bsddb ", import bsddb print " OK" # win32com.shell - See http://starship.python.net/crew/theller/moin.cgi/WinShell if sys.platform == 'win32': import py2exe.mf as modulefinder # in py2exe < 0.6.4 use "import modulefinder" import win32com for p in win32com.__path__[1:]: modulefinder.AddPackagePath("win32com", p) for extra in ["win32com.shell"]: #,"win32com.mapi" __import__(extra) m = sys.modules[extra] for p in m.__path__[1:]: modulefinder.AddPackagePath(extra, p) print "=== All checks out ==="
class APSWDatabase(SqliteExtDatabase): server_version = tuple(int(i) for i in apsw.sqlitelibversion().split('.')) def __init__(self, database, **kwargs): self._modules = {} super(APSWDatabase, self).__init__(database, **kwargs) def register_module(self, mod_name, mod_inst): self._modules[mod_name] = mod_inst if not self.is_closed(): self.connection().createmodule(mod_name, mod_inst) def unregister_module(self, mod_name): del (self._modules[mod_name]) def _connect(self): conn = apsw.Connection(self.database, **self.connect_params) if self._timeout is not None: conn.setbusytimeout(self._timeout * 1000) try: self._add_conn_hooks(conn) except: conn.close() raise return conn def _add_conn_hooks(self, conn): super(APSWDatabase, self)._add_conn_hooks(conn) self._load_modules(conn) # APSW-only. def _load_modules(self, conn): for mod_name, mod_inst in self._modules.items(): conn.createmodule(mod_name, mod_inst) return conn def _load_aggregates(self, conn): for name, (klass, num_params) in self._aggregates.items(): def make_aggregate(): return (klass(), klass.step, klass.finalize) conn.createaggregatefunction(name, make_aggregate) def _load_collations(self, conn): for name, fn in self._collations.items(): conn.createcollation(name, fn) def _load_functions(self, conn): for name, (fn, num_params) in self._functions.items(): conn.createscalarfunction(name, fn, num_params) def _load_extensions(self, conn): conn.enableloadextension(True) for extension in self._extensions: conn.loadextension(extension) def load_extension(self, extension): self._extensions.add(extension) if not self.is_closed(): conn = self.connection() conn.enableloadextension(True) conn.loadextension(extension) def last_insert_id(self, cursor, query_type=None): return cursor.getconnection().last_insert_rowid() def rows_affected(self, cursor): return cursor.getconnection().changes() def begin(self, lock_type='deferred'): self.cursor().execute('begin %s;' % lock_type) def commit(self): curs = self.cursor() if curs.getconnection().getautocommit(): return False curs.execute('commit;') return True def rollback(self): curs = self.cursor() if curs.getconnection().getautocommit(): return False curs.execute('rollback;') return True def execute_sql(self, sql, params=None, commit=True): logger.debug((sql, params)) with __exception_wrapper__: cursor = self.cursor() cursor.execute(sql, params or ()) return cursor
t = config[table] v = t.get(key,None) t[key] = value return v except KeyError: pass db.createscalarfunction(name,getter,2) db.createscalarfunction(name,setter,3) return mod __all__ = ['WebQueryModule','attach'] if __name__ == '__main__': import sys, re shell = apsw.Shell() mod = attach(shell.db,'webquery') loaded = [] for fn in sys.argv[1:]: tbl = re.sub(r'\W','_',fn) if tbl.endswith('_yml'): tbl = tbl[:-4] sql = 'CREATE VIRTUAL TABLE %s USING webquery(%s);'%(tbl,fn) try: shell.process_sql(sql) loaded.append('> '+sql+'\r\n') except KeyboardInterrupt: raise except: pass shell.cmdloop(intro=('SQLite version %s (APSW %s)\r\nEnter ".help" for instructions\r\nEnter SQL statements terminated with a ";"\r\n'%(apsw.sqlitelibversion(),apsw.apswversion()))+''.join(loaded))
except ImportError: # Python 2.6 from lib.collections26 import OrderedDict try: from inspect import isgeneratorfunction except ImportError: # Python < 2.6 def isgeneratorfunction(obj): return bool((inspect.isfunction(object) or inspect.ismethod(object)) and obj.func_code.co_flags & CO_GENERATOR) sys.setcheckinterval(1000) sqlite_version = apsw.sqlitelibversion() apsw_version = apsw.apswversion() VTCREATE = 'create virtual table temp.' SQLITEAFTER3711 = False SQLITEAFTER380 = False sqlite_version_split = [int(x) for x in sqlite_version.split('.')] if sqlite_version_split[0:3] >= [3, 8, 0]: SQLITEAFTER380 = True try: if sqlite_version_split[0:3] >= [3, 7, 11]: VTCREATE = 'create virtual table if not exists temp.' SQLITEAFTER3711 = True except Exception, e:
def sanitycheck(): "Check all dependencies are present and at the correct version" print "=== Sanity check ===" print "svn location", if not "$HeadURL: https://bitpim.svn.sourceforge.net/svnroot/bitpim/releases/1.0.7/src/package.py $".split(":",1)[1].strip().startswith("https://bitpim.svn.sourceforge.net"): raise Exception("Needs to be checked out from https://bitpim.svn.sourceforge.net") print " OK" print "python version", if sys.version_info[:2]!=(2,5): raise Exception("Should be Python 2.5 - this is "+sys.version) print " OK" print "wxPython version", import wx if wx.VERSION[:4]!=(2,8,8,1): raise Exception("Should be wxPython 2.8.8.1. This is "+`wx.VERSION`) print " OK" print "wxPython is unicode build", if not wx.USE_UNICODE: raise Exception("You need a unicode build of wxPython") print " OK" if sys.platform!='win32': print "native.usb", import native.usb print " OK" print "pycrypto version", expect='2.0.1' import Crypto if Crypto.__version__!=expect: raise Exception("Should be %s version of pycrypto - you have %s" % (expect, Crypto.__version__)) print " OK" print "paramiko version", expect='1.7.4 (Desmond)' import paramiko if paramiko.__version__!=expect: raise Exception("Should be %s version of paramiko - you have %s" % (expect, paramiko.__version__)) print " OK" print "bitfling", import bitfling print " OK" print "pyserial", import serial print " OK" print "apsw", import apsw ver="3.5.9-r2" if apsw.apswversion()!=ver: raise Exception("Should be apsw version %s - you have %s" % (ver, apsw.apswversion())) print " OK" print "sqlite", ver="3.6.1" if apsw.sqlitelibversion()!=ver: raise Exception("Should be sqlite version %s - you have %s" % (ver, apsw.sqlitelibversion())) print " OK" print "jaro/winkler string matcher", import native.strings.jarow print " OK" # bsddb (Linux only, for evolution) if sys.platform=="linux2": print "bsddb ", import bsddb print " OK" # win32com.shell - See http://starship.python.net/crew/theller/moin.cgi/WinShell if sys.platform=='win32': import py2exe.mf as modulefinder # in py2exe < 0.6.4 use "import modulefinder" import win32com for p in win32com.__path__[1:]: modulefinder.AddPackagePath("win32com", p) for extra in ["win32com.shell"]: #,"win32com.mapi" __import__(extra) m = sys.modules[extra] for p in m.__path__[1:]: modulefinder.AddPackagePath(extra, p) print "=== All checks out ==="
def deserialize(self, ignore_drm=False): self.ignore_drm = ignore_drm self.fragments.clear() self.kpf_datafile = self.kdf_datafile = self.kcb_data = None if self.datafile.is_zipfile(): self.kpf_datafile = self.datafile with self.kpf_datafile.as_ZipFile() as zf: for info in zf.infolist(): ext = os.path.splitext(info.filename)[1] if ext == ".kdf": self.kdf_datafile = DataFile(info.filename, zf.read(info), self.kpf_datafile) elif ext == ".kdf-journal": if len(zf.read(info)) > 0: raise Exception("kdf-journal is not empty in %s" % self.kpf_datafile.name) elif ext == ".kcb": self.kcb_data = json_deserialize(zf.read(info)) if self.kdf_datafile is None: raise Exception("Failed to locate KDF within %s" % self.datafile.name) else: self.kdf_datafile = self.datafile unwrapped_kdf_datafile = SQLiteFingerprintWrapper( self.kdf_datafile).remove() db_filename = (unwrapped_kdf_datafile.name if unwrapped_kdf_datafile.is_real_file and not self.book.is_netfs else temp_filename( "kdf", unwrapped_kdf_datafile.get_data())) if have_apsw: if natural_sort_key( apsw.sqlitelibversion()) < natural_sort_key("3.8.2"): raise Exception( "SQLite version 3.8.2 or later is necessary in order to use a WITHOUT ROWID table. Found version %s" % apsw.sqlitelibversion()) conn = apsw.Connection(db_filename) else: if sqlite3.sqlite_version_info < (3, 8, 2): raise Exception( "SQLite version 3.8.2 or later is necessary in order to use a WITHOUT ROWID table. Found version %s" % sqlite3.sqlite_version) conn = sqlite3.connect(db_filename, KpfContainer.db_timeout) cursor = conn.cursor() sql_list = cursor.execute( "SELECT sql FROM sqlite_master WHERE type='table';").fetchall() schema = set([x[0] for x in sql_list]) dictionary_index_terms = set() first_head_word = "" INDEX_INFO_SCHEMA = ( "CREATE TABLE index_info(namespace char(256), index_name char(256), property char(40), " "primary key (namespace, index_name)) without rowid") if INDEX_INFO_SCHEMA in schema: schema.remove(INDEX_INFO_SCHEMA) self.book.is_dictionary = True for namespace, index_name, property in cursor.execute( "SELECT * FROM index_info;"): if namespace != "dictionary" or property != "yj.dictionary.term": log.error( "unexpected index_info: namespace=%s, index_name=%s, property=%s" % (namespace, index_name, property)) table_name = "index_%s_%s" % (namespace, index_name) index_schema = ( "CREATE TABLE %s ([%s] char(256), id char(40), " "primary key ([%s], id)) without rowid") % ( table_name, property, property) if index_schema in schema: schema.remove(index_schema) num_entries = 0 index_words = set() index_kfx_ids = set() for dictionary_term, kfx_id in cursor.execute( "SELECT * FROM %s;" % table_name): num_entries += 1 dictionary_index_terms.add( (dictionary_term, IS(kfx_id))) index_words.add(dictionary_term) index_kfx_ids.add(kfx_id) if dictionary_term < first_head_word or not first_head_word: first_head_word = dictionary_term log.info( "Dictionary %s table has %d entries with %d terms and %d definitions" % (table_name, num_entries, len(index_words), len(index_kfx_ids))) else: log.error("KPF database is missing the '%s' table" % table_name) self.eid_symbol = {} KFXID_TRANSLATION_SCHEMA = "CREATE TABLE kfxid_translation(eid INTEGER, kfxid char(40), primary key(eid)) without rowid" if KFXID_TRANSLATION_SCHEMA in schema: schema.remove(KFXID_TRANSLATION_SCHEMA) for eid, kfx_id in cursor.execute( "SELECT * FROM kfxid_translation;"): self.eid_symbol[eid] = self.create_local_symbol(kfx_id) self.element_type = {} FRAGMENT_PROPERTIES_SCHEMA = ( "CREATE TABLE fragment_properties(id char(40), key char(40), value char(40), " "primary key (id, key, value)) without rowid") if FRAGMENT_PROPERTIES_SCHEMA in schema: schema.remove(FRAGMENT_PROPERTIES_SCHEMA) for id, key, value in cursor.execute( "SELECT * FROM fragment_properties;"): if key == "child": pass elif key == "element_type": self.element_type[id] = value else: log.error( "fragment_property has unknown key: id=%s key=%s value=%s" % (id, key, value)) self.max_eid_in_sections = None FRAGMENTS_SCHEMA = "CREATE TABLE fragments(id char(40), payload_type char(10), payload_value blob, primary key (id))" if FRAGMENTS_SCHEMA in schema: schema.remove(FRAGMENTS_SCHEMA) for id in ["$ion_symbol_table", "max_id"]: rows = cursor.execute( "SELECT payload_value FROM fragments WHERE id = ? AND payload_type = 'blob';", (id, )).fetchall() if rows: payload_data = self.prep_payload_blob(rows[0][0]) if payload_data is None: pass elif id == "$ion_symbol_table": self.symtab.creating_yj_local_symbols = True sym_import = IonBinary( self.symtab).deserialize_annotated_value( payload_data, expect_annotation="$ion_symbol_table", import_symbols=True) self.symtab.creating_yj_local_symbols = False if DEBUG: log.info("kdf symbol import = %s" % json_serialize(sym_import)) self.fragments.append(YJFragment(sym_import)) break else: max_id = IonBinary( self.symtab).deserialize_single_value(payload_data) if DEBUG: log.info("kdf max_id = %d" % max_id) self.symtab.clear() self.symtab.import_shared_symbol_table( "YJ_symbols", max_id=max_id - len(SYSTEM_SYMBOL_TABLE.symbols)) self.fragments.append( YJFragment(self.symtab.create_import())) for id, payload_type, payload_value in cursor.execute( "SELECT * FROM fragments;"): ftype = id if payload_type == "blob": payload_data = self.prep_payload_blob(payload_value) if id in ["max_id", "$ion_symbol_table"]: pass elif payload_data is None: ftype = self.element_type.get(id) elif id == "max_eid_in_sections": ftype = None self.max_eid_in_sections = IonBinary( self.symtab).deserialize_single_value(payload_data) if self.book.is_dictionary: pass else: log.warning( "Unexpected max_eid_in_sections for non-dictionary: %d" % self.max_eid_in_sections) elif not payload_data.startswith(IonBinary.SIGNATURE): ftype = None self.fragments.append( YJFragment(ftype="$417", fid=self.create_local_symbol(id), value=IonBLOB(payload_data))) elif len(payload_data) == len(IonBinary.SIGNATURE): if id != "book_navigation": log.warning("Ignoring empty %s fragment" % id) else: value = IonBinary( self.symtab).deserialize_annotated_value( payload_data) if not isinstance(value, IonAnnotation): log.error( "KDF fragment id=%s is missing annotation: %s" % (id, repr(value))) continue elif len(value.annotations ) == 2 and value.annotations[1] == "$608": pass elif len(value.annotations) > 1: log.error( "KDF fragment should have one annotation: %s" % repr(value)) ftype = value.annotations[0] if ftype in ROOT_FRAGMENT_TYPES: # shortcut when symbol table unavailable fid = None else: fid = self.create_local_symbol(id) self.fragments.append( YJFragment(ftype=ftype, fid=fid, value=self.deref_kfx_ids(value.value))) elif payload_type == "path": ftype = "$417" resource_data = self.get_resource_data( self.prep_payload_blob(payload_value).decode("utf8")) if resource_data is not None: self.fragments.append( YJFragment(ftype=ftype, fid=self.create_local_symbol(id), value=IonBLOB(resource_data))) else: log.error( "Unexpected KDF payload_type=%s, id=%s, value=%d bytes" % (payload_type, id, len(payload_value))) else: log.error("KPF database is missing the 'fragments' table") GC_FRAGMENT_PROPERTIES_SCHEMA = ( "CREATE TABLE gc_fragment_properties(id varchar(40), key varchar(40), " "value varchar(40), primary key (id, key, value)) without rowid") if GC_FRAGMENT_PROPERTIES_SCHEMA in schema: schema.remove(GC_FRAGMENT_PROPERTIES_SCHEMA) GC_REACHABLE_SCHEMA = ( "CREATE TABLE gc_reachable(id varchar(40), primary key (id)) without rowid" ) if GC_REACHABLE_SCHEMA in schema: schema.remove(GC_REACHABLE_SCHEMA) CAPABILITIES_SCHEMA = "CREATE TABLE capabilities(key char(20), version smallint, primary key (key, version)) without rowid" if CAPABILITIES_SCHEMA in schema: schema.remove(CAPABILITIES_SCHEMA) capabilities = cursor.execute( "SELECT * FROM capabilities;").fetchall() if capabilities: format_capabilities = [ IonStruct(IS("$492"), key, IS("version"), version) for key, version in capabilities ] self.fragments.append( YJFragment(ftype="$593", value=format_capabilities)) else: log.error("KPF database is missing the 'capabilities' table") if len(schema) > 0: for s in list(schema): log.error("Unexpected KDF database schema: %s" % s) cursor.close() conn.close() self.book.is_kpf_prepub = True book_metadata_fragment = self.fragments.get("$490") if book_metadata_fragment is not None: for cm in book_metadata_fragment.value.get("$491", {}): if cm.get("$495", "") == "kindle_title_metadata": for kv in cm.get("$258", []): if kv.get("$492", "") in [ "ASIN", "asset_id", "cde_content_type", "content_id" ]: self.book.is_kpf_prepub = False break break self.fragments.append( YJFragment(ftype="$270", value=IonStruct(IS("$587"), "", IS("$588"), "", IS("$161"), CONTAINER_FORMAT_KPF))) additional_metadata = self.get_resource_data( ADDITIONAL_METADATA_FILENAME, report_missing=False) self.additional_metadata = json_deserialize( additional_metadata) if additional_metadata else {}
dotcompletions = ['.help ', '.colnums', '.schema ', '.functions ', '.tables', '.explain ', '.vacuum', '.queryplan '] allfuncs = functions.functions['vtable'].keys() + functions.functions['row'].keys() + functions.functions[ 'aggregate'].keys() alltables = [] alltablescompl = [] updated_tables = set() update_tablelist() lastcols = [] newcols = [] colscompl = [] # Intro Message if not pipedinput: print mtermdetails print "running on Python: " + '.'.join([str(x) for x in sys.version_info[ 0:3]]) + ', APSW: ' + apsw.apswversion() + ', SQLite: ' + apsw.sqlitelibversion(), try: sys.stdout.write(", madIS: " + functions.VERSION + '\n') except: print print intromessage number_of_kb_exceptions = 0 while True: statement = raw_input_no_history("mterm> ") if statement == None: number_of_kb_exceptions += 1 print if number_of_kb_exceptions < 2: continue else:
def printAPSWinfo(): print " Using APSW file",apsw.__file__ # from the extension module print " APSW version",apsw.apswversion() # from the extension module print " SQLite lib version",apsw.sqlitelibversion() # from the sqlite library code print "SQLite header version",apsw.SQLITE_VERSION_NUMBER # from the sqlite header file at compile time
'row'].keys() + functions.functions['aggregate'].keys() alltables = [] alltablescompl = [] updated_tables = set() update_tablelist() lastcols = [] newcols = [] colscompl = [] #Intro Message if not pipedinput: print mtermdetails print "running on Python: " + '.'.join( [str(x) for x in sys.version_info[0:3]]) + ', APSW: ' + apsw.apswversion( ) + ', SQLite: ' + apsw.sqlitelibversion(), try: sys.stdout.write(", madIS: " + functions.VERSION + '\n') except: print print intromessage number_of_kb_exceptions = 0 while True: statement = raw_input_no_history("mterm> ") if statement == None: number_of_kb_exceptions += 1 print if number_of_kb_exceptions < 2: continue else:
import os, sys, time import apsw ### ### Check we have the expected version of apsw and sqlite ### print (" Using APSW file",apsw.__file__ ) # from the extension module print (" APSW version",apsw.apswversion() ) # from the extension module print (" SQLite lib version",apsw.sqlitelibversion() ) # from the sqlite library code print ("SQLite header version",apsw.SQLITE_VERSION_NUMBER ) # from the sqlite header file at compile time ### ### Opening/creating database ### connection=apsw.Connection("dbfile.apsw_db") cursor=connection.cursor() ### ### simple statement ### cursor.execute("create table foo(x,y,z)") ### ### using different types ### cursor.execute("insert into foo values(?,?,?)", (1, 1.1, None)) # integer, float/real, Null
def sanitycheck(): "Check all dependencies are present and at the correct version" print "=== Sanity check ===" print "python version", if sys.version_info[:2]!=(2,3): raise Exception("Should be Python 2.3 - this is "+sys.version) print " OK" print "wxPython version", import wx if wx.VERSION[:4]!=(2,6,2,1): raise Exception("Should be wxPython 2.6.2.1. This is "+`wx.VERSION`) print " OK" print "wxPython is unicode build", if not wx.USE_UNICODE: raise Exception("You need a unicode build of wxPython") print " OK" if sys.platform!='win32': print "native.usb", import native.usb print " OK" print "pycrypto version", expect='2.0.1' import Crypto if Crypto.__version__!=expect: raise Exception("Should be %s version of pycrypto - you have %s" % (expect, Crypto.__version__)) print " OK" print "paramiko version", expect='1.4 (oddish)' import paramiko if paramiko.__version__!=expect: raise Exception("Should be %s version of paramiko - you have %s" % (expect, paramiko.__version__)) print " OK" print "bitfling", import bitfling print " OK" print "pyserial", import serial print " OK" print "apsw", import apsw ver="3.2.7-r1" if apsw.apswversion()!=ver: raise Exception("Should be apsw version %s - you have %s" % (ver, apsw.apswversion())) print " OK" print "sqlite", ver="3.2.7" if apsw.sqlitelibversion()!=ver: raise Exception("Should be sqlite version %s - you have %s" % (ver, apsw.sqlitelibversion())) print " OK" print "jaro/winkler string matcher", import native.strings.jarow print " OK" if sys.platform=="linux2": print "bsddb ", import bsddb print " OK" if sys.platform=='win32': import py2exe.mf as modulefinder import win32com for p in win32com.__path__[1:]: modulefinder.AddPackagePath("win32com", p) for extra in ["win32com.shell"]: __import__(extra) m = sys.modules[extra] for p in m.__path__[1:]: modulefinder.AddPackagePath(extra, p) print "=== All checks out ==="
Module Attributes: ----------- :initsql: SQL commands that are executed whenever a new connection is created. ''' from .logging import logging, QuietError # Ensure use of custom logger class import apsw import os log = logging.getLogger(__name__) sqlite_ver = tuple([ int(x) for x in apsw.sqlitelibversion().split('.') ]) if sqlite_ver < (3, 7, 0): raise QuietError('SQLite version too old, must be 3.7.0 or newer!\n') initsql = ( # WAL mode causes trouble with e.g. copy_tree, so we don't use it at the moment # (cf. http://article.gmane.org/gmane.comp.db.sqlite.general/65243). # However, if we start using it we must initiaze it *before* setting # locking_mode to EXCLUSIVE, otherwise we can't switch the locking # mode without first disabling WAL. 'PRAGMA synchronous = OFF', 'PRAGMA journal_mode = OFF', #'PRAGMA synchronous = NORMAL', #'PRAGMA journal_mode = WAL',
'''The current state of things as we approach version 3 is not bad at all. We still need to do some work on stability and flexibility, and there's still a lot of duplicated code in this file (db.py,) but overall we're making good progress. The cache works fine, and UDP works as expected. Some work should be put in on simplifying and strengthening the code in here, since this is the part of the program that does the most work with the least clarity.''' import os.path import apsw import config import udp version = "Database:\n APSW version: " + apsw.apswversion() + "\n SQLite version: " + apsw.sqlitelibversion() + "\n" # Do we need to call Connection.close()? # View this next line monospace... def doclose(con): # If APSW's version's third major number is greater than 8... if int(apsw.apswversion().split('.')[2].split('-')[0]) >= 8: con.close() conf = config.config() db = os.path.normpath(os.path.expanduser("~") + "/.oadb/anime.db") data = { "anime": ["aid", "eps", "epcount", "spcount", "arating", "avotes", "tmprating", "tmpvotes", "average", "ratings", "year", "type", "aromaji", "akanji", "aenglish", "aother", "ashort", "synonyms", "cats"],
<td>v1.10.24</td> <td> <a href="https://datatables.net/"> https://datatables.net </a> <td> </tr> </table> """.format(version=KRAIT_VERSION, build=KRAIT_BUILD, python=sys.version.split()[0], pyside=PySide6.__version__, stria=stria.version(), pyfastx=pyfastx.version(), apsw=apsw.apswversion(), sqlite=apsw.sqlitelibversion(), primerpy=primer3.__version__) #default parameter and type for krait KRAIT_PARAMETERS = { 'SSR/mono': (12, int), 'SSR/di': (7, int), 'SSR/tri': (5, int), 'SSR/tetra': (4, int), 'SSR/penta': (4, int), 'SSR/hexa': (4, int), 'CSSR/dmax': (10, int), 'VNTR/minmotif': (7, int), 'VNTR/maxmotif': (30, int), 'VNTR/minrep': (3, int), #'ITR/minmsize': (1, int),
'''The current state of things as we approach version 3 is not bad at all. We still need to do some work on stability and flexibility, and there's still a lot of duplicated code in this file (db.py,) but overall we're making good progress. The cache works fine, and UDP works as expected. Some work should be put in on simplifying and strengthening the code in here, since this is the part of the program that does the most work with the least clarity.''' import os.path import apsw import config import udp version = "Database:\n APSW version: " + apsw.apswversion( ) + "\n SQLite version: " + apsw.sqlitelibversion() + "\n" # Do we need to call Connection.close()? # View this next line monospace... def doclose(con): # If APSW's version's third major number is greater than 8... if int(apsw.apswversion().split('.')[2].split('-')[0]) >= 8: con.close() conf = config.config() db = os.path.normpath(os.path.expanduser("~") + "/.oadb/anime.db") data = {
Module Attributes: ----------- :initsql: SQL commands that are executed whenever a new connection is created. ''' from .logging import logging # Ensure use of custom logger class from .common import QuietError import apsw import os log = logging.getLogger(__name__) sqlite_ver = tuple([ int(x) for x in apsw.sqlitelibversion().split('.') ]) if sqlite_ver < (3, 7, 0): raise QuietError('SQLite version too old, must be 3.7.0 or newer!\n') initsql = ( # WAL mode causes trouble with e.g. copy_tree, so we don't use it at the moment # (cf. http://article.gmane.org/gmane.comp.db.sqlite.general/65243). # However, if we start using it we must initiaze it *before* setting # locking_mode to EXCLUSIVE, otherwise we can't switch the locking # mode without first disabling WAL. 'PRAGMA synchronous = OFF', 'PRAGMA journal_mode = OFF', #'PRAGMA synchronous = NORMAL', #'PRAGMA journal_mode = WAL',
def doit(): random.seed(0) options.tests=[t.strip() for t in options.tests.split(",")] write(" Python %s %s\n" % (sys.executable, str(sys.version_info))) write(" Scale %d\n" % (options.scale,)) write(" Database %s\n" % (options.database,)) write(" Tests %s\n" % (", ".join(options.tests),)) write(" Iterations %d\n" % (options.iterations,)) write("Statement Cache %d\n" % (options.scsize,)) write("\n") if options.apsw: import apsw write(" Testing with APSW file "+apsw.__file__+"\n") write(" APSW version "+apsw.apswversion()+"\n") write(" SQLite lib version "+apsw.sqlitelibversion()+"\n") write(" SQLite headers version "+str(apsw.SQLITE_VERSION_NUMBER)+"\n\n") def apsw_setup(dbfile): con=apsw.Connection(dbfile, statementcachesize=options.scsize) con.createscalarfunction("number_name", number_name, 1) return con if options.pysqlite: try: from pysqlite2 import dbapi2 as pysqlite except ImportError: import sqlite3 as pysqlite write("Testing with pysqlite file "+pysqlite.__file__+"\n") write(" pysqlite version "+pysqlite.version+"\n") write(" SQLite version "+pysqlite.sqlite_version+"\n\n") def pysqlite_setup(dbfile): con=pysqlite.connect(dbfile, isolation_level=None, cached_statements=options.scsize) con.create_function("number_name", 1, number_name) return con ones=("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen") tens=("", "ten", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety") others=("thousand", "hundred", "zero") def _number_name(n): if n>=1000: txt="%s %s" % (_number_name(int(n/1000)), others[0]) n=n%1000 else: txt="" if n>=100: txt=txt+" "+ones[int(n/100)]+" "+others[1] n=n%100 if n>=20: txt=txt+" "+tens[int(n/10)] n=n%10 if n>0: txt=txt+" "+ones[n] txt=txt.strip() if txt=="": txt=others[2] return txt def unicodify(text): if options.unicode and len(text): newt=[] c=options.unicode/100.0 for t in text: if random.random()>c: newt.append(t) continue while True: t=random.randint(0xa1, maxuni) # we don't want the surrogate range or apostrophe if t<0xd800 or t>0xdfff: break newt.append(unichr(t)) text="".join(newt) return text if options.unicode: ones=tuple([unicodify(s) for s in ones]) tens=tuple([unicodify(s) for s in tens]) others=tuple([unicodify(s) for s in others]) def number_name(n): text=_number_name(n) if options.size: text=text*int(random.randint(0, options.size)/len(text)) return text def getlines(scale=50, bindings=False): random.seed(0) # RogerB added two pragmas so that only memory is used. This means that the # vagaries of disk access times don't alter the results # database schema for i in """PRAGMA page_size=1024; PRAGMA cache_size=8192; PRAGMA locking_mode=EXCLUSIVE; PRAGMA journal_mode = OFF; PRAGMA temp_store = MEMORY; CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT); CREATE TABLE t2(a INTEGER, b INTEGER, c TEXT); CREATE INDEX i2a ON t2(a); CREATE INDEX i2b ON t2(b); SELECT name FROM sqlite_master ORDER BY 1""".split(";"): yield (i,) # 50,000 inserts on an unindexed table yield ("BEGIN",) for i in xrange(1,scale*10000+1): r=random.randint(0,500000) if bindings: yield ("INSERT INTO t1 VALUES(:1, :2, number_name(:2))", (i, r)) else: yield ("INSERT INTO t1 VALUES(%d, %d, '%s')" % (i, r, number_name(r)),) yield ("COMMIT",) # 50,000 inserts on an indexed table t1c_list=[] yield ("BEGIN",) for i in xrange(1,scale*10000+1): r=random.randint(0,500000) x=number_name(r) t1c_list.append(x) if bindings: yield ("INSERT INTO t2 VALUES(:1, :2, number_name(:2))", (i, r)) else: yield ("INSERT INTO t2 VALUES(%d, %d, '%s')" % (i, r, x),) yield ("COMMIT",) # 50 SELECTs on an integer comparison. There is no index so # a full table scan is required. for i in xrange(scale): yield ("SELECT count(*), avg(b) FROM t1 WHERE b>=%d AND b<%d" % (i*100, (i+10)*100),) # 50 SELECTs on an LIKE comparison. There is no index so a full # table scan is required. for i in xrange(scale): yield ("SELECT count(*), avg(b) FROM t1 WHERE c LIKE '%%%s%%'" % (number_name(i),),) # Create indices yield ("BEGIN",) for i in """CREATE INDEX i1a ON t1(a); CREATE INDEX i1b ON t1(b); CREATE INDEX i1c ON t1(c);""".split(";"): yield (i,) yield ("COMMIT",) # 5000 SELECTs on an integer comparison where the integer is # indexed. for i in xrange(scale*100): yield ("SELECT count(*), avg(b) FROM t1 WHERE b>=%d AND b<%d" % (i*100, (i+10)*100),) # 100000 random SELECTs against rowid. for i in xrange(1,scale*2000+1): yield ("SELECT c FROM t1 WHERE rowid=%d" % (1+random.randint(0,50000),),) # 100000 random SELECTs against a unique indexed column. for i in xrange(1,scale*2000+1): yield ("SELECT c FROM t1 WHERE a=%d" % (1+random.randint(0,50000),),) # 50000 random SELECTs against an indexed column text column for i in xrange(scale*1000): if bindings: yield ("SELECT c FROM t1 WHERE c=?", (random.choice(t1c_list),),) else: yield ("SELECT c FROM t1 WHERE c='%s'" % (random.choice(t1c_list),),) # Vacuum if options.database!=":memory:": # opens a disk file yield ("VACUUM",) # 5000 updates of ranges where the field being compared is indexed. yield ("BEGIN",) for i in xrange(scale*100): yield ("UPDATE t1 SET b=b*2 WHERE a>=%d AND a<%d" % (i*2, (i+1)*2),) yield ("COMMIT",) # 50000 single-row updates. An index is used to find the row quickly. yield ("BEGIN",) for i in xrange(scale*1000): if bindings: yield ("UPDATE t1 SET b=? WHERE a=%d" % (i,), (random.randint(0,500000),)) else: yield ("UPDATE t1 SET b=%d WHERE a=%d" % (random.randint(0,500000), i),) yield ("COMMIT",) # 1 big text update that touches every row in the table. yield ("UPDATE t1 SET c=a",) # Many individual text updates. Each row in the table is # touched through an index. yield ("BEGIN",) for i in xrange(1,scale*1000+1): if bindings: yield ("UPDATE t1 SET c=? WHERE a=%d" % (i,), (number_name(random.randint(0,500000)),)) else: yield ("UPDATE t1 SET c='%s' WHERE a=%d" % (number_name(random.randint(0,500000)),i),) yield ("COMMIT",) # Delete all content in a table. yield ("DELETE FROM t1",) # Copy one table into another yield ("INSERT INTO t1 SELECT * FROM t2",) # Delete all content in a table, one row at a time. yield ("DELETE FROM t1 WHERE 1",) # Refill the table yet again yield ("INSERT INTO t1 SELECT * FROM t2",) # Drop the table and recreate it without its indices. yield ("BEGIN",) yield ("DROP TABLE t1",) yield ("CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT)",) yield ("COMMIT",) # Refill the table yet again. This copy should be faster because # there are no indices to deal with. yield ("INSERT INTO t1 SELECT * FROM t2",) # The three following used "ORDER BY random()" but we can't do that # as it causes each run to have different values, and hence different # amounts of sorting that have to go on. The "random()" has been # replaced by "c", the column that has the stringified number # Select 20000 rows from the table at random. yield ("SELECT rowid FROM t1 ORDER BY c LIMIT %d" % (scale*400,),) # Delete 20000 random rows from the table. yield (""" DELETE FROM t1 WHERE rowid IN (SELECT rowid FROM t1 ORDER BY c LIMIT %d)""" % (scale*400,),) yield ("SELECT count(*) FROM t1",) # Delete 20000 more rows at random from the table. yield ("""DELETE FROM t1 WHERE rowid IN (SELECT rowid FROM t1 ORDER BY c LIMIT %d)""" % (scale*400,),) yield ("SELECT count(*) FROM t1",) # Do a correctness test first if options.correctness: write("Correctness test\n") if 'bigstmt' in options.tests: text=";\n".join([x[0] for x in getlines(scale=1)])+";" if 'statements' in options.tests: withbindings=[line for line in getlines(scale=1, bindings=True)] if 'statements_nobindings' in options.tests: withoutbindings=[line for line in getlines(scale=1, bindings=False)] res={} for driver in ('apsw', 'pysqlite'): if not getattr(options, driver): continue for test in options.tests: name=driver+"_"+test write(name+'\t') sys.stdout.flush() if name=='pysqlite_bigstmt': write('limited functionality (ignoring)\n') continue con=globals().get(driver+"_setup")(":memory:") # we always correctness test on memory if test=='bigstmt': cursor=con.cursor() if driver=='apsw': func=cursor.execute else: func=cursor.executescript res[name]=[row for row in func(text)] write(str(len(res[name]))+"\n") continue cursor=con.cursor() if test=='statements': sql=withbindings elif test=='statements_nobindings': sql=withoutbindings l=[] for s in sql: for row in cursor.execute(*s): l.append(row) res[name]=l write(str(len(res[name]))+"\n") # All elements of res should be identical elements=res.keys() elements.sort() for i in range(0,len(elements)-1): write("%s == %s %s\n" % (elements[i], elements[i+1], res[elements[i]]==res[elements[i+1]])) del res text=None withbindings=None withoutbindings=None if options.dump_filename or "bigstmt" in options.tests: text=";\n".join([x[0] for x in getlines(scale=options.scale)])+";" # pysqlite requires final semicolon if options.dump_filename: open(options.dump_filename, "wt").write(text.encode("utf8")) sys.exit(0) if "statements" in options.tests: withbindings=list(getlines(scale=options.scale, bindings=True)) if "statements_nobindings" in options.tests: withoutbindings=list(getlines(scale=options.scale, bindings=False)) # Each test returns the amount of time taken. Note that we include # the close time as well. Otherwise the numbers become a function of # cache and other collection sizes as freeing members gets deferred to # close time. def apsw_bigstmt(con): "APSW big statement" try: for row in con.cursor().execute(text): pass except: import pdb ; pdb.set_trace() pass def pysqlite_bigstmt(con): "pysqlite big statement" for row in con.executescript(text): pass def apsw_statements(con, bindings=withbindings): "APSW individual statements with bindings" cursor=con.cursor() for b in bindings: for row in cursor.execute(*b): pass def pysqlite_statements(con, bindings=withbindings): "pysqlite individual statements with bindings" cursor=con.cursor() for b in bindings: for row in cursor.execute(*b): pass def apsw_statements_nobindings(con): "APSW individual statements without bindings" return apsw_statements(con, withoutbindings) def pysqlite_statements_nobindings(con): "pysqlite individual statements without bindings" return pysqlite_statements(con, withoutbindings) # Do the work write("\nRunning tests - elapsed, CPU (results in seconds, lower is better)\n") for i in range(options.iterations): write("%d/%d\n" % (i+1, options.iterations)) for test in options.tests: # funky stuff is to alternate order each round for driver in ( ("apsw", "pysqlite"), ("pysqlite", "apsw"))[i%2]: if getattr(options, driver): name=driver+"_"+test func=locals().get(name, None) if not func: sys.stderr.write("No such test "+name+"\n") sys.exit(1) if os.path.exists(options.database): os.remove(options.database) write("\t"+func.__name__+(" "*(40-len(func.__name__)))) sys.stdout.flush() con=locals().get(driver+"_setup")(options.database) gc.collect(2) b4cpu=time.clock() b4=time.time() func(con) con.close() # see note above as to why we include this in the timing gc.collect(2) after=time.time() aftercpu=time.clock() write("%0.3f %0.3f\n" % (after-b4, aftercpu-b4cpu)) # Cleanup if using valgrind if options.apsw: if hasattr(apsw, "_fini"): # Cleans out buffer recycle cache apsw._fini()
from collections import OrderedDict except ImportError: # Python 2.6 from lib.collections26 import OrderedDict try: from inspect import isgeneratorfunction except ImportError: # Python < 2.6 def isgeneratorfunction(obj): return bool((inspect.isfunction(object) or inspect.ismethod(object)) and obj.func_code.co_flags & CO_GENERATOR) sys.setcheckinterval(1000) sqlite_version = apsw.sqlitelibversion() apsw_version = apsw.apswversion() VTCREATE = 'create virtual table temp.' SQLITEAFTER3711 = False SQLITEAFTER380 = False sqlite_version_split = [int(x) for x in sqlite_version.split('.')] if sqlite_version_split[0:3] >= [3, 8, 0]: SQLITEAFTER380 = True try: if sqlite_version_split[0:3] >= [3, 7, 11]: VTCREATE = 'create virtual table if not exists temp.' SQLITEAFTER3711 = True except Exception, e:
try: import apsw except ImportError: # https://github.com/ghaering/pysqlite (https://docs.python.org/2/library/sqlite3.html) -- is C code... # pypi.python.org/pypi/PyDbLite , www.pydblite.net/en/index.html -- pure python, but a tad different from sqlite3. # from pydblite import sqlite # pydblite relies on built-in sqlite3 or pysqlite2... try: from .sqlite3_adhoc import apsw print("chrome_extract: Using sqlite3_adhoc apsw replacement module...") except ImportError as exc: print("ImportError while importing sqlite3 apsw stand-in module:", exc) #raise exc # Not fatal... apsw = None print("apsw module:", apsw) if apsw: print("apsw sqlite version:", apsw.sqlitelibversion()) print("apsw version:", apsw.apswversion()) try: from Crypto.Cipher import AES except ImportError: #warnings.warn("Could not import Crypto.Cipher AES module.") """ PyCrypto (Crypto) alternatives: (https://github.com/dlitz/pycrypto) pyOCB (github.com/kravietz/pyOCB) - could be used, but Chrome encrypts as CBC, not OCB. AES-Python (https://github.com/bozhu/AES-Python) --- another pure-python implementation. However, does not have an easy-to-use interface. --- Not sure what modes are supported, seems quite ad-hoc. Like... "hey, let me try to implement AES..." --- "only for AES-128"
return next(v) if py3 else v.next() import os import time import apsw ### ### Check we have the expected version of apsw and sqlite ### #@@CAPTURE print(" Using APSW file", apsw.__file__) # from the extension module print(" APSW version", apsw.apswversion()) # from the extension module print(" SQLite lib version", apsw.sqlitelibversion()) # from the sqlite library code print( "SQLite header version", apsw.SQLITE_VERSION_NUMBER) # from the sqlite header file at compile time #@@ENDCAPTURE ### ### Opening/creating database ### connection = apsw.Connection("dbfile") cursor = connection.cursor() ### ### simple statement @@ example-cursor ###
def __init__(self, cons_list, func_list, struc_list, apps_list): print "Using APSW file",apsw.__file__ # from the extension module print "APSW version",apsw.apswversion() # from the extension module print "SQLite version",apsw.sqlitelibversion() # from the sqlite library code ### ### Opening/creating database, initialize database ### self.apsw_version = apsw.apswversion() self.release_number = self.apsw_version[4:6] self.db_path = os.path.join(os.environ['PWD'],'db') #self.confReader = ConfReader('sockets_analysis.conf') #self.functions = self.confReader.getItems('functions') #self.structures = self.confReader.getItems('structures') self.functions = func_list self.structures = struc_list self.constants = cons_list function_temp = "" structure_temp = "" constant_temp = "" for constant in self.constants: constant_temp = constant_temp + constant[0].strip() + " int," for function in self.functions: function_temp = function_temp + function[0].strip() + " int," i = 0 len_item = len(self.structures) # length of items for structure in self.structures: if i < len_item - 1: structure_temp = structure_temp + structure[0].strip() + " int," else: structure_temp = structure_temp + structure[0].strip() + " int" i = i + 1 creat_table = "CREATE TABLE socket_statistic (name varchar PRIMARY KEY, " + constant_temp + function_temp + structure_temp + ")" creat_sum_table = "CREATE TABLE socket_statistic_sum (socket_api_name varchar PRIMARY KEY , sum_number int)" print creat_table print creat_sum_table #print creat_table if os.path.exists(self.db_path): print "database path existing......" #print "delete the existing", self.db_path #shutil.rmtree(self.db_path) #Removes directories recursively #pass else: print "create the db directory" os.mkdir('db') database_file = os.path.join(self.db_path, 'socket_analysis_data_sos.db') self.connection=apsw.Connection(database_file) self.cursor=self.connection.cursor() """ self.cursor.execute(creat_table) """ try: self.cursor.execute(creat_table) except: print "socket_statistic table is already there or something wrong with creating DB!!!" try: self.cursor.execute(creat_sum_table) except: print "socket_statistic_sum table is already there or something wrong with creating DB!!!"