def run(self):
# In Windows APSW should be installed manually
if os.name == 'nt':
print(
'To complete the installation you have to install APSW: https://github.com/rogerbinns/apsw/releases'
)
return
try:
import apsw
if apsw.apswversion() == APSW_VERSION:
print('APSW %s already installed' % apsw.apswversion())
return
else:
print('APSW %s already installed, need %s' %
(apsw.apswversion(), APSW_VERSION))
except Exception:
pass
from subprocess import check_call
print('Installing APSW...')
check_call(f'''\
pip install \
https://github.com/rogerbinns/apsw/releases/download/{APSW_VERSION}/apsw-{APSW_VERSION}.zip \
--global-option=fetch \
--global-option=--version \
--global-option={SQLITE_VERSION} \
--global-option=--all \
--global-option=build \
--global-option=--enable-all-extensions \
'''.split())
def sanitycheck():
"Check your dependencies are present and the correct versions."
# Raise an exception if there is a problem
# This tests apsw as an example
print "apsw",
import apsw
ver="3.2.7-r1"
if apsw.apswversion()!=ver:
raise Exception("Should be apsw version %s - you have %s" % (ver, apsw.apswversion()))
print " OK"
def help_about(self):
year = datetime.date.today().year
year = "2020-{}".format(str(year)[-2:]) if year != 2020 else "2020"
TEMPLATE = ('Python {}.{}.{} <br> {} <br> {}'
'<br>APSW {} <br> SQLite {}<br>{}<br>{}')
BINDING = f'PySide2 {PySide2.__version__}'
QT = f'Qt {PySide2.QtCore.qVersion()}'
info = TEMPLATE.format(sys.version_info.major, sys.version_info.minor,
sys.version_info.micro, BINDING, QT,
apsw.apswversion(), apsw.sqlitelibversion(),
QSysInfo.prettyProductName(),
platform.platform())
QMessageBox.about(self, f'About — {APPNAME}', f'''<p>
<font color=navy><b>{APPNAME} {VERSION}</b></font></p>
<p>
<font color=navy>{APPNAME} is an easy to learn and use GUI application for
viewing, creating, editing, and updating SQLite and {APPNAME} databases.
</font>
</p>
<p><a href="https://github.com/mark-summerfield/songbird">Source Code</a>
</p>
<p>Copyright © {year} Mark Summerfield.<br>
All Rights Reserved.</p>
<p>
This software is Free Open Source Software (FOSS) licensed under the
GNU Public License version 3 (GPLv3).
</p>
<hr>
<p><font color=teal>{info}</font></p>
'''.format(year=year, info=info)) # noqa
def run(self):
# In Windows APSW should be installed manually
if os.name == 'nt':
print(
'To complete the installation you have to install APSW: https://github.com/rogerbinns/apsw/releases'
)
return
try:
import apsw
if apsw.apswversion() == APSW_VERSION:
print('APSW %s already installed' % apsw.apswversion())
return
else:
print('APSW %s already installed, need %s' %
(apsw.apswversion(), APSW_VERSION))
except:
pass
print("downloading apsw.")
with urllib.request.urlopen('https://github.com/rogerbinns/apsw/releases/download/%s/apsw-%s.zip' % (APSW_VERSION, APSW_VERSION)) as u, \
open('apsw-%s.zip' % APSW_VERSION, 'wb') as f:
f.write(u.read())
print("extracting.")
with zipfile.ZipFile('apsw-%s.zip' % APSW_VERSION, 'r') as zip_file:
zip_file.extractall()
executable = sys.executable
if executable is None:
executable = "python"
print("install apsw.")
install_command = (
'cd apsw-{version} && {executable} '
'setup.py fetch --version={shortversion} --all build '
'--enable-all-extensions install'.format(
executable=executable,
version=APSW_VERSION,
shortversion=APSW_SHORT_VERSION))
os.system(install_command)
print("clean files.")
shutil.rmtree('apsw-%s' % APSW_VERSION)
os.remove('apsw-%s.zip' % APSW_VERSION)
def test_import():
import apsw
return {
'versions': {
'apsw': apsw.apswversion(),
'sqlite': apsw.SQLITE_VERSION_NUMBER
}
}
def run(self):
# In Windows APSW should be installed manually
if os.name == 'nt':
print('To complete the installation you have to install APSW: https://github.com/rogerbinns/apsw/releases')
return
try:
import apsw
if apsw.apswversion() == APSW_VERSION:
print('APSW %s already installed' % apsw.apswversion())
return
else:
print('APSW %s already installed, need %s' % (apsw.apswversion(), APSW_VERSION))
except:
pass
print("downloading apsw.")
with urllib.request.urlopen('https://github.com/rogerbinns/apsw/releases/download/%s/apsw-%s.zip' % (APSW_VERSION, APSW_VERSION)) as u, \
open('apsw-%s.zip' % APSW_VERSION, 'wb') as f:
f.write(u.read())
print("extracting.")
with zipfile.ZipFile('apsw-%s.zip' % APSW_VERSION, 'r') as zip_file:
zip_file.extractall()
executable = sys.executable
if executable is None:
executable = "python"
print("install apsw.")
install_command = ('cd apsw-{version} && {executable} '
'setup.py fetch --version={shortversion} --all build '
'--enable-all-extensions install'.format(executable=executable, version=APSW_VERSION, shortversion=APSW_SHORT_VERSION)
)
os.system(install_command)
print("clean files.")
shutil.rmtree('apsw-%s' % APSW_VERSION)
os.remove('apsw-%s.zip' % APSW_VERSION)
def init_db():
tmp_db = None
_connect_db(_db)
sqlite_version = ".".join([str(num) for num in _db.server_version])
log.info("SQLite version: {}, APSW version: {}".format(
sqlite_version, apswversion()))
if Settings.table_exists():
update_db()
else:
tmp_db = PooledSqliteDatabase(os.path.join(get_data_dir(), "cozy.db"))
if PeeweeVersion[0] == '2':
tmp_db.create_tables([
Track, Book, Settings, ArtworkCache, Storage, StorageBlackList,
OfflineCache
], True)
else:
with tmp_db.connection_context():
tmp_db.create_tables([
Track, Book, Settings, ArtworkCache, Storage,
StorageBlackList, OfflineCache
])
# this is necessary to ensure that the tables have indeed been created
if tmp_db:
if PeeweeVersion[0] == '2':
while not Settings.table_exists():
time.sleep(0.01)
else:
while not tmp_db.table_exists("settings"):
time.sleep(0.01)
_connect_db(_db)
if PeeweeVersion[0] == '3':
_db.bind([
Book, Track, Settings, ArtworkCache, StorageBlackList,
OfflineCache, Storage
],
bind_refs=False,
bind_backrefs=False)
if (Settings.select().count() == 0):
Settings.create(path="", last_played_book=None)
# TODO: Properly handle errors within the database
# Remove this later. It prevents empty book objects in the database
clean_books()
def initialise_db():
if config.FORCE:
logger.warning('THE OPTION `--force` IS NOT FOR USE ON PRODUCTION SYSTEMS.')
# Lock
if not config.FORCE:
get_lock()
# Database
logger.info('Connecting to database (SQLite %s).' % apsw.apswversion())
db = database.get_connection(read_only=False)
util.CURRENT_BLOCK_INDEX = blocks.last_db_index(db)
return db
def parse_args(cls, argv):
parser = argparse.ArgumentParser(
description='clquery command line tool.')
parser.add_argument('--aws-profile',
default=None,
required=False,
help='AWS profile name of the credentials to use.')
parser.add_argument(
'-V',
'--version',
action='version',
version=', '.join([
'clquery ' + pkg_resources.require('clquery')[0].version,
'apsw ' + apsw.apswversion(),
'sqlite ' + apsw.sqlitelibversion()
]))
args = parser.parse_args()
cls.set_all(args)
def inext(v): # next value from iterator
return next(v) if py3 else v.next()
###
### Check we have the expected version of apsw and sqlite
###
print(" Using APSW file", apsw.__file__) # from the extension module
print(" APSW version", apsw.apswversion()) # from the extension module
print(" SQLite lib version", apsw.sqlitelibversion()) # from the sqlite library code
print("SQLite header version", apsw.SQLITE_VERSION_NUMBER) # from the sqlite header file at compile time
#%%
###
### Opening/creating database
###
connection = apsw.Connection("dbfile")
cursor = connection.cursor()
#%%
###
### update hook
###
def myupdatehook(type, databasename, tablename, rowid):
print("Updated: %s database %s, table %s, row %d" % (
apsw.mapping_authorizer_function[type], databasename, tablename, rowid))
cursor.execute("create table foo(x,y,z)")
connection.setupdatehook(myupdatehook)
cursor.execute("insert into foo values(?,?,?)", (1, 1.1, None)) # integer, float/real, Null
cursor.execute("insert into foo(x) values(?)", ("abc", )) # string (note trailing comma to ensure tuple!)
cursor.execute("insert into foo(x) values(?)", (b"abc\xff\xfe",)) # a blob (binary data)
connection.setupdatehook(None)
def __init__(self):
try:
if debuglevel > 1:
print "\nUsing APSW file", apsw.__file__ # from the extension module
print "APSW version", apsw.apswversion(
) # from the extension module
print "SQLite lib version", apsw.sqlitelibversion(
) # from the sqlite library code
print "SQLite header version", apsw.SQLITE_VERSION_NUMBER # from the sqlite header file at compile time
self.dbcon = apsw.Connection(settings.BAKER_DB)
self.lastrowid = 0
self.seconds = 6 # time to sleep before final close of DB
pub.subscribe(self.close, 'ShutdownDB')
pub.subscribe(self.bmRecSave, 'RecSaveDB')
pub.subscribe(self.bmSendSave, 'SendSaveDB')
pub.subscribe(self.BakerCmdInsertRunner, 'BakerCmdInsertRunner')
pub.subscribe(self.BakerCmdReport1, 'BakerCmdReport1')
except apsw.Error, e:
print "APSW error - all args:", e.args
self.dbcon = False
print "Error - clsBakerDB %s:" % e.args[0]
if self.dbcon:
self.dbcon.close()
def sanitycheck():
"Check all dependencies are present and at the correct version"
print "=== Sanity check ==="
print "svn location",
if not "$HeadURL: https://bitpim.svn.sourceforge.net/svnroot/bitpim/releases/1.0.7/src/package.py $".split(":",1)[1].strip().startswith("https://bitpim.svn.sourceforge.net"):
raise Exception("Needs to be checked out from https://bitpim.svn.sourceforge.net")
print " OK"
print "python version",
if sys.version_info[:2]!=(2,5):
raise Exception("Should be Python 2.5 - this is "+sys.version)
print " OK"
print "wxPython version",
import wx
if wx.VERSION[:4]!=(2,8,8,1):
raise Exception("Should be wxPython 2.8.8.1. This is "+`wx.VERSION`)
print " OK"
print "wxPython is unicode build",
if not wx.USE_UNICODE:
raise Exception("You need a unicode build of wxPython")
print " OK"
if sys.platform!='win32':
print "native.usb",
import native.usb
print " OK"
print "pycrypto version",
expect='2.0.1'
import Crypto
if Crypto.__version__!=expect:
raise Exception("Should be %s version of pycrypto - you have %s" % (expect, Crypto.__version__))
print " OK"
print "paramiko version",
expect='1.7.4 (Desmond)'
import paramiko
if paramiko.__version__!=expect:
raise Exception("Should be %s version of paramiko - you have %s" % (expect, paramiko.__version__))
print " OK"
print "bitfling",
import bitfling
print " OK"
print "pyserial",
import serial
print " OK"
print "apsw",
import apsw
ver="3.5.9-r2"
if apsw.apswversion()!=ver:
raise Exception("Should be apsw version %s - you have %s" % (ver, apsw.apswversion()))
print " OK"
print "sqlite",
ver="3.6.1"
if apsw.sqlitelibversion()!=ver:
raise Exception("Should be sqlite version %s - you have %s" % (ver, apsw.sqlitelibversion()))
print " OK"
print "jaro/winkler string matcher",
import native.strings.jarow
print " OK"
# bsddb (Linux only, for evolution)
if sys.platform=="linux2":
print "bsddb ",
import bsddb
print " OK"
# win32com.shell - See http://starship.python.net/crew/theller/moin.cgi/WinShell
if sys.platform=='win32':
import py2exe.mf as modulefinder # in py2exe < 0.6.4 use "import modulefinder"
import win32com
for p in win32com.__path__[1:]:
modulefinder.AddPackagePath("win32com", p)
for extra in ["win32com.shell"]: #,"win32com.mapi"
__import__(extra)
m = sys.modules[extra]
for p in m.__path__[1:]:
modulefinder.AddPackagePath(extra, p)
print "=== All checks out ==="
def __init__(self, db_filename):
self._logger = logging.getLogger(__name__)
self._logger.debug('APSW version ' + apsw.apswversion())
self._logger.debug('SQLite version ' + apsw.sqlitelibversion())
self._conn = None
self._connect_to_db(db_filename)
import os, sys, time
import apsw
###
### Check we have the expected version of apsw and sqlite
###
#@@CAPTURE
print " Using APSW file",apsw.__file__ # from the extension module
print " APSW version",apsw.apswversion() # from the extension module
print " SQLite lib version",apsw.sqlitelibversion() # from the sqlite library code
print "SQLite header version",apsw.SQLITE_VERSION_NUMBER # from the sqlite header file at compile time
#@@ENDCAPTURE
###
### Opening/creating database
###
connection=apsw.Connection("dbfile")
cursor=connection.cursor()
###
### simple statement @@ example-cursor
###
cursor.execute("create table foo(x,y,z)")
###
### using different types
###
def outputData(diter, schema, connection, *args, **formatArgs):
### Parameter handling ###
where=None
if len(args)>0:
where=args[0]
elif 'file' in formatArgs:
where=formatArgs['file']
else:
raise functions.OperatorError(__name__.rsplit('.')[-1],"No destination provided")
if 'file' in formatArgs:
del formatArgs['file']
if 'mode' not in formatArgs:
formatArgs['mode']=autotype(where, {'csv':'csv', 'tsv':'tsv', 'xls':'tsv', 'db':'db', 'json':'json'})
if 'header' not in formatArgs:
header=False
else:
header=formatArgs['header']
del formatArgs['header']
if 'compression' not in formatArgs:
formatArgs['compression']=False
if 'compressiontype' not in formatArgs:
formatArgs['compressiontype']='gz'
append=False
if 'append' in formatArgs:
append=formatArgs['append']
del formatArgs['append']
type2ext={'csv':'csv', 'tsv':'xls', 'plain':'txt', 'db':'db', 'json':'json'}
where=autoext(where, formatArgs['mode'], type2ext)
filename, ext=os.path.splitext(os.path.basename(where))
fullpath=os.path.split(where)[0]
if not (formatArgs['mode'] == 'db' or (formatArgs['mode']=='json' and 'split' in formatArgs)):
fileIter=getoutput(where,append,formatArgs['compression'],formatArgs['compressiontype'])
del formatArgs['compressiontype']
del formatArgs['compression']
try:
if formatArgs['mode']=='json':
del formatArgs['mode']
import json
je = json.JSONEncoder(separators = (',',':'), ensure_ascii = True, check_circular = False).encode
if 'split' in formatArgs:
def cjs():
unikey = unicode(key)
t=open(os.path.join(fullpath, filename+'.'+unikey+ext), 'w')
print >> t, je( {'schema':schema[1:]} )
splitkeys[unikey]=t
jsfiles[key]=t
# Case for number as key
if unikey != key:
splitkeys[key] = splitkeys[unikey]
return splitkeys[key]
jsfiles = {}
splitkeys=defaultdict(cjs)
for row in diter:
key=row[0]
print >> splitkeys[key], je(row[1:])
# Create other parts
maxparts = 1
try:
maxparts = int(formatArgs['split'])
except ValueError:
maxparts = 1
if maxparts > 1:
for i in xrange(0, maxparts):
if i not in splitkeys:
key = i
tmp = splitkeys[key]
for f in jsfiles.values():
if f != None:
f.close()
else:
fileIter.write( je( {'schema':schema} ) + '\n')
for row in diter:
print >> fileIter, je(row)
elif formatArgs['mode']=='csv':
del formatArgs['mode']
csvprinter=writer(fileIter,'excel',**formatArgs)
if header:
csvprinter.writerow([h[0] for h in schema])
for row in diter:
csvprinter.writerow(row)
elif formatArgs['mode']=='tsv':
del formatArgs['mode']
csvprinter=writer(fileIter,'excel-tab',**formatArgs)
if header:
csvprinter.writerow([h[0] for h in schema])
for row in diter:
csvprinter.writerow([x.replace('\t',' ') if type(x)==str or type(x)==unicode else x for x in row])
elif formatArgs['mode']=='gtable':
vtoutpugtformat(fileIter,diter,simplejson=False)
elif formatArgs['mode']=='gjson':
vtoutpugtformat(fileIter,diter,simplejson=True)
elif formatArgs['mode']=='html':
raise functions.OperatorError(__name__.rsplit('.')[-1],"HTML format not available yet")
elif formatArgs['mode']=='plain':
for row in diter:
fileIter.write(((''.join([unicode(x) for x in row]))+'\n').encode('utf-8'))
elif formatArgs['mode']=='db':
def createdb(where, tname, schema, page_size=16384):
c=apsw.Connection(where)
cursor=c.cursor()
list(cursor.execute('pragma page_size='+str(page_size)+';pragma cache_size=-1000;pragma legacy_file_format=false;pragma synchronous=0;pragma journal_mode=OFF;PRAGMA locking_mode = EXCLUSIVE'))
create_schema='create table '+tname+' ('
create_schema+='`'+unicode(schema[0][0])+'`'+ (' '+unicode(schema[0][1]) if schema[0][1]!=None else '')
for colname, coltype in schema[1:]:
create_schema+=',`'+unicode(colname)+'`'+ (' '+unicode(coltype) if coltype!=None else '')
create_schema+='); begin exclusive;'
list(cursor.execute(create_schema))
insertquery="insert into "+tname+' values('+','.join(['?']*len(schema))+')'
return c, cursor, insertquery
if 'pagesize' in formatArgs:
page_size=int(formatArgs['pagesize'])
else:
page_size=list(connection.cursor().execute('pragma page_size'))[0][0]
tablename=filename
if 'tablename' in formatArgs:
tablename=formatArgs['tablename']
if 'split' in formatArgs:
maxparts = 0
try:
maxparts = int(formatArgs['split'])
except ValueError:
maxparts = 0
# If not split parts is defined
if maxparts == 0:
ns = lambda x:x
def cdb():
unikey = unicode(key)
t=createdb(os.path.join(fullpath, filename+'.'+unikey+ext), tablename, schema[1:], page_size)
splitkeys[unikey]=t[1].execute
ns.insertqueryw = t[2]
dbcon[key]=t[0], t[1]
# Case for number as key
if unikey != key:
splitkeys[key] = splitkeys[unikey]
return splitkeys[key]
dbcon = {}
splitkeys=defaultdict(cdb)
for row in diter:
key=row[0]
splitkeys[key](ns.insertqueryw, row[1:])
for c, cursor in dbcon.values():
if c != None:
cursor.execute('commit')
c.close()
# Splitparts defined
else:
cursors = []
dbcon = []
if "MSPW" in apsw.apswversion():
prepedqueries = []
for i in xrange(0,maxparts):
t=createdb(os.path.join(fullpath, filename+'.'+str(i)+ext), tablename, schema[1:], page_size)
cursors.append(t[1].executedirect)
prepedqueries.append(t[1].prepare(t[2]))
dbcon.append((t[0], t[1]))
for row in diter:
row0 = row[0]
cursors[row0](prepedqueries[row0], row[1:])
else:
for i in xrange(0,maxparts):
t=createdb(os.path.join(fullpath, filename+'.'+str(i)+ext), tablename, schema[1:], page_size)
cursors.append(t[1].execute)
dbcon.append((t[0], t[1]))
insertqueryw = t[2]
for row in diter:
cursors[row[0]](insertqueryw, row[1:])
for c, cursor in dbcon:
if c != None:
cursor.execute('commit')
c.close()
else:
# Write to db without split
c, cursor, insertquery=createdb(where, tablename, schema, page_size)
cursor.executemany(insertquery, diter)
list(cursor.execute('commit'))
c.close()
else:
raise functions.OperatorError(__name__.rsplit('.')[-1],"Unknown mode value")
except StopIteration,e:
pass
a few things to know; read the comment(s) below for details.'''
'''The current state of things as we approach version 3 is not bad at all. We
still need to do some work on stability and flexibility, and there's still a
lot of duplicated code in this file (db.py,) but overall we're making good
progress. The cache works fine, and UDP works as expected. Some work should
be put in on simplifying and strengthening the code in here, since this is
the part of the program that does the most work with the least clarity.'''
import os.path
import apsw
import config
import udp
version = "Database:\n APSW version: " + apsw.apswversion(
) + "\n SQLite version: " + apsw.sqlitelibversion() + "\n"
# Do we need to call Connection.close()?
# View this next line monospace...
def doclose(con):
# If APSW's version's third major number is greater than 8...
if int(apsw.apswversion().split('.')[2].split('-')[0]) >= 8:
con.close()
conf = config.config()
db = os.path.normpath(os.path.expanduser("~") + "/.oadb/anime.db")
data = {
t = config[table]
v = t.get(key,None)
t[key] = value
return v
except KeyError:
pass
db.createscalarfunction(name,getter,2)
db.createscalarfunction(name,setter,3)
return mod
__all__ = ['WebQueryModule','attach']
if __name__ == '__main__':
import sys, re
shell = apsw.Shell()
mod = attach(shell.db,'webquery')
loaded = []
for fn in sys.argv[1:]:
tbl = re.sub(r'\W','_',fn)
if tbl.endswith('_yml'):
tbl = tbl[:-4]
sql = 'CREATE VIRTUAL TABLE %s USING webquery(%s);'%(tbl,fn)
try:
shell.process_sql(sql)
loaded.append('> '+sql+'\r\n')
except KeyboardInterrupt:
raise
except:
pass
shell.cmdloop(intro=('SQLite version %s (APSW %s)\r\nEnter ".help" for instructions\r\nEnter SQL statements terminated with a ";"\r\n'%(apsw.sqlitelibversion(),apsw.apswversion()))+''.join(loaded))
# Python 2.6
from lib.collections26 import OrderedDict
try:
from inspect import isgeneratorfunction
except ImportError:
# Python < 2.6
def isgeneratorfunction(obj):
return bool((inspect.isfunction(object) or inspect.ismethod(object))
and obj.func_code.co_flags & CO_GENERATOR)
sys.setcheckinterval(1000)
sqlite_version = apsw.sqlitelibversion()
apsw_version = apsw.apswversion()
VTCREATE = 'create virtual table temp.'
SQLITEAFTER3711 = False
SQLITEAFTER380 = False
sqlite_version_split = [int(x) for x in sqlite_version.split('.')]
if sqlite_version_split[0:3] >= [3, 8, 0]:
SQLITEAFTER380 = True
try:
if sqlite_version_split[0:3] >= [3, 7, 11]:
VTCREATE = 'create virtual table if not exists temp.'
SQLITEAFTER3711 = True
except Exception, e:
VTCREATE = 'create virtual table if not exists temp.'
def doclose(con):
# If APSW's version's third major number is greater than 8...
if int(apsw.apswversion().split('.')[2].split('-')[0]) >= 8:
con.close()
def test_apsw():
import apsw
log.debug('apsw: available (v%s) [sqlite: %s]', apsw.apswversion(),
apsw.SQLITE_VERSION_NUMBER)
dotcompletions = ['.help ', '.colnums', '.schema ', '.functions ', '.tables', '.explain ', '.vacuum', '.queryplan ']
allfuncs = functions.functions['vtable'].keys() + functions.functions['row'].keys() + functions.functions[
'aggregate'].keys()
alltables = []
alltablescompl = []
updated_tables = set()
update_tablelist()
lastcols = []
newcols = []
colscompl = []
# Intro Message
if not pipedinput:
print mtermdetails
print "running on Python: " + '.'.join([str(x) for x in sys.version_info[
0:3]]) + ', APSW: ' + apsw.apswversion() + ', SQLite: ' + apsw.sqlitelibversion(),
try:
sys.stdout.write(", madIS: " + functions.VERSION + '\n')
except:
print
print intromessage
number_of_kb_exceptions = 0
while True:
statement = raw_input_no_history("mterm> ")
if statement == None:
number_of_kb_exceptions += 1
print
if number_of_kb_exceptions < 2:
continue
else:
sqlandmtermstatements=['select ', 'create ', 'where ', 'table ', 'group by ', 'drop ', 'order by ', 'index ', 'from ', 'alter ', 'limit ', 'delete ', '..',
"attach database '", 'detach database ', 'distinct', 'exists ']
dotcompletions=['.help ', '.colnums', '.schema ', '.functions ', '.tables', '.quote', '.explain ', '.vacuum', '.quit']
allfuncs=functions.functions['vtable'].keys()+functions.functions['row'].keys()+functions.functions['aggregate'].keys()
alltables=[]
alltablescompl=[]
updated_tables=set()
update_tablelist()
lastcols=[]
newcols=[]
colscompl=[]
#Intro Message
if not pipedinput:
print mtermdetails
print "running on Python: "+'.'.join([str(x) for x in sys.version_info[0:3]])+', APSW: '+apsw.apswversion()+', SQLite: '+apsw.sqlitelibversion(),
try:
sys.stdout.write(", madIS: "+functions.VERSION+'\n')
except:
print
print intromessage
number_of_kb_exceptions=0
while True:
statement = raw_input_no_history("mterm> ")
if statement==None:
number_of_kb_exceptions+=1
print
if number_of_kb_exceptions<2:
continue
else:
def printAPSWinfo():
print " Using APSW file",apsw.__file__ # from the extension module
print " APSW version",apsw.apswversion() # from the extension module
print " SQLite lib version",apsw.sqlitelibversion() # from the sqlite library code
print "SQLite header version",apsw.SQLITE_VERSION_NUMBER # from the sqlite header file at compile time
allfuncs = functions.functions['vtable'].keys() + functions.functions[
'row'].keys() + functions.functions['aggregate'].keys()
alltables = []
alltablescompl = []
updated_tables = set()
update_tablelist()
lastcols = []
newcols = []
colscompl = []
#Intro Message
if not pipedinput:
print mtermdetails
print "running on Python: " + '.'.join(
[str(x)
for x in sys.version_info[0:3]]) + ', APSW: ' + apsw.apswversion(
) + ', SQLite: ' + apsw.sqlitelibversion(),
try:
sys.stdout.write(", madIS: " + functions.VERSION + '\n')
except:
print
print intromessage
number_of_kb_exceptions = 0
while True:
statement = raw_input_no_history("mterm> ")
if statement == None:
number_of_kb_exceptions += 1
print
if number_of_kb_exceptions < 2:
continue
else:
def sanitycheck():
"Check all dependencies are present and at the correct version"
print "=== Sanity check ==="
print "python version",
if sys.version_info[:2]!=(2,3):
raise Exception("Should be Python 2.3 - this is "+sys.version)
print " OK"
print "wxPython version",
import wx
if wx.VERSION[:4]!=(2,6,2,1):
raise Exception("Should be wxPython 2.6.2.1. This is "+`wx.VERSION`)
print " OK"
print "wxPython is unicode build",
if not wx.USE_UNICODE:
raise Exception("You need a unicode build of wxPython")
print " OK"
if sys.platform!='win32':
print "native.usb",
import native.usb
print " OK"
print "pycrypto version",
expect='2.0.1'
import Crypto
if Crypto.__version__!=expect:
raise Exception("Should be %s version of pycrypto - you have %s" % (expect, Crypto.__version__))
print " OK"
print "paramiko version",
expect='1.4 (oddish)'
import paramiko
if paramiko.__version__!=expect:
raise Exception("Should be %s version of paramiko - you have %s" % (expect, paramiko.__version__))
print " OK"
print "bitfling",
import bitfling
print " OK"
print "pyserial",
import serial
print " OK"
print "apsw",
import apsw
ver="3.2.7-r1"
if apsw.apswversion()!=ver:
raise Exception("Should be apsw version %s - you have %s" % (ver, apsw.apswversion()))
print " OK"
print "sqlite",
ver="3.2.7"
if apsw.sqlitelibversion()!=ver:
raise Exception("Should be sqlite version %s - you have %s" % (ver, apsw.sqlitelibversion()))
print " OK"
print "jaro/winkler string matcher",
import native.strings.jarow
print " OK"
if sys.platform=="linux2":
print "bsddb ",
import bsddb
print " OK"
if sys.platform=='win32':
import py2exe.mf as modulefinder
import win32com
for p in win32com.__path__[1:]:
modulefinder.AddPackagePath("win32com", p)
for extra in ["win32com.shell"]:
__import__(extra)
m = sys.modules[extra]
for p in m.__path__[1:]:
modulefinder.AddPackagePath(extra, p)
print "=== All checks out ==="
def __init__(self, cons_list, func_list, struc_list, apps_list):
print "Using APSW file",apsw.__file__ # from the extension module
print "APSW version",apsw.apswversion() # from the extension module
print "SQLite version",apsw.sqlitelibversion() # from the sqlite library code
###
### Opening/creating database, initialize database
###
self.apsw_version = apsw.apswversion()
self.release_number = self.apsw_version[4:6]
self.db_path = os.path.join(os.environ['PWD'],'db')
#self.confReader = ConfReader('sockets_analysis.conf')
#self.functions = self.confReader.getItems('functions')
#self.structures = self.confReader.getItems('structures')
self.functions = func_list
self.structures = struc_list
self.constants = cons_list
function_temp = ""
structure_temp = ""
constant_temp = ""
for constant in self.constants:
constant_temp = constant_temp + constant[0].strip() + " int,"
for function in self.functions:
function_temp = function_temp + function[0].strip() + " int,"
i = 0
len_item = len(self.structures) # length of items
for structure in self.structures:
if i < len_item - 1:
structure_temp = structure_temp + structure[0].strip() + " int,"
else:
structure_temp = structure_temp + structure[0].strip() + " int"
i = i + 1
creat_table = "CREATE TABLE socket_statistic (name varchar PRIMARY KEY, " + constant_temp + function_temp + structure_temp + ")"
creat_sum_table = "CREATE TABLE socket_statistic_sum (socket_api_name varchar PRIMARY KEY , sum_number int)"
print creat_table
print creat_sum_table
#print creat_table
if os.path.exists(self.db_path):
print "database path existing......"
#print "delete the existing", self.db_path
#shutil.rmtree(self.db_path) #Removes directories recursively
#pass
else:
print "create the db directory"
os.mkdir('db')
database_file = os.path.join(self.db_path, 'socket_analysis_data_sos.db')
self.connection=apsw.Connection(database_file)
self.cursor=self.connection.cursor()
"""
self.cursor.execute(creat_table)
"""
try:
self.cursor.execute(creat_table)
except:
print "socket_statistic table is already there or something wrong with creating DB!!!"
try:
self.cursor.execute(creat_sum_table)
except:
print "socket_statistic_sum table is already there or something wrong with creating DB!!!"
'''The current state of things as we approach version 3 is not bad at all. We
still need to do some work on stability and flexibility, and there's still a
lot of duplicated code in this file (db.py,) but overall we're making good
progress. The cache works fine, and UDP works as expected. Some work should
be put in on simplifying and strengthening the code in here, since this is
the part of the program that does the most work with the least clarity.'''
import os.path
import apsw
import config
import udp
version = "Database:\n APSW version: " + apsw.apswversion() + "\n SQLite version: " + apsw.sqlitelibversion() + "\n"
# Do we need to call Connection.close()?
# View this next line monospace...
def doclose(con):
# If APSW's version's third major number is greater than 8...
if int(apsw.apswversion().split('.')[2].split('-')[0]) >= 8:
con.close()
conf = config.config()
db = os.path.normpath(os.path.expanduser("~") + "/.oadb/anime.db")
data = {
"anime":
["aid", "eps", "epcount", "spcount", "arating", "avotes", "tmprating", "tmpvotes", "average", "ratings", "year", "type", "aromaji", "akanji", "aenglish", "aother", "ashort", "synonyms", "cats"],
<td>DataTables</td>
<td>v1.10.24</td>
<td>
<a href="https://datatables.net/">
https://datatables.net
</a>
<td>
</tr>
</table>
""".format(version=KRAIT_VERSION,
build=KRAIT_BUILD,
python=sys.version.split()[0],
pyside=PySide6.__version__,
stria=stria.version(),
pyfastx=pyfastx.version(),
apsw=apsw.apswversion(),
sqlite=apsw.sqlitelibversion(),
primerpy=primer3.__version__)
#default parameter and type for krait
KRAIT_PARAMETERS = {
'SSR/mono': (12, int),
'SSR/di': (7, int),
'SSR/tri': (5, int),
'SSR/tetra': (4, int),
'SSR/penta': (4, int),
'SSR/hexa': (4, int),
'CSSR/dmax': (10, int),
'VNTR/minmotif': (7, int),
'VNTR/maxmotif': (30, int),
'VNTR/minrep': (3, int),
import os, sys, time
import apsw
print "using APSW file: ", apsw.__file__
print "APSW version: ", apsw.apswversion()
print "SQLite version: ", apsw.sqlitelibversion()
print "SQLite header version: ", apsw.SQLITE_VERSION_NUMBER
connection = apsw.Connection("yogeshmk.db")
cursor = connection.cursor()
cursor.execute("create table tt (a, b, c)")
cursor.execute("insert into tt values (?, ?, ?)", (1, 1.1, None)) # int, float, NULL
cursor.execute("insert into tt (a) values(?)", ("abc",)) # single element tuple
cursor.execute("insert into tt (a) values(?)", (buffer("abc\xff\xfe"), ))
# multiple statements
cursor.execute("delete from tt; insert into tt values(1,2,3); create table bar(a,b,c) ; insert into tt values(4, 'five', 6.0)")
# iterator
for a, b, c in cursor.execute("select a, b, c from tt"):
print cursor.getdescription() # show column names and datatypes
print a, b, c
cursor.execute("insert into tt (:aplha, :beta, :gamma)", {'beta': 2, 'aplha':1, 'gamma':3})
except ImportError:
# Python 2.6
from lib.collections26 import OrderedDict
try:
from inspect import isgeneratorfunction
except ImportError:
# Python < 2.6
def isgeneratorfunction(obj):
return bool((inspect.isfunction(object) or inspect.ismethod(object)) and
obj.func_code.co_flags & CO_GENERATOR)
sys.setcheckinterval(1000)
sqlite_version = apsw.sqlitelibversion()
apsw_version = apsw.apswversion()
VTCREATE = 'create virtual table temp.'
SQLITEAFTER3711 = False
SQLITEAFTER380 = False
sqlite_version_split = [int(x) for x in sqlite_version.split('.')]
if sqlite_version_split[0:3] >= [3, 8, 0]:
SQLITEAFTER380 = True
try:
if sqlite_version_split[0:3] >= [3, 7, 11]:
VTCREATE = 'create virtual table if not exists temp.'
SQLITEAFTER3711 = True
except Exception, e:
VTCREATE = 'create virtual table if not exists temp.'
def doclose(con):
# If APSW's version's third major number is greater than 8...
if int(apsw.apswversion().split('.')[2].split('-')[0]) >= 8:
con.close()
def sanitycheck():
"Check all dependencies are present and at the correct version"
print "=== Sanity check ==="
print "svn location",
if not "$HeadURL$".split(
":",
1)[1].strip().startswith("https://bitpim.svn.sourceforge.net"):
raise Exception(
"Needs to be checked out from https://bitpim.svn.sourceforge.net")
print " OK"
print "python version",
if sys.version_info[:2] != (2, 5):
raise Exception("Should be Python 2.5 - this is " + sys.version)
print " OK"
print "wxPython version",
import wx
if wx.VERSION[:4] != (2, 8, 8, 1):
raise Exception("Should be wxPython 2.8.8.1. This is " +
` wx.VERSION `)
print " OK"
print "wxPython is unicode build",
if not wx.USE_UNICODE:
raise Exception("You need a unicode build of wxPython")
print " OK"
if sys.platform != 'win32':
print "native.usb",
import native.usb
print " OK"
print "pycrypto version",
expect = '2.0.1'
import Crypto
if Crypto.__version__ != expect:
raise Exception("Should be %s version of pycrypto - you have %s" %
(expect, Crypto.__version__))
print " OK"
print "paramiko version",
expect = '1.7.4 (Desmond)'
import paramiko
if paramiko.__version__ != expect:
raise Exception("Should be %s version of paramiko - you have %s" %
(expect, paramiko.__version__))
print " OK"
print "bitfling",
import bitfling
print " OK"
print "pyserial",
import serial
print " OK"
print "apsw",
import apsw
ver = "3.5.9-r2"
if apsw.apswversion() != ver:
raise Exception("Should be apsw version %s - you have %s" %
(ver, apsw.apswversion()))
print " OK"
print "sqlite",
ver = "3.6.1"
if apsw.sqlitelibversion() != ver:
raise Exception("Should be sqlite version %s - you have %s" %
(ver, apsw.sqlitelibversion()))
print " OK"
print "jaro/winkler string matcher",
import native.strings.jarow
print " OK"
# bsddb (Linux only, for evolution)
if sys.platform == "linux2":
print "bsddb ",
import bsddb
print " OK"
# win32com.shell - See http://starship.python.net/crew/theller/moin.cgi/WinShell
if sys.platform == 'win32':
import py2exe.mf as modulefinder # in py2exe < 0.6.4 use "import modulefinder"
import win32com
for p in win32com.__path__[1:]:
modulefinder.AddPackagePath("win32com", p)
for extra in ["win32com.shell"]: #,"win32com.mapi"
__import__(extra)
m = sys.modules[extra]
for p in m.__path__[1:]:
modulefinder.AddPackagePath(extra, p)
print "=== All checks out ==="
def doit():
random.seed(0)
options.tests=[t.strip() for t in options.tests.split(",")]
write(" Python %s %s\n" % (sys.executable, str(sys.version_info)))
write(" Scale %d\n" % (options.scale,))
write(" Database %s\n" % (options.database,))
write(" Tests %s\n" % (", ".join(options.tests),))
write(" Iterations %d\n" % (options.iterations,))
write("Statement Cache %d\n" % (options.scsize,))
write("\n")
if options.apsw:
import apsw
write(" Testing with APSW file "+apsw.__file__+"\n")
write(" APSW version "+apsw.apswversion()+"\n")
write(" SQLite lib version "+apsw.sqlitelibversion()+"\n")
write(" SQLite headers version "+str(apsw.SQLITE_VERSION_NUMBER)+"\n\n")
def apsw_setup(dbfile):
con=apsw.Connection(dbfile, statementcachesize=options.scsize)
con.createscalarfunction("number_name", number_name, 1)
return con
if options.pysqlite:
try:
from pysqlite2 import dbapi2 as pysqlite
except ImportError:
import sqlite3 as pysqlite
write("Testing with pysqlite file "+pysqlite.__file__+"\n")
write(" pysqlite version "+pysqlite.version+"\n")
write(" SQLite version "+pysqlite.sqlite_version+"\n\n")
def pysqlite_setup(dbfile):
con=pysqlite.connect(dbfile, isolation_level=None, cached_statements=options.scsize)
con.create_function("number_name", 1, number_name)
return con
ones=("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
"ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen",
"eighteen", "nineteen")
tens=("", "ten", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety")
others=("thousand", "hundred", "zero")
def _number_name(n):
if n>=1000:
txt="%s %s" % (_number_name(int(n/1000)), others[0])
n=n%1000
else:
txt=""
if n>=100:
txt=txt+" "+ones[int(n/100)]+" "+others[1]
n=n%100
if n>=20:
txt=txt+" "+tens[int(n/10)]
n=n%10
if n>0:
txt=txt+" "+ones[n]
txt=txt.strip()
if txt=="":
txt=others[2]
return txt
def unicodify(text):
if options.unicode and len(text):
newt=[]
c=options.unicode/100.0
for t in text:
if random.random()>c:
newt.append(t)
continue
while True:
t=random.randint(0xa1, maxuni)
# we don't want the surrogate range or apostrophe
if t<0xd800 or t>0xdfff: break
newt.append(unichr(t))
text="".join(newt)
return text
if options.unicode:
ones=tuple([unicodify(s) for s in ones])
tens=tuple([unicodify(s) for s in tens])
others=tuple([unicodify(s) for s in others])
def number_name(n):
text=_number_name(n)
if options.size:
text=text*int(random.randint(0, options.size)/len(text))
return text
def getlines(scale=50, bindings=False):
random.seed(0)
# RogerB added two pragmas so that only memory is used. This means that the
# vagaries of disk access times don't alter the results
# database schema
for i in """PRAGMA page_size=1024;
PRAGMA cache_size=8192;
PRAGMA locking_mode=EXCLUSIVE;
PRAGMA journal_mode = OFF;
PRAGMA temp_store = MEMORY;
CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT);
CREATE TABLE t2(a INTEGER, b INTEGER, c TEXT);
CREATE INDEX i2a ON t2(a);
CREATE INDEX i2b ON t2(b);
SELECT name FROM sqlite_master ORDER BY 1""".split(";"):
yield (i,)
# 50,000 inserts on an unindexed table
yield ("BEGIN",)
for i in xrange(1,scale*10000+1):
r=random.randint(0,500000)
if bindings:
yield ("INSERT INTO t1 VALUES(:1, :2, number_name(:2))", (i, r))
else:
yield ("INSERT INTO t1 VALUES(%d, %d, '%s')" % (i, r, number_name(r)),)
yield ("COMMIT",)
# 50,000 inserts on an indexed table
t1c_list=[]
yield ("BEGIN",)
for i in xrange(1,scale*10000+1):
r=random.randint(0,500000)
x=number_name(r)
t1c_list.append(x)
if bindings:
yield ("INSERT INTO t2 VALUES(:1, :2, number_name(:2))", (i, r))
else:
yield ("INSERT INTO t2 VALUES(%d, %d, '%s')" % (i, r, x),)
yield ("COMMIT",)
# 50 SELECTs on an integer comparison. There is no index so
# a full table scan is required.
for i in xrange(scale):
yield ("SELECT count(*), avg(b) FROM t1 WHERE b>=%d AND b<%d" % (i*100, (i+10)*100),)
# 50 SELECTs on an LIKE comparison. There is no index so a full
# table scan is required.
for i in xrange(scale):
yield ("SELECT count(*), avg(b) FROM t1 WHERE c LIKE '%%%s%%'" % (number_name(i),),)
# Create indices
yield ("BEGIN",)
for i in """CREATE INDEX i1a ON t1(a);
CREATE INDEX i1b ON t1(b);
CREATE INDEX i1c ON t1(c);""".split(";"):
yield (i,)
yield ("COMMIT",)
# 5000 SELECTs on an integer comparison where the integer is
# indexed.
for i in xrange(scale*100):
yield ("SELECT count(*), avg(b) FROM t1 WHERE b>=%d AND b<%d" % (i*100, (i+10)*100),)
# 100000 random SELECTs against rowid.
for i in xrange(1,scale*2000+1):
yield ("SELECT c FROM t1 WHERE rowid=%d" % (1+random.randint(0,50000),),)
# 100000 random SELECTs against a unique indexed column.
for i in xrange(1,scale*2000+1):
yield ("SELECT c FROM t1 WHERE a=%d" % (1+random.randint(0,50000),),)
# 50000 random SELECTs against an indexed column text column
for i in xrange(scale*1000):
if bindings:
yield ("SELECT c FROM t1 WHERE c=?", (random.choice(t1c_list),),)
else:
yield ("SELECT c FROM t1 WHERE c='%s'" % (random.choice(t1c_list),),)
# Vacuum
if options.database!=":memory:":
# opens a disk file
yield ("VACUUM",)
# 5000 updates of ranges where the field being compared is indexed.
yield ("BEGIN",)
for i in xrange(scale*100):
yield ("UPDATE t1 SET b=b*2 WHERE a>=%d AND a<%d" % (i*2, (i+1)*2),)
yield ("COMMIT",)
# 50000 single-row updates. An index is used to find the row quickly.
yield ("BEGIN",)
for i in xrange(scale*1000):
if bindings:
yield ("UPDATE t1 SET b=? WHERE a=%d" % (i,), (random.randint(0,500000),))
else:
yield ("UPDATE t1 SET b=%d WHERE a=%d" % (random.randint(0,500000), i),)
yield ("COMMIT",)
# 1 big text update that touches every row in the table.
yield ("UPDATE t1 SET c=a",)
# Many individual text updates. Each row in the table is
# touched through an index.
yield ("BEGIN",)
for i in xrange(1,scale*1000+1):
if bindings:
yield ("UPDATE t1 SET c=? WHERE a=%d" % (i,), (number_name(random.randint(0,500000)),))
else:
yield ("UPDATE t1 SET c='%s' WHERE a=%d" % (number_name(random.randint(0,500000)),i),)
yield ("COMMIT",)
# Delete all content in a table.
yield ("DELETE FROM t1",)
# Copy one table into another
yield ("INSERT INTO t1 SELECT * FROM t2",)
# Delete all content in a table, one row at a time.
yield ("DELETE FROM t1 WHERE 1",)
# Refill the table yet again
yield ("INSERT INTO t1 SELECT * FROM t2",)
# Drop the table and recreate it without its indices.
yield ("BEGIN",)
yield ("DROP TABLE t1",)
yield ("CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT)",)
yield ("COMMIT",)
# Refill the table yet again. This copy should be faster because
# there are no indices to deal with.
yield ("INSERT INTO t1 SELECT * FROM t2",)
# The three following used "ORDER BY random()" but we can't do that
# as it causes each run to have different values, and hence different
# amounts of sorting that have to go on. The "random()" has been
# replaced by "c", the column that has the stringified number
# Select 20000 rows from the table at random.
yield ("SELECT rowid FROM t1 ORDER BY c LIMIT %d" % (scale*400,),)
# Delete 20000 random rows from the table.
yield (""" DELETE FROM t1 WHERE rowid IN
(SELECT rowid FROM t1 ORDER BY c LIMIT %d)""" % (scale*400,),)
yield ("SELECT count(*) FROM t1",)
# Delete 20000 more rows at random from the table.
yield ("""DELETE FROM t1 WHERE rowid IN
(SELECT rowid FROM t1 ORDER BY c LIMIT %d)""" % (scale*400,),)
yield ("SELECT count(*) FROM t1",)
# Do a correctness test first
if options.correctness:
write("Correctness test\n")
if 'bigstmt' in options.tests:
text=";\n".join([x[0] for x in getlines(scale=1)])+";"
if 'statements' in options.tests:
withbindings=[line for line in getlines(scale=1, bindings=True)]
if 'statements_nobindings' in options.tests:
withoutbindings=[line for line in getlines(scale=1, bindings=False)]
res={}
for driver in ('apsw', 'pysqlite'):
if not getattr(options, driver):
continue
for test in options.tests:
name=driver+"_"+test
write(name+'\t')
sys.stdout.flush()
if name=='pysqlite_bigstmt':
write('limited functionality (ignoring)\n')
continue
con=globals().get(driver+"_setup")(":memory:") # we always correctness test on memory
if test=='bigstmt':
cursor=con.cursor()
if driver=='apsw':
func=cursor.execute
else:
func=cursor.executescript
res[name]=[row for row in func(text)]
write(str(len(res[name]))+"\n")
continue
cursor=con.cursor()
if test=='statements':
sql=withbindings
elif test=='statements_nobindings':
sql=withoutbindings
l=[]
for s in sql:
for row in cursor.execute(*s):
l.append(row)
res[name]=l
write(str(len(res[name]))+"\n")
# All elements of res should be identical
elements=res.keys()
elements.sort()
for i in range(0,len(elements)-1):
write("%s == %s %s\n" % (elements[i], elements[i+1], res[elements[i]]==res[elements[i+1]]))
del res
text=None
withbindings=None
withoutbindings=None
if options.dump_filename or "bigstmt" in options.tests:
text=";\n".join([x[0] for x in getlines(scale=options.scale)])+";" # pysqlite requires final semicolon
if options.dump_filename:
open(options.dump_filename, "wt").write(text.encode("utf8"))
sys.exit(0)
if "statements" in options.tests:
withbindings=list(getlines(scale=options.scale, bindings=True))
if "statements_nobindings" in options.tests:
withoutbindings=list(getlines(scale=options.scale, bindings=False))
# Each test returns the amount of time taken. Note that we include
# the close time as well. Otherwise the numbers become a function of
# cache and other collection sizes as freeing members gets deferred to
# close time.
def apsw_bigstmt(con):
"APSW big statement"
try:
for row in con.cursor().execute(text): pass
except:
import pdb ; pdb.set_trace()
pass
def pysqlite_bigstmt(con):
"pysqlite big statement"
for row in con.executescript(text): pass
def apsw_statements(con, bindings=withbindings):
"APSW individual statements with bindings"
cursor=con.cursor()
for b in bindings:
for row in cursor.execute(*b): pass
def pysqlite_statements(con, bindings=withbindings):
"pysqlite individual statements with bindings"
cursor=con.cursor()
for b in bindings:
for row in cursor.execute(*b): pass
def apsw_statements_nobindings(con):
"APSW individual statements without bindings"
return apsw_statements(con, withoutbindings)
def pysqlite_statements_nobindings(con):
"pysqlite individual statements without bindings"
return pysqlite_statements(con, withoutbindings)
# Do the work
write("\nRunning tests - elapsed, CPU (results in seconds, lower is better)\n")
for i in range(options.iterations):
write("%d/%d\n" % (i+1, options.iterations))
for test in options.tests:
# funky stuff is to alternate order each round
for driver in ( ("apsw", "pysqlite"), ("pysqlite", "apsw"))[i%2]:
if getattr(options, driver):
name=driver+"_"+test
func=locals().get(name, None)
if not func:
sys.stderr.write("No such test "+name+"\n")
sys.exit(1)
if os.path.exists(options.database):
os.remove(options.database)
write("\t"+func.__name__+(" "*(40-len(func.__name__))))
sys.stdout.flush()
con=locals().get(driver+"_setup")(options.database)
gc.collect(2)
b4cpu=time.clock()
b4=time.time()
func(con)
con.close() # see note above as to why we include this in the timing
gc.collect(2)
after=time.time()
aftercpu=time.clock()
write("%0.3f %0.3f\n" % (after-b4, aftercpu-b4cpu))
# Cleanup if using valgrind
if options.apsw:
if hasattr(apsw, "_fini"):
# Cleans out buffer recycle cache
apsw._fini()
sqlandmtermstatements=['select ', 'create ', 'where ', 'table ', 'group by ', 'drop ', 'order by ', 'index ', 'from ', 'alter ', 'limit ', 'delete ', '..',
"attach database '", 'detach database ', 'distinct', 'exists ']
dotcompletions=['.help ', '.colnums', '.schema ', '.functions ', '.tables', '.quote', '.explain ', '.vacuum', '.quit']
allfuncs=functions.functions['vtable'].keys()+functions.functions['row'].keys()+functions.functions['aggregate'].keys()
alltables=[]
alltablescompl=[]
updated_tables=set()
update_tablelist()
lastcols=[]
newcols=[]
colscompl=[]
#Intro Message
if not pipedinput:
print mtermdetails
print "running on Python: "+'.'.join([str(x) for x in sys.version_info[0:3]])+', APSW: '+apsw.apswversion()+', SQLite: '+apsw.sqlitelibversion(),
try:
sys.stdout.write(", madIS: "+functions.VERSION+'\n')
except:
print
print intromessage
number_of_kb_exceptions=0
while True:
statement = raw_input_no_history("mterm> ")
if statement==None:
number_of_kb_exceptions+=1
print
if number_of_kb_exceptions<2:
continue
else:
import apsw
except ImportError:
# https://github.com/ghaering/pysqlite (https://docs.python.org/2/library/sqlite3.html) -- is C code...
# pypi.python.org/pypi/PyDbLite , www.pydblite.net/en/index.html -- pure python, but a tad different from sqlite3.
# from pydblite import sqlite # pydblite relies on built-in sqlite3 or pysqlite2...
try:
from .sqlite3_adhoc import apsw
print("chrome_extract: Using sqlite3_adhoc apsw replacement module...")
except ImportError as exc:
print("ImportError while importing sqlite3 apsw stand-in module:", exc)
#raise exc # Not fatal...
apsw = None
print("apsw module:", apsw)
if apsw:
print("apsw sqlite version:", apsw.sqlitelibversion())
print("apsw version:", apsw.apswversion())
try:
from Crypto.Cipher import AES
except ImportError:
#warnings.warn("Could not import Crypto.Cipher AES module.")
"""
PyCrypto (Crypto) alternatives: (https://github.com/dlitz/pycrypto)
pyOCB (github.com/kravietz/pyOCB) - could be used, but Chrome encrypts as CBC, not OCB.
AES-Python (https://github.com/bozhu/AES-Python)
--- another pure-python implementation. However, does not have an easy-to-use interface.
--- Not sure what modes are supported, seems quite ad-hoc. Like... "hey, let me try to implement AES..."
--- "only for AES-128"
---
def apsw_version():
"""Show apsw version."""
return apsw.apswversion()
def inext(v): # next value from iterator
return next(v) if py3 else v.next()
import os
import time
import apsw
###
### Check we have the expected version of apsw and sqlite
###
#@@CAPTURE
print(" Using APSW file", apsw.__file__) # from the extension module
print(" APSW version", apsw.apswversion()) # from the extension module
print(" SQLite lib version",
apsw.sqlitelibversion()) # from the sqlite library code
print(
"SQLite header version",
apsw.SQLITE_VERSION_NUMBER) # from the sqlite header file at compile time
#@@ENDCAPTURE
###
### Opening/creating database
###
connection = apsw.Connection("dbfile")
cursor = connection.cursor()
###
def test_apsw():
import apsw
log.debug('apsw: available (v%s) [sqlite: %s]', apsw.apswversion(), apsw.SQLITE_VERSION_NUMBER)
