def openDBFile(self, dbfile, FskHz, updateOnIdle=True):
self.queryDatabase = sqlite.sqlite3MDIO()
self.queryDatabase.openDB(dbfile)
# self update on idle flag
self.updateDataOnIdle = updateOnIdle
# Store the analysis algorithm
try:
self.analysisAlgorithm = str(
self.queryDatabase.readAnalysisInfo()['processingAlgorithm'])
except:
# If the database doesn't have information on the alogirthm type,
# default to adept2State
self.analysisAlgorithm = "adept2State"
try:
# Generate the query string based on the algorithm in the database
self.queryString = self.queryStringDict[self.analysisAlgorithm]
# Setup the fit function based on the algorithm
self.fitFuncHnd = self.fitFuncHndDict[self.analysisAlgorithm]
self.fitFuncArgs = self.fitFuncArgsDict[self.analysisAlgorithm]
self.stepFuncHnd = self.stepFuncHndDict[self.analysisAlgorithm]
self.stepFuncArgs = self.stepFuncArgsDict[self.analysisAlgorithm]
self.bdFuncArgs = self.blockDepthArgsDict[self.analysisAlgorithm]
except KeyError:
from mosaic.settings import __legacy_settings__
legacyAlgo = __legacy_settings__[self.analysisAlgorithm]
self.queryString = self.queryStringDict[legacyAlgo]
self.fitFuncHnd = self.fitFuncHndDict[legacyAlgo]
self.fitFuncArgs = self.fitFuncArgsDict[legacyAlgo]
self.stepFuncHnd = self.stepFuncHndDict[legacyAlgo]
self.stepFuncArgs = self.stepFuncArgsDict[legacyAlgo]
self.bdFuncArgs = self.blockDepthArgsDict[legacyAlgo]
# self.fitFuncHnd=None
# self.fitFuncArgs="[]"
# self.stepFuncHnd=None
# self.stepFuncArgs="[]"
self.FskHz = float(FskHz)
# print self.FskHz
self._updatequery()
self.update_graph()
self.eventIndexLineEdit.setText(str(self.eventIndex))
# Idle processing
QtCore.QObject.connect(self.idleTimer, QtCore.SIGNAL('timeout()'),
self.OnAppIdle)
def csv(self):
dbHnd=sqlite.sqlite3MDIO()
dbHnd.openDB(self.AnalysisDBFile)
self.responseDict["dbName"]=self.AnalysisDBFile.split(path_separator())[-1].split('.')[0]
self.responseDict["dbData"]=base64.b64encode(dbHnd.csvString(self.queryString))
return self.responseDict
def openDBFile(self, dbfile, FskHz, updateOnIdle=True):
self.queryDatabase=sqlite.sqlite3MDIO()
self.queryDatabase.openDB(dbfile)
# self update on idle flag
self.updateDataOnIdle=updateOnIdle
# Store the analysis algorithm
try:
self.analysisAlgorithm=str(self.queryDatabase.readAnalysisInfo()['processingAlgorithm'])
except:
# If the database doesn't have information on the alogirthm type,
# default to adept2State
self.analysisAlgorithm="adept2State"
try:
# Generate the query string based on the algorithm in the database
self.queryString=self.queryStringDict[self.analysisAlgorithm]
# Setup the fit function based on the algorithm
self.fitFuncHnd=self.fitFuncHndDict[self.analysisAlgorithm]
self.fitFuncArgs=self.fitFuncArgsDict[self.analysisAlgorithm]
self.stepFuncHnd=self.stepFuncHndDict[self.analysisAlgorithm]
self.stepFuncArgs=self.stepFuncArgsDict[self.analysisAlgorithm]
self.bdFuncArgs=self.blockDepthArgsDict[self.analysisAlgorithm]
except KeyError:
from mosaic.settings import __legacy_settings__
legacyAlgo=__legacy_settings__[self.analysisAlgorithm]
self.queryString=self.queryStringDict[legacyAlgo]
self.fitFuncHnd=self.fitFuncHndDict[legacyAlgo]
self.fitFuncArgs=self.fitFuncArgsDict[legacyAlgo]
self.stepFuncHnd=self.stepFuncHndDict[legacyAlgo]
self.stepFuncArgs=self.stepFuncArgsDict[legacyAlgo]
self.bdFuncArgs=self.blockDepthArgsDict[legacyAlgo]
# self.fitFuncHnd=None
# self.fitFuncArgs="[]"
# self.stepFuncHnd=None
# self.stepFuncArgs="[]"
self.FskHz=float(FskHz)
# print self.FskHz
self._updatequery()
self.update_graph()
self.eventIndexLineEdit.setText(str(self.eventIndex))
# Idle processing
QtCore.QObject.connect(self.idleTimer, QtCore.SIGNAL('timeout()'), self.OnAppIdle)
def dbColumnNames(self, filterRealList): self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile, timeout=3.0) if filterRealList: self.dbColumnsReady.emit( [ col[0] for col in zip( self.queryDatabase.mdColumnNames, self.queryDatabase.mdColumnTypes ) if col[1] != 'REAL_LIST' ] ) else: self.dbColumnsReady.emit(self.queryDatabase.mdColumnNames) self.queryDatabase.closeDB() self.finished.emit()
def __init__(self, trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings, settingsString):
"""
Initialize a new event segment object
"""
# Required arguments
self.trajDataObj=trajDataObj
self.eventProcHnd=eventProcHnd
# Reset function timer since esTimer is a class variable
partitionTimer.Reset()
self.settingsDict = eventPartitionSettings
self.eventProcSettingsDict = eventProcSettings
self.procTime=0.0
self.FsHz=self.trajDataObj.FsHz
self.DataLengthSec=self.trajDataObj.DataLengthSec
try:
self.writeEventTS=int(self.settingsDict.pop("writeEventTS",1))
self.parallelProc=int(self.settingsDict.pop("parallelProc",1))
self.reserveNCPU=int(self.settingsDict.pop("reserveNCPU",2))
except ValueError as err:
raise mosaic.commonExceptions.SettingsTypeError( err )
sys.stdout.flush()
self.tEventProcObj=self.eventProcHnd([], self.FsHz, eventstart=0,eventend=0, baselinestats=[ 0,0,0 ], algosettingsdict=self.eventProcSettingsDict.copy(), savets=False, absdatidx=0, datafileHnd=None )
self.mdioDBHnd=sqlite3MDIO.sqlite3MDIO()
self.mdioDBHnd.initDB(
dbPath=self.trajDataObj.datPath,
tableName='metadata',
colNames=(self.tEventProcObj.mdHeadings()),
colNames_t=(self.tEventProcObj.mdHeadingDataType())
)
self.mdioDBHnd.writeSettings(settingsString)
self.logger=mlog.mosaicLogging().getLogger(name=__name__, dbHnd=self.mdioDBHnd)
self.logger.debug(_d("Event Segment Initialization"))
self.logger.debug(_d("{0}", settingsString))
if self.trajDataObj.dataFilter:
self.fstring=type(self.trajDataObj.dataFilterObj).__name__
else:
self.fstring='None'
self._writeanalysisinfo()
if self.parallelProc:
self._setupparallel()
# Setup function timing
self.timingObj=mosaicTiming.mosaicTiming()
self._init(trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings)
def main(): args=parseCLArgs() db=sqlite3MDIO.sqlite3MDIO() db.openDB(args.dbname) if args.raw: return [ list(l) for l in db.rawQuery(args.query) ] else: return db.queryDB(args.query)
def query(dbname, query_str): """ Simple wrapper to perform a query on a MOSAIC database. """ db=sql.sqlite3MDIO() db.openDB(dbname) q=db.queryDB(query_str) db.closeDB() return q
def _dbInfo(dbname):
db = sqlite.sqlite3MDIO()
db.openDB(dbname)
info = db.readAnalysisInfo()
settings = db.readSettings()
db.closeDB()
return info, settings
def csv(self):
dbHnd = sqlite.sqlite3MDIO()
dbHnd.openDB(self.AnalysisDBFile)
self.responseDict["dbName"] = self.AnalysisDBFile.split(
path_separator())[-1].split('.')[0]
self.responseDict["dbData"] = base64.b64encode(
dbHnd.csvString(self.queryString))
return self.responseDict
def query(dbname, query_str):
"""
Simple wrapper to perform a query on a MOSAIC database.
"""
db = sql.sqlite3MDIO()
db.openDB(dbname)
q = db.queryDB(query_str)
db.closeDB()
return q
def main():
args = parseCLArgs()
db = sqlite3MDIO.sqlite3MDIO()
db.openDB(args.dbname)
if args.raw:
return [list(l) for l in db.rawQuery(args.query)]
else:
return db.queryDB(args.query)
def _dbInfo(dbname): db=sqlite.sqlite3MDIO() db.openDB(dbname) info=db.readAnalysisInfo() settings=db.readSettings() db.closeDB() return info, settings
def queryDB(self, q):
try:
self.queryDatabase = sqlite.sqlite3MDIO()
self.queryDatabase.openDB(self.dbFile)
self.resultsReady.emit(self.queryDatabase.queryDB(str(q)), "")
self.queryDatabase.closeDB()
except sqlite3.OperationalError, err:
self.queryDatabase.closeDB()
self.resultsReady.emit([], str(err))
def queryDB(self, q): try: self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile) self.resultsReady.emit(self.queryDatabase.queryDB(str(q)), "") self.queryDatabase.closeDB() except sqlite3.OperationalError, err: self.queryDatabase.closeDB() self.resultsReady.emit([], str(err))
def dbColumnNames(self, filterRealList):
self.queryDatabase = sqlite.sqlite3MDIO()
self.queryDatabase.openDB(self.dbFile, timeout=3.0)
if filterRealList:
self.dbColumnsReady.emit([
col[0] for col in zip(self.queryDatabase.mdColumnNames,
self.queryDatabase.mdColumnTypes)
if col[1] != 'REAL_LIST'
])
else:
self.dbColumnsReady.emit(self.queryDatabase.mdColumnNames)
self.queryDatabase.closeDB()
self.finished.emit()
def queryDB2(self, q1, q2): try: self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile) r1=self.queryDatabase.queryDB(str(q1)) r2=self.queryDatabase.queryDB(str(q2)) self.resultsReady2.emit(r1, r2, "") self.queryDatabase.closeDB() except sqlite3.OperationalError, err: self.queryDatabase.closeDB() self.resultsReady.emit([], str(err))
def queryDB2(self, q1, q2):
try:
self.queryDatabase = sqlite.sqlite3MDIO()
self.queryDatabase.openDB(self.dbFile)
r1 = self.queryDatabase.queryDB(str(q1))
r2 = self.queryDatabase.queryDB(str(q2))
self.resultsReady2.emit(r1, r2, "")
self.queryDatabase.closeDB()
except sqlite3.OperationalError, err:
self.queryDatabase.closeDB()
self.resultsReady.emit([], str(err))
def __init__(self, dbFile, qstr, bins, showContours):
self.AnalysisDBFile = dbFile
self.numBins = bins
dbHnd = sqlite.sqlite3MDIO()
dbHnd.openDB(self.AnalysisDBFile)
analysisInfo = dbHnd.readAnalysisInfo()
self.processingAlgorithm = analysisInfo['processingAlgorithm']
self.queryString = self._queryString(qstr)
if showContours:
self.plotType = "contour"
else:
self.plotType = "heatmap"
self.responseDict = {}
def __init__(self, dbFile, qstr, bins, showContours):
self.AnalysisDBFile = dbFile
self.numBins=bins
dbHnd=sqlite.sqlite3MDIO()
dbHnd.openDB(self.AnalysisDBFile)
analysisInfo=dbHnd.readAnalysisInfo()
self.processingAlgorithm=analysisInfo['processingAlgorithm']
self.queryString=self._queryString(qstr)
if showContours:
self.plotType="contour"
else:
self.plotType="heatmap"
self.responseDict={}
def executeSQL(self, q): try: # t1=time.time() self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile) # t1a=time.time() self.resultsReady.emit(self.queryDatabase.executeSQL(str(q)), "") # t2a=time.time() self.queryDatabase.closeDB() # t2=time.time() # print "query time = ", t2a-t1a # print "overhead = ", (t2-t1)-(t2a-t1a) # print "total time = ", t2-t1 except sqlite3.OperationalError, err: self.queryDatabase.closeDB() self.resultsReady.emit([], str(err))
def executeSQL(self, q):
try:
# t1=time.time()
self.queryDatabase = sqlite.sqlite3MDIO()
self.queryDatabase.openDB(self.dbFile)
# t1a=time.time()
self.resultsReady.emit(self.queryDatabase.executeSQL(str(q)), "")
# t2a=time.time()
self.queryDatabase.closeDB()
# t2=time.time()
# print "query time = ", t2a-t1a
# print "overhead = ", (t2-t1)-(t2a-t1a)
# print "total time = ", t2-t1
except sqlite3.OperationalError, err:
self.queryDatabase.closeDB()
self.resultsReady.emit([], str(err))
def openDBFile(self, dbfile, updateOnIdle=True):
"""
Open a specific database file.
"""
# Create an index to speed up queries
self._createDBIndex(dbfile)
self.qWorker = sqlworker.sqlQueryWorker(dbfile)
self.dbFile = dbfile
# Open a handle to the database
self.dbHnd = sqlite.sqlite3MDIO()
self.dbHnd.openDB(self.dbFile)
# set the length of the trajectory in sec.
self.trajLength = self.dbHnd.readAnalysisInfo()[-1]
# Connect signals and slots
self.qWorker.resultsReady2.connect(self.OnDataReady)
self.qWorker.moveToThread(self.qThread)
self.qWorker.finished.connect(self.qThread.quit)
self.qThread.start()
# reset elapsed time
self.analysisTime = 0.0
# reset wall time and analysis start time
self.wallTime = 0.0
self.startTime = time.time()
# self update on idle flag
self.updateDataOnIdle = updateOnIdle
# Query the DB
self._updatequery()
# Idle processing
QtCore.QObject.connect(self.idleTimer, QtCore.SIGNAL('timeout()'),
self.OnAppIdle)
def openDBFile(self, dbfile, updateOnIdle=True):
"""
Open a specific database file.
"""
# Create an index to speed up queries
self._createDBIndex(dbfile)
self.qWorker=sqlworker.sqlQueryWorker(dbfile)
self.dbFile=dbfile
# Open a handle to the database
self.dbHnd=sqlite.sqlite3MDIO()
self.dbHnd.openDB(self.dbFile)
# set the length of the trajectory in sec.
self.trajLength=self.dbHnd.readAnalysisInfo()[-1]
# Connect signals and slots
self.qWorker.resultsReady2.connect(self.OnDataReady)
self.qWorker.moveToThread(self.qThread)
self.qWorker.finished.connect(self.qThread.quit)
self.qThread.start()
# reset elapsed time
self.analysisTime=0.0
# reset wall time and analysis start time
self.wallTime=0.0
self.startTime=time.time()
# self update on idle flag
self.updateDataOnIdle=updateOnIdle
# Query the DB
self._updatequery()
# Idle processing
QtCore.QObject.connect(self.idleTimer, QtCore.SIGNAL('timeout()'), self.OnAppIdle)
def analysisStatistics(self):
statsDict = {}
s = self._eventStats()
statsDict['fractionNormal'] = s[0]
statsDict['fractionWarn'] = s[1]
statsDict['fractionError'] = s[2]
statsDict['nTotal'] = s[3]
c = self._caprate()
statsDict['captureRateMean'] = c[0]
statsDict['captureRateSigma'] = c[1]
t = self._timePerEvent()
statsDict['processTimePerEventMean'] = t[0]
statsDict['processTimePerEventSigma'] = t[1]
o = self._openChanCurrent()
statsDict['openChannelCurrentMean'] = o[0]
statsDict['openChannelCurrentSigma'] = o[1]
statsDict['analysisProgressPercent'] = self._analysisProgress()
dbHnd = sqlite.sqlite3MDIO()
dbHnd.openDB(self.analysisDB)
analysisInfo = dbHnd.readAnalysisInfo()
procName = analysisStatistics.processingAlgorithmName
partName = analysisStatistics.partitionAlgorithmName
statsDict['partitionAlgorithm'] = partName[
analysisInfo['partitionAlgorithm']]
statsDict['processingAlgorithm'] = procName[
analysisInfo['processingAlgorithm']]
statsDict['FskHz'] = analysisInfo['FsHz'] / 1000.
statsDict['dataType'] = analysisInfo['dataType']
statsDict['datPath'] = format_path((analysisInfo['datPath'].replace(
str(WebServerDataLocation), "<Data Root>/")).replace('//', '/'))
statsDict['timeseriesSaved'] = self._timeseriesSaved()
return statsDict
def analysisStatistics(self):
statsDict={}
s=self._eventStats()
statsDict['fractionNormal']=s[0]
statsDict['fractionWarn']=s[1]
statsDict['fractionError']=s[2]
statsDict['nTotal']=s[3]
c=self._caprate()
statsDict['captureRateMean']=c[0]
statsDict['captureRateSigma']=c[1]
t=self._timePerEvent()
statsDict['processTimePerEventMean']=t[0]
statsDict['processTimePerEventSigma']=t[1]
o=self._openChanCurrent()
statsDict['openChannelCurrentMean']=o[0]
statsDict['openChannelCurrentSigma']=o[1]
statsDict['analysisProgressPercent']=self._analysisProgress()
dbHnd=sqlite.sqlite3MDIO()
dbHnd.openDB(self.analysisDB)
analysisInfo=dbHnd.readAnalysisInfo()
procName=analysisStatistics.processingAlgorithmName
partName=analysisStatistics.partitionAlgorithmName
statsDict['partitionAlgorithm']=partName[analysisInfo['partitionAlgorithm']]
statsDict['processingAlgorithm']=procName[analysisInfo['processingAlgorithm']]
statsDict['FskHz']=analysisInfo['FsHz']/1000.
statsDict['dataType']=analysisInfo['dataType']
statsDict['datPath']=format_path((analysisInfo['datPath'].replace(str(WebServerDataLocation), "<Data Root>/")).replace('//', '/'))
statsDict['timeseriesSaved']=self._timeseriesSaved()
return statsDict
def onDataReady(results, errorstr):
print results
print errorstr
if __name__ == '__main__':
from mosaic.utilities.resource_path import resource_path
app = QtGui.QApplication(sys.argv)
dbfile = resource_path('eventMD-PEG29-Reference.sqlite')
q = "select filename, fileformat, modifiedtime from processedfiles"
c = sqlite.sqlite3MDIO()
c.openDB(dbfile)
print c.executeSQL(q)
c.closeDB()
thread = QtCore.QThread()
obj = sqlQueryWorker(dbfile)
obj.resultsReady.connect(onDataReady)
obj.dbColumnsReady.connect(OnDBColsReady)
obj.moveToThread(thread)
obj.finished.connect(thread.quit)
thread.start()
def OnDBColsReady(cols):
print cols
def onDataReady(results, errorstr):
print results
print errorstr
if __name__ == '__main__':
from mosaic.utilities.resource_path import resource_path
app = QtGui.QApplication(sys.argv)
dbfile=resource_path('eventMD-PEG29-Reference.sqlite')
q="select filename, fileformat, modifiedtime from processedfiles"
c=sqlite.sqlite3MDIO()
c.openDB(dbfile)
print c.executeSQL( q )
c.closeDB()
thread = QtCore.QThread()
obj = sqlQueryWorker(dbfile)
obj.resultsReady.connect(onDataReady)
obj.dbColumnsReady.connect(OnDBColsReady)
obj.moveToThread(thread)
obj.finished.connect(thread.quit)
thread.start()
import glob
import os
import numpy as np
import sqlite3
import mosaic.mdio.sqlite3MDIO as sql
import Tkinter as tk
import pandas as pd
import pandasql as sqldf
import tkFileDialog
from progress.bar import ChargingBar, Bar
db=sql.sqlite3MDIO()
pd.options.mode.chained_assignment = None
def do_Stuff(file_path_string,events_directory_path,directory_path,file_name):
db.openDB(glob.glob(file_path_string)[-1])
q = "SELECT * from metadata WHERE ProcessingStatus='normal'"
column_list = ['recIDX',
'ProcessingStatus',
'OpenChCurrent',
'NStates',
'CurrentStep',
'BlockDepth',
'EventStart',
'EventEnd',
'EventDelay',
'StateResTime',
'ResTime',
'RCConstant',
'AbsEventStart',
'ReducedChiSquared',
def openDBFile(self, dbfile): """ Open a specific database file. """ self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(dbfile)
def _trajLength(self, analysisDB): dbHnd=sqlite.sqlite3MDIO() dbHnd.openDB(analysisDB) return dbHnd.readAnalysisInfo()['dataLengthSec']
def openDBFile(self, dbfile, updateOnIdle=True): self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(dbfile) self._updateDBFields()
def __init__(self, trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings, settingsString, **kwargs):
"""
Initialize a new event segment object
"""
# Required arguments
self.trajDataObj=trajDataObj
self.eventProcHnd=eventProcHnd
# Reset function timer since esTimer is a class variable
partitionTimer.Reset()
self.settingsDict = eventPartitionSettings
self.eventProcSettingsDict = eventProcSettings
self.procTime=0.0
self.FsHz=self.trajDataObj.FsHz
self.DataLengthSec=self.trajDataObj.DataLengthSec
try:
self.writeEventTS=int(self.settingsDict.pop("writeEventTS",1))
self.parallelProc=int(self.settingsDict.pop("parallelProc",1))
self.reserveNCPU=int(self.settingsDict.pop("reserveNCPU",2))
self.driftThreshold=float(self.settingsDict.pop("driftThreshold",2.0))
self.maxDriftRate=float(self.settingsDict.pop("maxDriftRate",2.0))
self.minBaseline=float(self.settingsDict.pop("minBaseline",-1.))
self.maxBaseline=float(self.settingsDict.pop("maxBaseline",-1.))
except ValueError as err:
raise mosaic.commonExceptions.SettingsTypeError( err )
sys.stdout.flush()
self.tEventProcObj=self.eventProcHnd([], [], self.FsHz, eventstart=0,eventend=0, baselinestats=[ 0,0,0 ], algosettingsdict=self.eventProcSettingsDict.copy(), savets=False, absdatidx=0, datafileHnd=None )
self.mdioDBHnd=sqlite3MDIO.sqlite3MDIO()
self.mdioDBHnd.initDB(
dbPath=self.trajDataObj.datPath,
tableName='metadata',
colNames=(self.tEventProcObj.mdHeadings()),
colNames_t=(self.tEventProcObj.mdHeadingDataType()),
dbFilename=kwargs.get('dbFilename', '')
)
self.mdioDBHnd.writeSettings(settingsString)
self.logger=mlog.mosaicLogging().getLogger(name=__name__, dbHnd=self.mdioDBHnd)
self.logger.debug(_d("Event Segment Initialization"))
self.logger.debug(_d("{0}", settingsString))
if self.trajDataObj.dataFilter:
self.fstring=type(self.trajDataObj.dataFilterObj).__name__
else:
self.fstring='None'
self._writeanalysisinfo()
if self.parallelProc:
self._setupparallel()
# Setup function timing
self.timingObj=mosaicTiming.mosaicTiming()
self._init(trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings)
def _trajLength(self, analysisDB):
dbHnd = sqlite.sqlite3MDIO()
dbHnd.openDB(analysisDB)
return dbHnd.readAnalysisInfo()['dataLengthSec']
def openDBFile(self, dbfile):
"""
Open a specific database file.
"""
self.queryDatabase = sqlite.sqlite3MDIO()
self.queryDatabase.openDB(dbfile)
