def openDBFile(self, dbfile, FskHz, updateOnIdle=True): self.queryDatabase = sqlite.sqlite3MDIO() self.queryDatabase.openDB(dbfile) # self update on idle flag self.updateDataOnIdle = updateOnIdle # Store the analysis algorithm try: self.analysisAlgorithm = str( self.queryDatabase.readAnalysisInfo()['processingAlgorithm']) except: # If the database doesn't have information on the alogirthm type, # default to adept2State self.analysisAlgorithm = "adept2State" try: # Generate the query string based on the algorithm in the database self.queryString = self.queryStringDict[self.analysisAlgorithm] # Setup the fit function based on the algorithm self.fitFuncHnd = self.fitFuncHndDict[self.analysisAlgorithm] self.fitFuncArgs = self.fitFuncArgsDict[self.analysisAlgorithm] self.stepFuncHnd = self.stepFuncHndDict[self.analysisAlgorithm] self.stepFuncArgs = self.stepFuncArgsDict[self.analysisAlgorithm] self.bdFuncArgs = self.blockDepthArgsDict[self.analysisAlgorithm] except KeyError: from mosaic.settings import __legacy_settings__ legacyAlgo = __legacy_settings__[self.analysisAlgorithm] self.queryString = self.queryStringDict[legacyAlgo] self.fitFuncHnd = self.fitFuncHndDict[legacyAlgo] self.fitFuncArgs = self.fitFuncArgsDict[legacyAlgo] self.stepFuncHnd = self.stepFuncHndDict[legacyAlgo] self.stepFuncArgs = self.stepFuncArgsDict[legacyAlgo] self.bdFuncArgs = self.blockDepthArgsDict[legacyAlgo] # self.fitFuncHnd=None # self.fitFuncArgs="[]" # self.stepFuncHnd=None # self.stepFuncArgs="[]" self.FskHz = float(FskHz) # print self.FskHz self._updatequery() self.update_graph() self.eventIndexLineEdit.setText(str(self.eventIndex)) # Idle processing QtCore.QObject.connect(self.idleTimer, QtCore.SIGNAL('timeout()'), self.OnAppIdle)
def csv(self): dbHnd=sqlite.sqlite3MDIO() dbHnd.openDB(self.AnalysisDBFile) self.responseDict["dbName"]=self.AnalysisDBFile.split(path_separator())[-1].split('.')[0] self.responseDict["dbData"]=base64.b64encode(dbHnd.csvString(self.queryString)) return self.responseDict
def openDBFile(self, dbfile, FskHz, updateOnIdle=True): self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(dbfile) # self update on idle flag self.updateDataOnIdle=updateOnIdle # Store the analysis algorithm try: self.analysisAlgorithm=str(self.queryDatabase.readAnalysisInfo()['processingAlgorithm']) except: # If the database doesn't have information on the alogirthm type, # default to adept2State self.analysisAlgorithm="adept2State" try: # Generate the query string based on the algorithm in the database self.queryString=self.queryStringDict[self.analysisAlgorithm] # Setup the fit function based on the algorithm self.fitFuncHnd=self.fitFuncHndDict[self.analysisAlgorithm] self.fitFuncArgs=self.fitFuncArgsDict[self.analysisAlgorithm] self.stepFuncHnd=self.stepFuncHndDict[self.analysisAlgorithm] self.stepFuncArgs=self.stepFuncArgsDict[self.analysisAlgorithm] self.bdFuncArgs=self.blockDepthArgsDict[self.analysisAlgorithm] except KeyError: from mosaic.settings import __legacy_settings__ legacyAlgo=__legacy_settings__[self.analysisAlgorithm] self.queryString=self.queryStringDict[legacyAlgo] self.fitFuncHnd=self.fitFuncHndDict[legacyAlgo] self.fitFuncArgs=self.fitFuncArgsDict[legacyAlgo] self.stepFuncHnd=self.stepFuncHndDict[legacyAlgo] self.stepFuncArgs=self.stepFuncArgsDict[legacyAlgo] self.bdFuncArgs=self.blockDepthArgsDict[legacyAlgo] # self.fitFuncHnd=None # self.fitFuncArgs="[]" # self.stepFuncHnd=None # self.stepFuncArgs="[]" self.FskHz=float(FskHz) # print self.FskHz self._updatequery() self.update_graph() self.eventIndexLineEdit.setText(str(self.eventIndex)) # Idle processing QtCore.QObject.connect(self.idleTimer, QtCore.SIGNAL('timeout()'), self.OnAppIdle)
def dbColumnNames(self, filterRealList): self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile, timeout=3.0) if filterRealList: self.dbColumnsReady.emit( [ col[0] for col in zip( self.queryDatabase.mdColumnNames, self.queryDatabase.mdColumnTypes ) if col[1] != 'REAL_LIST' ] ) else: self.dbColumnsReady.emit(self.queryDatabase.mdColumnNames) self.queryDatabase.closeDB() self.finished.emit()
def __init__(self, trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings, settingsString): """ Initialize a new event segment object """ # Required arguments self.trajDataObj=trajDataObj self.eventProcHnd=eventProcHnd # Reset function timer since esTimer is a class variable partitionTimer.Reset() self.settingsDict = eventPartitionSettings self.eventProcSettingsDict = eventProcSettings self.procTime=0.0 self.FsHz=self.trajDataObj.FsHz self.DataLengthSec=self.trajDataObj.DataLengthSec try: self.writeEventTS=int(self.settingsDict.pop("writeEventTS",1)) self.parallelProc=int(self.settingsDict.pop("parallelProc",1)) self.reserveNCPU=int(self.settingsDict.pop("reserveNCPU",2)) except ValueError as err: raise mosaic.commonExceptions.SettingsTypeError( err ) sys.stdout.flush() self.tEventProcObj=self.eventProcHnd([], self.FsHz, eventstart=0,eventend=0, baselinestats=[ 0,0,0 ], algosettingsdict=self.eventProcSettingsDict.copy(), savets=False, absdatidx=0, datafileHnd=None ) self.mdioDBHnd=sqlite3MDIO.sqlite3MDIO() self.mdioDBHnd.initDB( dbPath=self.trajDataObj.datPath, tableName='metadata', colNames=(self.tEventProcObj.mdHeadings()), colNames_t=(self.tEventProcObj.mdHeadingDataType()) ) self.mdioDBHnd.writeSettings(settingsString) self.logger=mlog.mosaicLogging().getLogger(name=__name__, dbHnd=self.mdioDBHnd) self.logger.debug(_d("Event Segment Initialization")) self.logger.debug(_d("{0}", settingsString)) if self.trajDataObj.dataFilter: self.fstring=type(self.trajDataObj.dataFilterObj).__name__ else: self.fstring='None' self._writeanalysisinfo() if self.parallelProc: self._setupparallel() # Setup function timing self.timingObj=mosaicTiming.mosaicTiming() self._init(trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings)
def main(): args=parseCLArgs() db=sqlite3MDIO.sqlite3MDIO() db.openDB(args.dbname) if args.raw: return [ list(l) for l in db.rawQuery(args.query) ] else: return db.queryDB(args.query)
def query(dbname, query_str): """ Simple wrapper to perform a query on a MOSAIC database. """ db=sql.sqlite3MDIO() db.openDB(dbname) q=db.queryDB(query_str) db.closeDB() return q
def _dbInfo(dbname): db = sqlite.sqlite3MDIO() db.openDB(dbname) info = db.readAnalysisInfo() settings = db.readSettings() db.closeDB() return info, settings
def csv(self): dbHnd = sqlite.sqlite3MDIO() dbHnd.openDB(self.AnalysisDBFile) self.responseDict["dbName"] = self.AnalysisDBFile.split( path_separator())[-1].split('.')[0] self.responseDict["dbData"] = base64.b64encode( dbHnd.csvString(self.queryString)) return self.responseDict
def query(dbname, query_str): """ Simple wrapper to perform a query on a MOSAIC database. """ db = sql.sqlite3MDIO() db.openDB(dbname) q = db.queryDB(query_str) db.closeDB() return q
def main(): args = parseCLArgs() db = sqlite3MDIO.sqlite3MDIO() db.openDB(args.dbname) if args.raw: return [list(l) for l in db.rawQuery(args.query)] else: return db.queryDB(args.query)
def _dbInfo(dbname): db=sqlite.sqlite3MDIO() db.openDB(dbname) info=db.readAnalysisInfo() settings=db.readSettings() db.closeDB() return info, settings
def queryDB(self, q): try: self.queryDatabase = sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile) self.resultsReady.emit(self.queryDatabase.queryDB(str(q)), "") self.queryDatabase.closeDB() except sqlite3.OperationalError, err: self.queryDatabase.closeDB() self.resultsReady.emit([], str(err))
def queryDB(self, q): try: self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile) self.resultsReady.emit(self.queryDatabase.queryDB(str(q)), "") self.queryDatabase.closeDB() except sqlite3.OperationalError, err: self.queryDatabase.closeDB() self.resultsReady.emit([], str(err))
def dbColumnNames(self, filterRealList): self.queryDatabase = sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile, timeout=3.0) if filterRealList: self.dbColumnsReady.emit([ col[0] for col in zip(self.queryDatabase.mdColumnNames, self.queryDatabase.mdColumnTypes) if col[1] != 'REAL_LIST' ]) else: self.dbColumnsReady.emit(self.queryDatabase.mdColumnNames) self.queryDatabase.closeDB() self.finished.emit()
def queryDB2(self, q1, q2): try: self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile) r1=self.queryDatabase.queryDB(str(q1)) r2=self.queryDatabase.queryDB(str(q2)) self.resultsReady2.emit(r1, r2, "") self.queryDatabase.closeDB() except sqlite3.OperationalError, err: self.queryDatabase.closeDB() self.resultsReady.emit([], str(err))
def queryDB2(self, q1, q2): try: self.queryDatabase = sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile) r1 = self.queryDatabase.queryDB(str(q1)) r2 = self.queryDatabase.queryDB(str(q2)) self.resultsReady2.emit(r1, r2, "") self.queryDatabase.closeDB() except sqlite3.OperationalError, err: self.queryDatabase.closeDB() self.resultsReady.emit([], str(err))
def __init__(self, dbFile, qstr, bins, showContours): self.AnalysisDBFile = dbFile self.numBins = bins dbHnd = sqlite.sqlite3MDIO() dbHnd.openDB(self.AnalysisDBFile) analysisInfo = dbHnd.readAnalysisInfo() self.processingAlgorithm = analysisInfo['processingAlgorithm'] self.queryString = self._queryString(qstr) if showContours: self.plotType = "contour" else: self.plotType = "heatmap" self.responseDict = {}
def __init__(self, dbFile, qstr, bins, showContours): self.AnalysisDBFile = dbFile self.numBins=bins dbHnd=sqlite.sqlite3MDIO() dbHnd.openDB(self.AnalysisDBFile) analysisInfo=dbHnd.readAnalysisInfo() self.processingAlgorithm=analysisInfo['processingAlgorithm'] self.queryString=self._queryString(qstr) if showContours: self.plotType="contour" else: self.plotType="heatmap" self.responseDict={}
def executeSQL(self, q): try: # t1=time.time() self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile) # t1a=time.time() self.resultsReady.emit(self.queryDatabase.executeSQL(str(q)), "") # t2a=time.time() self.queryDatabase.closeDB() # t2=time.time() # print "query time = ", t2a-t1a # print "overhead = ", (t2-t1)-(t2a-t1a) # print "total time = ", t2-t1 except sqlite3.OperationalError, err: self.queryDatabase.closeDB() self.resultsReady.emit([], str(err))
def executeSQL(self, q): try: # t1=time.time() self.queryDatabase = sqlite.sqlite3MDIO() self.queryDatabase.openDB(self.dbFile) # t1a=time.time() self.resultsReady.emit(self.queryDatabase.executeSQL(str(q)), "") # t2a=time.time() self.queryDatabase.closeDB() # t2=time.time() # print "query time = ", t2a-t1a # print "overhead = ", (t2-t1)-(t2a-t1a) # print "total time = ", t2-t1 except sqlite3.OperationalError, err: self.queryDatabase.closeDB() self.resultsReady.emit([], str(err))
def openDBFile(self, dbfile, updateOnIdle=True): """ Open a specific database file. """ # Create an index to speed up queries self._createDBIndex(dbfile) self.qWorker = sqlworker.sqlQueryWorker(dbfile) self.dbFile = dbfile # Open a handle to the database self.dbHnd = sqlite.sqlite3MDIO() self.dbHnd.openDB(self.dbFile) # set the length of the trajectory in sec. self.trajLength = self.dbHnd.readAnalysisInfo()[-1] # Connect signals and slots self.qWorker.resultsReady2.connect(self.OnDataReady) self.qWorker.moveToThread(self.qThread) self.qWorker.finished.connect(self.qThread.quit) self.qThread.start() # reset elapsed time self.analysisTime = 0.0 # reset wall time and analysis start time self.wallTime = 0.0 self.startTime = time.time() # self update on idle flag self.updateDataOnIdle = updateOnIdle # Query the DB self._updatequery() # Idle processing QtCore.QObject.connect(self.idleTimer, QtCore.SIGNAL('timeout()'), self.OnAppIdle)
def openDBFile(self, dbfile, updateOnIdle=True): """ Open a specific database file. """ # Create an index to speed up queries self._createDBIndex(dbfile) self.qWorker=sqlworker.sqlQueryWorker(dbfile) self.dbFile=dbfile # Open a handle to the database self.dbHnd=sqlite.sqlite3MDIO() self.dbHnd.openDB(self.dbFile) # set the length of the trajectory in sec. self.trajLength=self.dbHnd.readAnalysisInfo()[-1] # Connect signals and slots self.qWorker.resultsReady2.connect(self.OnDataReady) self.qWorker.moveToThread(self.qThread) self.qWorker.finished.connect(self.qThread.quit) self.qThread.start() # reset elapsed time self.analysisTime=0.0 # reset wall time and analysis start time self.wallTime=0.0 self.startTime=time.time() # self update on idle flag self.updateDataOnIdle=updateOnIdle # Query the DB self._updatequery() # Idle processing QtCore.QObject.connect(self.idleTimer, QtCore.SIGNAL('timeout()'), self.OnAppIdle)
def analysisStatistics(self): statsDict = {} s = self._eventStats() statsDict['fractionNormal'] = s[0] statsDict['fractionWarn'] = s[1] statsDict['fractionError'] = s[2] statsDict['nTotal'] = s[3] c = self._caprate() statsDict['captureRateMean'] = c[0] statsDict['captureRateSigma'] = c[1] t = self._timePerEvent() statsDict['processTimePerEventMean'] = t[0] statsDict['processTimePerEventSigma'] = t[1] o = self._openChanCurrent() statsDict['openChannelCurrentMean'] = o[0] statsDict['openChannelCurrentSigma'] = o[1] statsDict['analysisProgressPercent'] = self._analysisProgress() dbHnd = sqlite.sqlite3MDIO() dbHnd.openDB(self.analysisDB) analysisInfo = dbHnd.readAnalysisInfo() procName = analysisStatistics.processingAlgorithmName partName = analysisStatistics.partitionAlgorithmName statsDict['partitionAlgorithm'] = partName[ analysisInfo['partitionAlgorithm']] statsDict['processingAlgorithm'] = procName[ analysisInfo['processingAlgorithm']] statsDict['FskHz'] = analysisInfo['FsHz'] / 1000. statsDict['dataType'] = analysisInfo['dataType'] statsDict['datPath'] = format_path((analysisInfo['datPath'].replace( str(WebServerDataLocation), "<Data Root>/")).replace('//', '/')) statsDict['timeseriesSaved'] = self._timeseriesSaved() return statsDict
def analysisStatistics(self): statsDict={} s=self._eventStats() statsDict['fractionNormal']=s[0] statsDict['fractionWarn']=s[1] statsDict['fractionError']=s[2] statsDict['nTotal']=s[3] c=self._caprate() statsDict['captureRateMean']=c[0] statsDict['captureRateSigma']=c[1] t=self._timePerEvent() statsDict['processTimePerEventMean']=t[0] statsDict['processTimePerEventSigma']=t[1] o=self._openChanCurrent() statsDict['openChannelCurrentMean']=o[0] statsDict['openChannelCurrentSigma']=o[1] statsDict['analysisProgressPercent']=self._analysisProgress() dbHnd=sqlite.sqlite3MDIO() dbHnd.openDB(self.analysisDB) analysisInfo=dbHnd.readAnalysisInfo() procName=analysisStatistics.processingAlgorithmName partName=analysisStatistics.partitionAlgorithmName statsDict['partitionAlgorithm']=partName[analysisInfo['partitionAlgorithm']] statsDict['processingAlgorithm']=procName[analysisInfo['processingAlgorithm']] statsDict['FskHz']=analysisInfo['FsHz']/1000. statsDict['dataType']=analysisInfo['dataType'] statsDict['datPath']=format_path((analysisInfo['datPath'].replace(str(WebServerDataLocation), "<Data Root>/")).replace('//', '/')) statsDict['timeseriesSaved']=self._timeseriesSaved() return statsDict
def onDataReady(results, errorstr): print results print errorstr if __name__ == '__main__': from mosaic.utilities.resource_path import resource_path app = QtGui.QApplication(sys.argv) dbfile = resource_path('eventMD-PEG29-Reference.sqlite') q = "select filename, fileformat, modifiedtime from processedfiles" c = sqlite.sqlite3MDIO() c.openDB(dbfile) print c.executeSQL(q) c.closeDB() thread = QtCore.QThread() obj = sqlQueryWorker(dbfile) obj.resultsReady.connect(onDataReady) obj.dbColumnsReady.connect(OnDBColsReady) obj.moveToThread(thread) obj.finished.connect(thread.quit) thread.start()
def OnDBColsReady(cols): print cols def onDataReady(results, errorstr): print results print errorstr if __name__ == '__main__': from mosaic.utilities.resource_path import resource_path app = QtGui.QApplication(sys.argv) dbfile=resource_path('eventMD-PEG29-Reference.sqlite') q="select filename, fileformat, modifiedtime from processedfiles" c=sqlite.sqlite3MDIO() c.openDB(dbfile) print c.executeSQL( q ) c.closeDB() thread = QtCore.QThread() obj = sqlQueryWorker(dbfile) obj.resultsReady.connect(onDataReady) obj.dbColumnsReady.connect(OnDBColsReady) obj.moveToThread(thread) obj.finished.connect(thread.quit) thread.start()
import glob import os import numpy as np import sqlite3 import mosaic.mdio.sqlite3MDIO as sql import Tkinter as tk import pandas as pd import pandasql as sqldf import tkFileDialog from progress.bar import ChargingBar, Bar db=sql.sqlite3MDIO() pd.options.mode.chained_assignment = None def do_Stuff(file_path_string,events_directory_path,directory_path,file_name): db.openDB(glob.glob(file_path_string)[-1]) q = "SELECT * from metadata WHERE ProcessingStatus='normal'" column_list = ['recIDX', 'ProcessingStatus', 'OpenChCurrent', 'NStates', 'CurrentStep', 'BlockDepth', 'EventStart', 'EventEnd', 'EventDelay', 'StateResTime', 'ResTime', 'RCConstant', 'AbsEventStart', 'ReducedChiSquared',
def openDBFile(self, dbfile): """ Open a specific database file. """ self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(dbfile)
def _trajLength(self, analysisDB): dbHnd=sqlite.sqlite3MDIO() dbHnd.openDB(analysisDB) return dbHnd.readAnalysisInfo()['dataLengthSec']
def openDBFile(self, dbfile, updateOnIdle=True): self.queryDatabase=sqlite.sqlite3MDIO() self.queryDatabase.openDB(dbfile) self._updateDBFields()
def __init__(self, trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings, settingsString, **kwargs): """ Initialize a new event segment object """ # Required arguments self.trajDataObj=trajDataObj self.eventProcHnd=eventProcHnd # Reset function timer since esTimer is a class variable partitionTimer.Reset() self.settingsDict = eventPartitionSettings self.eventProcSettingsDict = eventProcSettings self.procTime=0.0 self.FsHz=self.trajDataObj.FsHz self.DataLengthSec=self.trajDataObj.DataLengthSec try: self.writeEventTS=int(self.settingsDict.pop("writeEventTS",1)) self.parallelProc=int(self.settingsDict.pop("parallelProc",1)) self.reserveNCPU=int(self.settingsDict.pop("reserveNCPU",2)) self.driftThreshold=float(self.settingsDict.pop("driftThreshold",2.0)) self.maxDriftRate=float(self.settingsDict.pop("maxDriftRate",2.0)) self.minBaseline=float(self.settingsDict.pop("minBaseline",-1.)) self.maxBaseline=float(self.settingsDict.pop("maxBaseline",-1.)) except ValueError as err: raise mosaic.commonExceptions.SettingsTypeError( err ) sys.stdout.flush() self.tEventProcObj=self.eventProcHnd([], [], self.FsHz, eventstart=0,eventend=0, baselinestats=[ 0,0,0 ], algosettingsdict=self.eventProcSettingsDict.copy(), savets=False, absdatidx=0, datafileHnd=None ) self.mdioDBHnd=sqlite3MDIO.sqlite3MDIO() self.mdioDBHnd.initDB( dbPath=self.trajDataObj.datPath, tableName='metadata', colNames=(self.tEventProcObj.mdHeadings()), colNames_t=(self.tEventProcObj.mdHeadingDataType()), dbFilename=kwargs.get('dbFilename', '') ) self.mdioDBHnd.writeSettings(settingsString) self.logger=mlog.mosaicLogging().getLogger(name=__name__, dbHnd=self.mdioDBHnd) self.logger.debug(_d("Event Segment Initialization")) self.logger.debug(_d("{0}", settingsString)) if self.trajDataObj.dataFilter: self.fstring=type(self.trajDataObj.dataFilterObj).__name__ else: self.fstring='None' self._writeanalysisinfo() if self.parallelProc: self._setupparallel() # Setup function timing self.timingObj=mosaicTiming.mosaicTiming() self._init(trajDataObj, eventProcHnd, eventPartitionSettings, eventProcSettings)
def _trajLength(self, analysisDB): dbHnd = sqlite.sqlite3MDIO() dbHnd.openDB(analysisDB) return dbHnd.readAnalysisInfo()['dataLengthSec']
def openDBFile(self, dbfile): """ Open a specific database file. """ self.queryDatabase = sqlite.sqlite3MDIO() self.queryDatabase.openDB(dbfile)